xref: /freebsd/sys/compat/linux/linux_futex.c (revision 9268022b74279434ed6300244e3f977e56a8ceb5)
1 /*	$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $ */
2 
3 /*-
4  * Copyright (c) 2005 Emmanuel Dreyfus, all rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. All advertising materials mentioning features or use of this software
15  *    must display the following acknowledgement:
16  *	This product includes software developed by Emmanuel Dreyfus
17  * 4. The name of the author may not be used to endorse or promote
18  *    products derived from this software without specific prior written
19  *    permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE THE AUTHOR AND CONTRIBUTORS ``AS IS''
22  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
23  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31  * POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 #if 0
37 __KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $");
38 #endif
39 
40 #include "opt_compat.h"
41 
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/imgact.h>
45 #include <sys/kernel.h>
46 #include <sys/ktr.h>
47 #include <sys/lock.h>
48 #include <sys/malloc.h>
49 #include <sys/mutex.h>
50 #include <sys/priv.h>
51 #include <sys/proc.h>
52 #include <sys/queue.h>
53 #include <sys/sched.h>
54 #include <sys/sdt.h>
55 #include <sys/sx.h>
56 #include <sys/umtx.h>
57 
58 #ifdef COMPAT_LINUX32
59 #include <machine/../linux32/linux.h>
60 #include <machine/../linux32/linux32_proto.h>
61 #else
62 #include <machine/../linux/linux.h>
63 #include <machine/../linux/linux_proto.h>
64 #endif
65 #include <compat/linux/linux_dtrace.h>
66 #include <compat/linux/linux_emul.h>
67 #include <compat/linux/linux_futex.h>
68 #include <compat/linux/linux_util.h>
69 
70 /* DTrace init */
71 LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
72 
73 /* Linuxulator-global DTrace probes */
74 LIN_SDT_PROBE_DECLARE(locks, emul_lock, locked);
75 LIN_SDT_PROBE_DECLARE(locks, emul_lock, unlock);
76 
77 /**
78  * Futex part for the special DTrace module "locks".
79  */
80 LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, locked, "struct mtx *");
81 LIN_SDT_PROBE_DEFINE1(locks, futex_mtx, unlock, "struct mtx *");
82 
83 /**
84  * Per futex probes.
85  */
86 LIN_SDT_PROBE_DEFINE1(futex, futex, create, "struct sx *");
87 LIN_SDT_PROBE_DEFINE1(futex, futex, destroy, "struct sx *");
88 
89 /**
90  * DTrace probes in this module.
91  */
92 LIN_SDT_PROBE_DEFINE2(futex, futex_put, entry, "struct futex *",
93     "struct waiting_proc *");
94 LIN_SDT_PROBE_DEFINE3(futex, futex_put, destroy, "uint32_t *", "uint32_t",
95     "int");
96 LIN_SDT_PROBE_DEFINE3(futex, futex_put, unlock, "uint32_t *", "uint32_t",
97     "int");
98 LIN_SDT_PROBE_DEFINE0(futex, futex_put, return);
99 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, entry, "uint32_t *", "struct futex **",
100     "uint32_t");
101 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, umtx_key_get_error, "int");
102 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, shared, "uint32_t *", "uint32_t",
103     "int");
104 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, null, "uint32_t *");
105 LIN_SDT_PROBE_DEFINE3(futex, futex_get0, new, "uint32_t *", "uint32_t", "int");
106 LIN_SDT_PROBE_DEFINE1(futex, futex_get0, return, "int");
107 LIN_SDT_PROBE_DEFINE3(futex, futex_get, entry, "uint32_t *",
108     "struct waiting_proc **", "struct futex **");
109 LIN_SDT_PROBE_DEFINE0(futex, futex_get, error);
110 LIN_SDT_PROBE_DEFINE1(futex, futex_get, return, "int");
111 LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, entry, "struct futex *",
112     "struct waiting_proc **", "int");
113 LIN_SDT_PROBE_DEFINE5(futex, futex_sleep, requeue_error, "int", "uint32_t *",
114     "struct waiting_proc *", "uint32_t *", "uint32_t");
115 LIN_SDT_PROBE_DEFINE3(futex, futex_sleep, sleep_error, "int", "uint32_t *",
116     "struct waiting_proc *");
117 LIN_SDT_PROBE_DEFINE1(futex, futex_sleep, return, "int");
118 LIN_SDT_PROBE_DEFINE3(futex, futex_wake, entry, "struct futex *", "int",
119     "uint32_t");
120 LIN_SDT_PROBE_DEFINE3(futex, futex_wake, iterate, "uint32_t",
121     "struct waiting_proc *", "uint32_t");
122 LIN_SDT_PROBE_DEFINE1(futex, futex_wake, wakeup, "struct waiting_proc *");
123 LIN_SDT_PROBE_DEFINE1(futex, futex_wake, return, "int");
124 LIN_SDT_PROBE_DEFINE4(futex, futex_requeue, entry, "struct futex *", "int",
125     "struct futex *", "int");
126 LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, wakeup, "struct waiting_proc *");
127 LIN_SDT_PROBE_DEFINE3(futex, futex_requeue, requeue, "uint32_t *",
128     "struct waiting_proc *", "uint32_t");
129 LIN_SDT_PROBE_DEFINE1(futex, futex_requeue, return, "int");
130 LIN_SDT_PROBE_DEFINE4(futex, futex_wait, entry, "struct futex *",
131     "struct waiting_proc **", "int", "uint32_t");
132 LIN_SDT_PROBE_DEFINE1(futex, futex_wait, sleep_error, "int");
133 LIN_SDT_PROBE_DEFINE1(futex, futex_wait, return, "int");
134 LIN_SDT_PROBE_DEFINE3(futex, futex_atomic_op, entry, "struct thread *",
135     "int", "uint32_t");
136 LIN_SDT_PROBE_DEFINE4(futex, futex_atomic_op, decoded_op, "int", "int", "int",
137     "int");
138 LIN_SDT_PROBE_DEFINE0(futex, futex_atomic_op, missing_access_check);
139 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_op, "int");
140 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, unimplemented_cmp, "int");
141 LIN_SDT_PROBE_DEFINE1(futex, futex_atomic_op, return, "int");
142 LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, entry, "struct thread *",
143     "struct linux_sys_futex_args *");
144 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_clockswitch);
145 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, itimerfix_error, "int");
146 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, copyin_error, "int");
147 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, invalid_cmp_requeue_use);
148 LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wait, "uint32_t *",
149     "uint32_t", "uint32_t");
150 LIN_SDT_PROBE_DEFINE4(futex, linux_sys_futex, debug_wait_value_neq,
151     "uint32_t *", "uint32_t", "int", "uint32_t");
152 LIN_SDT_PROBE_DEFINE3(futex, linux_sys_futex, debug_wake, "uint32_t *",
153     "uint32_t", "uint32_t");
154 LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_cmp_requeue, "uint32_t *",
155     "uint32_t", "uint32_t", "uint32_t *", "struct l_timespec *");
156 LIN_SDT_PROBE_DEFINE2(futex, linux_sys_futex, debug_cmp_requeue_value_neq,
157     "uint32_t", "int");
158 LIN_SDT_PROBE_DEFINE5(futex, linux_sys_futex, debug_wake_op, "uint32_t *",
159     "int", "uint32_t", "uint32_t *", "uint32_t");
160 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unhandled_efault);
161 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_lock_pi);
162 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_unlock_pi);
163 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_trylock_pi);
164 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, deprecated_requeue);
165 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_wait_requeue_pi);
166 LIN_SDT_PROBE_DEFINE0(futex, linux_sys_futex, unimplemented_cmp_requeue_pi);
167 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, unknown_operation, "int");
168 LIN_SDT_PROBE_DEFINE1(futex, linux_sys_futex, return, "int");
169 LIN_SDT_PROBE_DEFINE2(futex, linux_set_robust_list, entry, "struct thread *",
170     "struct linux_set_robust_list_args *");
171 LIN_SDT_PROBE_DEFINE0(futex, linux_set_robust_list, size_error);
172 LIN_SDT_PROBE_DEFINE1(futex, linux_set_robust_list, return, "int");
173 LIN_SDT_PROBE_DEFINE2(futex, linux_get_robust_list, entry, "struct thread *",
174     "struct linux_get_robust_list_args *");
175 LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, copyout_error, "int");
176 LIN_SDT_PROBE_DEFINE1(futex, linux_get_robust_list, return, "int");
177 LIN_SDT_PROBE_DEFINE3(futex, handle_futex_death, entry, "struct proc *",
178     "uint32_t *", "int");
179 LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, copyin_error, "int");
180 LIN_SDT_PROBE_DEFINE1(futex, handle_futex_death, return, "int");
181 LIN_SDT_PROBE_DEFINE3(futex, fetch_robust_entry, entry,
182     "struct linux_robust_list **", "struct linux_robust_list **", "int *");
183 LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, copyin_error, "int");
184 LIN_SDT_PROBE_DEFINE1(futex, fetch_robust_entry, return, "int");
185 LIN_SDT_PROBE_DEFINE1(futex, release_futexes, entry, "struct proc *");
186 LIN_SDT_PROBE_DEFINE1(futex, release_futexes, copyin_error, "int");
187 LIN_SDT_PROBE_DEFINE0(futex, release_futexes, return);
188 
189 static MALLOC_DEFINE(M_FUTEX, "futex", "Linux futexes");
190 static MALLOC_DEFINE(M_FUTEX_WP, "futex wp", "Linux futexes wp");
191 
192 struct futex;
193 
194 struct waiting_proc {
195 	uint32_t	wp_flags;
196 	struct futex	*wp_futex;
197 	TAILQ_ENTRY(waiting_proc) wp_list;
198 };
199 
200 struct futex {
201 	struct sx	f_lck;
202 	uint32_t	*f_uaddr;	/* user-supplied value, for debug */
203 	struct umtx_key	f_key;
204 	uint32_t	f_refcount;
205 	uint32_t	f_bitset;
206 	LIST_ENTRY(futex) f_list;
207 	TAILQ_HEAD(lf_waiting_proc, waiting_proc) f_waiting_proc;
208 };
209 
210 struct futex_list futex_list;
211 
212 #define FUTEX_LOCK(f)		sx_xlock(&(f)->f_lck)
213 #define FUTEX_UNLOCK(f)		sx_xunlock(&(f)->f_lck)
214 #define FUTEX_INIT(f)		do { \
215 				    sx_init_flags(&(f)->f_lck, "ftlk", \
216 					SX_DUPOK); \
217 				    LIN_SDT_PROBE1(futex, futex, create, \
218 					&(f)->f_lck); \
219 				} while (0)
220 #define FUTEX_DESTROY(f)	do { \
221 				    LIN_SDT_PROBE1(futex, futex, destroy, \
222 					&(f)->f_lck); \
223 				    sx_destroy(&(f)->f_lck); \
224 				} while (0)
225 #define FUTEX_ASSERT_LOCKED(f)	sx_assert(&(f)->f_lck, SA_XLOCKED)
226 
227 struct mtx futex_mtx;			/* protects the futex list */
228 #define FUTEXES_LOCK		do { \
229 				    mtx_lock(&futex_mtx); \
230 				    LIN_SDT_PROBE1(locks, futex_mtx, \
231 					locked, &futex_mtx); \
232 				} while (0)
233 #define FUTEXES_UNLOCK		do { \
234 				    LIN_SDT_PROBE1(locks, futex_mtx, \
235 					unlock, &futex_mtx); \
236 				    mtx_unlock(&futex_mtx); \
237 				} while (0)
238 
239 /* flags for futex_get() */
240 #define FUTEX_CREATE_WP		0x1	/* create waiting_proc */
241 #define FUTEX_DONTCREATE	0x2	/* don't create futex if not exists */
242 #define FUTEX_DONTEXISTS	0x4	/* return EINVAL if futex exists */
243 #define	FUTEX_SHARED		0x8	/* shared futex */
244 
245 /* wp_flags */
246 #define FUTEX_WP_REQUEUED	0x1	/* wp requeued - wp moved from wp_list
247 					 * of futex where thread sleep to wp_list
248 					 * of another futex.
249 					 */
250 #define FUTEX_WP_REMOVED	0x2	/* wp is woken up and removed from futex
251 					 * wp_list to prevent double wakeup.
252 					 */
253 
254 /* support.s */
255 int futex_xchgl(int oparg, uint32_t *uaddr, int *oldval);
256 int futex_addl(int oparg, uint32_t *uaddr, int *oldval);
257 int futex_orl(int oparg, uint32_t *uaddr, int *oldval);
258 int futex_andl(int oparg, uint32_t *uaddr, int *oldval);
259 int futex_xorl(int oparg, uint32_t *uaddr, int *oldval);
260 
261 static void
262 futex_put(struct futex *f, struct waiting_proc *wp)
263 {
264 	LIN_SDT_PROBE2(futex, futex_put, entry, f, wp);
265 
266 	FUTEX_ASSERT_LOCKED(f);
267 	if (wp != NULL) {
268 		if ((wp->wp_flags & FUTEX_WP_REMOVED) == 0)
269 			TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
270 		free(wp, M_FUTEX_WP);
271 	}
272 
273 	FUTEXES_LOCK;
274 	if (--f->f_refcount == 0) {
275 		LIST_REMOVE(f, f_list);
276 		FUTEXES_UNLOCK;
277 		FUTEX_UNLOCK(f);
278 
279 		LIN_SDT_PROBE3(futex, futex_put, destroy, f->f_uaddr,
280 		    f->f_refcount, f->f_key.shared);
281 		LINUX_CTR3(sys_futex, "futex_put destroy uaddr %p ref %d "
282 		    "shared %d", f->f_uaddr, f->f_refcount, f->f_key.shared);
283 		umtx_key_release(&f->f_key);
284 		FUTEX_DESTROY(f);
285 		free(f, M_FUTEX);
286 
287 		LIN_SDT_PROBE0(futex, futex_put, return);
288 		return;
289 	}
290 
291 	LIN_SDT_PROBE3(futex, futex_put, unlock, f->f_uaddr, f->f_refcount,
292 	    f->f_key.shared);
293 	LINUX_CTR3(sys_futex, "futex_put uaddr %p ref %d shared %d",
294 	    f->f_uaddr, f->f_refcount, f->f_key.shared);
295 	FUTEXES_UNLOCK;
296 	FUTEX_UNLOCK(f);
297 
298 	LIN_SDT_PROBE0(futex, futex_put, return);
299 }
300 
301 static int
302 futex_get0(uint32_t *uaddr, struct futex **newf, uint32_t flags)
303 {
304 	struct futex *f, *tmpf;
305 	struct umtx_key key;
306 	int error;
307 
308 	LIN_SDT_PROBE3(futex, futex_get0, entry, uaddr, newf, flags);
309 
310 	*newf = tmpf = NULL;
311 
312 	error = umtx_key_get(uaddr, TYPE_FUTEX, (flags & FUTEX_SHARED) ?
313 	    AUTO_SHARE : THREAD_SHARE, &key);
314 	if (error) {
315 		LIN_SDT_PROBE1(futex, futex_get0, umtx_key_get_error, error);
316 		LIN_SDT_PROBE1(futex, futex_get0, return, error);
317 		return (error);
318 	}
319 retry:
320 	FUTEXES_LOCK;
321 	LIST_FOREACH(f, &futex_list, f_list) {
322 		if (umtx_key_match(&f->f_key, &key)) {
323 			if (tmpf != NULL) {
324 				FUTEX_UNLOCK(tmpf);
325 				FUTEX_DESTROY(tmpf);
326 				free(tmpf, M_FUTEX);
327 			}
328 			if (flags & FUTEX_DONTEXISTS) {
329 				FUTEXES_UNLOCK;
330 				umtx_key_release(&key);
331 
332 				LIN_SDT_PROBE1(futex, futex_get0, return,
333 				    EINVAL);
334 				return (EINVAL);
335 			}
336 
337 			/*
338 			 * Increment refcount of the found futex to
339 			 * prevent it from deallocation before FUTEX_LOCK()
340 			 */
341 			++f->f_refcount;
342 			FUTEXES_UNLOCK;
343 			umtx_key_release(&key);
344 
345 			FUTEX_LOCK(f);
346 			*newf = f;
347 			LIN_SDT_PROBE3(futex, futex_get0, shared, uaddr,
348 			    f->f_refcount, f->f_key.shared);
349 			LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d",
350 			    uaddr, f->f_refcount, f->f_key.shared);
351 
352 			LIN_SDT_PROBE1(futex, futex_get0, return, 0);
353 			return (0);
354 		}
355 	}
356 
357 	if (flags & FUTEX_DONTCREATE) {
358 		FUTEXES_UNLOCK;
359 		umtx_key_release(&key);
360 		LIN_SDT_PROBE1(futex, futex_get0, null, uaddr);
361 		LINUX_CTR1(sys_futex, "futex_get uaddr %p null", uaddr);
362 
363 		LIN_SDT_PROBE1(futex, futex_get0, return, 0);
364 		return (0);
365 	}
366 
367 	if (tmpf == NULL) {
368 		FUTEXES_UNLOCK;
369 		tmpf = malloc(sizeof(*tmpf), M_FUTEX, M_WAITOK | M_ZERO);
370 		tmpf->f_uaddr = uaddr;
371 		tmpf->f_key = key;
372 		tmpf->f_refcount = 1;
373 		tmpf->f_bitset = FUTEX_BITSET_MATCH_ANY;
374 		FUTEX_INIT(tmpf);
375 		TAILQ_INIT(&tmpf->f_waiting_proc);
376 
377 		/*
378 		 * Lock the new futex before an insert into the futex_list
379 		 * to prevent futex usage by other.
380 		 */
381 		FUTEX_LOCK(tmpf);
382 		goto retry;
383 	}
384 
385 	LIST_INSERT_HEAD(&futex_list, tmpf, f_list);
386 	FUTEXES_UNLOCK;
387 
388 	LIN_SDT_PROBE3(futex, futex_get0, new, uaddr, tmpf->f_refcount,
389 	    tmpf->f_key.shared);
390 	LINUX_CTR3(sys_futex, "futex_get uaddr %p ref %d shared %d new",
391 	    uaddr, tmpf->f_refcount, tmpf->f_key.shared);
392 	*newf = tmpf;
393 
394 	LIN_SDT_PROBE1(futex, futex_get0, return, 0);
395 	return (0);
396 }
397 
398 static int
399 futex_get(uint32_t *uaddr, struct waiting_proc **wp, struct futex **f,
400     uint32_t flags)
401 {
402 	int error;
403 
404 	LIN_SDT_PROBE3(futex, futex_get, entry, uaddr, wp, f);
405 
406 	if (flags & FUTEX_CREATE_WP) {
407 		*wp = malloc(sizeof(struct waiting_proc), M_FUTEX_WP, M_WAITOK);
408 		(*wp)->wp_flags = 0;
409 	}
410 	error = futex_get0(uaddr, f, flags);
411 	if (error) {
412 		LIN_SDT_PROBE0(futex, futex_get, error);
413 
414 		if (flags & FUTEX_CREATE_WP)
415 			free(*wp, M_FUTEX_WP);
416 
417 		LIN_SDT_PROBE1(futex, futex_get, return, error);
418 		return (error);
419 	}
420 	if (flags & FUTEX_CREATE_WP) {
421 		TAILQ_INSERT_HEAD(&(*f)->f_waiting_proc, *wp, wp_list);
422 		(*wp)->wp_futex = *f;
423 	}
424 
425 	LIN_SDT_PROBE1(futex, futex_get, return, error);
426 	return (error);
427 }
428 
429 static int
430 futex_sleep(struct futex *f, struct waiting_proc *wp, int timeout)
431 {
432 	int error;
433 
434 	FUTEX_ASSERT_LOCKED(f);
435 	LIN_SDT_PROBE3(futex, futex_sleep, entry, f, wp, timeout);
436 	LINUX_CTR4(sys_futex, "futex_sleep enter uaddr %p wp %p timo %d ref %d",
437 	    f->f_uaddr, wp, timeout, f->f_refcount);
438 	error = sx_sleep(wp, &f->f_lck, PCATCH, "futex", timeout);
439 	if (wp->wp_flags & FUTEX_WP_REQUEUED) {
440 		KASSERT(f != wp->wp_futex, ("futex != wp_futex"));
441 
442 		if (error) {
443 			LIN_SDT_PROBE5(futex, futex_sleep, requeue_error, error,
444 			    f->f_uaddr, wp, wp->wp_futex->f_uaddr,
445 			    wp->wp_futex->f_refcount);
446 		}
447 
448 		LINUX_CTR5(sys_futex, "futex_sleep out error %d uaddr %p wp"
449 		    " %p requeued uaddr %p ref %d",
450 		    error, f->f_uaddr, wp, wp->wp_futex->f_uaddr,
451 		    wp->wp_futex->f_refcount);
452 		futex_put(f, NULL);
453 		f = wp->wp_futex;
454 		FUTEX_LOCK(f);
455 	} else {
456 		if (error) {
457 			LIN_SDT_PROBE3(futex, futex_sleep, sleep_error, error,
458 			    f->f_uaddr, wp);
459 		}
460 		LINUX_CTR3(sys_futex, "futex_sleep out error %d uaddr %p wp %p",
461 		    error, f->f_uaddr, wp);
462 	}
463 
464 	futex_put(f, wp);
465 
466 	LIN_SDT_PROBE1(futex, futex_sleep, return, error);
467 	return (error);
468 }
469 
470 static int
471 futex_wake(struct futex *f, int n, uint32_t bitset)
472 {
473 	struct waiting_proc *wp, *wpt;
474 	int count = 0;
475 
476 	LIN_SDT_PROBE3(futex, futex_wake, entry, f, n, bitset);
477 
478 	if (bitset == 0) {
479 		LIN_SDT_PROBE1(futex, futex_wake, return, EINVAL);
480 		return (EINVAL);
481 	}
482 
483 	FUTEX_ASSERT_LOCKED(f);
484 	TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
485 		LIN_SDT_PROBE3(futex, futex_wake, iterate, f->f_uaddr, wp,
486 		    f->f_refcount);
487 		LINUX_CTR3(sys_futex, "futex_wake uaddr %p wp %p ref %d",
488 		    f->f_uaddr, wp, f->f_refcount);
489 		/*
490 		 * Unless we find a matching bit in
491 		 * the bitset, continue searching.
492 		 */
493 		if (!(wp->wp_futex->f_bitset & bitset))
494 			continue;
495 
496 		wp->wp_flags |= FUTEX_WP_REMOVED;
497 		TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
498 		LIN_SDT_PROBE1(futex, futex_wake, wakeup, wp);
499 		wakeup_one(wp);
500 		if (++count == n)
501 			break;
502 	}
503 
504 	LIN_SDT_PROBE1(futex, futex_wake, return, count);
505 	return (count);
506 }
507 
508 static int
509 futex_requeue(struct futex *f, int n, struct futex *f2, int n2)
510 {
511 	struct waiting_proc *wp, *wpt;
512 	int count = 0;
513 
514 	LIN_SDT_PROBE4(futex, futex_requeue, entry, f, n, f2, n2);
515 
516 	FUTEX_ASSERT_LOCKED(f);
517 	FUTEX_ASSERT_LOCKED(f2);
518 
519 	TAILQ_FOREACH_SAFE(wp, &f->f_waiting_proc, wp_list, wpt) {
520 		if (++count <= n) {
521 			LINUX_CTR2(sys_futex, "futex_req_wake uaddr %p wp %p",
522 			    f->f_uaddr, wp);
523 			wp->wp_flags |= FUTEX_WP_REMOVED;
524 			TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
525 			LIN_SDT_PROBE1(futex, futex_requeue, wakeup, wp);
526 			wakeup_one(wp);
527 		} else {
528 			LIN_SDT_PROBE3(futex, futex_requeue, requeue,
529 			    f->f_uaddr, wp, f2->f_uaddr);
530 			LINUX_CTR3(sys_futex, "futex_requeue uaddr %p wp %p to %p",
531 			    f->f_uaddr, wp, f2->f_uaddr);
532 			wp->wp_flags |= FUTEX_WP_REQUEUED;
533 			/* Move wp to wp_list of f2 futex */
534 			TAILQ_REMOVE(&f->f_waiting_proc, wp, wp_list);
535 			TAILQ_INSERT_HEAD(&f2->f_waiting_proc, wp, wp_list);
536 
537 			/*
538 			 * Thread which sleeps on wp after waking should
539 			 * acquire f2 lock, so increment refcount of f2 to
540 			 * prevent it from premature deallocation.
541 			 */
542 			wp->wp_futex = f2;
543 			FUTEXES_LOCK;
544 			++f2->f_refcount;
545 			FUTEXES_UNLOCK;
546 			if (count - n >= n2)
547 				break;
548 		}
549 	}
550 
551 	LIN_SDT_PROBE1(futex, futex_requeue, return, count);
552 	return (count);
553 }
554 
555 static int
556 futex_wait(struct futex *f, struct waiting_proc *wp, int timeout_hz,
557     uint32_t bitset)
558 {
559 	int error;
560 
561 	LIN_SDT_PROBE4(futex, futex_wait, entry, f, wp, timeout_hz, bitset);
562 
563 	if (bitset == 0) {
564 		LIN_SDT_PROBE1(futex, futex_wait, return, EINVAL);
565 		return (EINVAL);
566 	}
567 
568 	f->f_bitset = bitset;
569 	error = futex_sleep(f, wp, timeout_hz);
570 	if (error)
571 		LIN_SDT_PROBE1(futex, futex_wait, sleep_error, error);
572 	if (error == EWOULDBLOCK)
573 		error = ETIMEDOUT;
574 
575 	LIN_SDT_PROBE1(futex, futex_wait, return, error);
576 	return (error);
577 }
578 
579 static int
580 futex_atomic_op(struct thread *td, int encoded_op, uint32_t *uaddr)
581 {
582 	int op = (encoded_op >> 28) & 7;
583 	int cmp = (encoded_op >> 24) & 15;
584 	int oparg = (encoded_op << 8) >> 20;
585 	int cmparg = (encoded_op << 20) >> 20;
586 	int oldval = 0, ret;
587 
588 	LIN_SDT_PROBE3(futex, futex_atomic_op, entry, td, encoded_op, uaddr);
589 
590 	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
591 		oparg = 1 << oparg;
592 
593 	LIN_SDT_PROBE4(futex, futex_atomic_op, decoded_op, op, cmp, oparg,
594 	    cmparg);
595 
596 	/* XXX: Linux verifies access here and returns EFAULT */
597 	LIN_SDT_PROBE0(futex, futex_atomic_op, missing_access_check);
598 
599 	switch (op) {
600 	case FUTEX_OP_SET:
601 		ret = futex_xchgl(oparg, uaddr, &oldval);
602 		break;
603 	case FUTEX_OP_ADD:
604 		ret = futex_addl(oparg, uaddr, &oldval);
605 		break;
606 	case FUTEX_OP_OR:
607 		ret = futex_orl(oparg, uaddr, &oldval);
608 		break;
609 	case FUTEX_OP_ANDN:
610 		ret = futex_andl(~oparg, uaddr, &oldval);
611 		break;
612 	case FUTEX_OP_XOR:
613 		ret = futex_xorl(oparg, uaddr, &oldval);
614 		break;
615 	default:
616 		LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_op, op);
617 		ret = -ENOSYS;
618 		break;
619 	}
620 
621 	if (ret) {
622 		LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
623 		return (ret);
624 	}
625 
626 	switch (cmp) {
627 	case FUTEX_OP_CMP_EQ:
628 		ret = (oldval == cmparg);
629 		break;
630 	case FUTEX_OP_CMP_NE:
631 		ret = (oldval != cmparg);
632 		break;
633 	case FUTEX_OP_CMP_LT:
634 		ret = (oldval < cmparg);
635 		break;
636 	case FUTEX_OP_CMP_GE:
637 		ret = (oldval >= cmparg);
638 		break;
639 	case FUTEX_OP_CMP_LE:
640 		ret = (oldval <= cmparg);
641 		break;
642 	case FUTEX_OP_CMP_GT:
643 		ret = (oldval > cmparg);
644 		break;
645 	default:
646 		LIN_SDT_PROBE1(futex, futex_atomic_op, unimplemented_cmp, cmp);
647 		ret = -ENOSYS;
648 	}
649 
650 	LIN_SDT_PROBE1(futex, futex_atomic_op, return, ret);
651 	return (ret);
652 }
653 
654 int
655 linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args)
656 {
657 	int clockrt, nrwake, op_ret, ret;
658 	struct linux_emuldata *em;
659 	struct waiting_proc *wp;
660 	struct futex *f, *f2;
661 	struct l_timespec timeout;
662 	struct timeval utv, ctv;
663 	int timeout_hz;
664 	int error;
665 	uint32_t flags, val;
666 
667 	LIN_SDT_PROBE2(futex, linux_sys_futex, entry, td, args);
668 
669 	if (args->op & LINUX_FUTEX_PRIVATE_FLAG) {
670 		flags = 0;
671 		args->op &= ~LINUX_FUTEX_PRIVATE_FLAG;
672 	} else
673 		flags = FUTEX_SHARED;
674 
675 	/*
676 	 * Currently support for switching between CLOCK_MONOTONIC and
677 	 * CLOCK_REALTIME is not present. However Linux forbids the use of
678 	 * FUTEX_CLOCK_REALTIME with any op except FUTEX_WAIT_BITSET and
679 	 * FUTEX_WAIT_REQUEUE_PI.
680 	 */
681 	clockrt = args->op & LINUX_FUTEX_CLOCK_REALTIME;
682 	args->op = args->op & ~LINUX_FUTEX_CLOCK_REALTIME;
683 	if (clockrt && args->op != LINUX_FUTEX_WAIT_BITSET &&
684 		args->op != LINUX_FUTEX_WAIT_REQUEUE_PI) {
685 		LIN_SDT_PROBE0(futex, linux_sys_futex,
686 		    unimplemented_clockswitch);
687 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
688 		return (ENOSYS);
689 	}
690 
691 	error = 0;
692 	f = f2 = NULL;
693 
694 	switch (args->op) {
695 	case LINUX_FUTEX_WAIT:
696 		args->val3 = FUTEX_BITSET_MATCH_ANY;
697 		/* FALLTHROUGH */
698 
699 	case LINUX_FUTEX_WAIT_BITSET:
700 		LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wait, args->uaddr,
701 		    args->val, args->val3);
702 		LINUX_CTR3(sys_futex, "WAIT uaddr %p val 0x%x bitset 0x%x",
703 		    args->uaddr, args->val, args->val3);
704 
705 		error = futex_get(args->uaddr, &wp, &f,
706 		    flags | FUTEX_CREATE_WP);
707 		if (error) {
708 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
709 			return (error);
710 		}
711 
712 		error = copyin(args->uaddr, &val, sizeof(val));
713 		if (error) {
714 			LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
715 			    error);
716 			LINUX_CTR1(sys_futex, "WAIT copyin failed %d",
717 			    error);
718 			futex_put(f, wp);
719 
720 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
721 			return (error);
722 		}
723 		if (val != args->val) {
724 			LIN_SDT_PROBE4(futex, linux_sys_futex,
725 			    debug_wait_value_neq, args->uaddr, args->val, val,
726 			    args->val3);
727 			LINUX_CTR3(sys_futex,
728 			    "WAIT uaddr %p val 0x%x != uval 0x%x",
729 			    args->uaddr, args->val, val);
730 			futex_put(f, wp);
731 
732 			LIN_SDT_PROBE1(futex, linux_sys_futex, return,
733 			    EWOULDBLOCK);
734 			return (EWOULDBLOCK);
735 		}
736 
737 		if (args->timeout != NULL) {
738 			error = copyin(args->timeout, &timeout, sizeof(timeout));
739 			if (error) {
740 				LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
741 				    error);
742 				LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
743 				futex_put(f, wp);
744 				return (error);
745 			}
746 			TIMESPEC_TO_TIMEVAL(&utv, &timeout);
747 			error = itimerfix(&utv);
748 			if (error) {
749 				LIN_SDT_PROBE1(futex, linux_sys_futex, itimerfix_error,
750 				    error);
751 				LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
752 				futex_put(f, wp);
753 				return (error);
754 			}
755 			if (clockrt) {
756 				microtime(&ctv);
757 				timevalsub(&utv, &ctv);
758 			} else if (args->op == LINUX_FUTEX_WAIT_BITSET) {
759 				microuptime(&ctv);
760 				timevalsub(&utv, &ctv);
761 			}
762 			if (utv.tv_sec < 0)
763 				timevalclear(&utv);
764 			timeout_hz = tvtohz(&utv);
765 		} else
766 			timeout_hz = 0;
767 
768 		error = futex_wait(f, wp, timeout_hz, args->val3);
769 		break;
770 
771 	case LINUX_FUTEX_WAKE:
772 		args->val3 = FUTEX_BITSET_MATCH_ANY;
773 		/* FALLTHROUGH */
774 
775 	case LINUX_FUTEX_WAKE_BITSET:
776 		LIN_SDT_PROBE3(futex, linux_sys_futex, debug_wake, args->uaddr,
777 		    args->val, args->val3);
778 		LINUX_CTR3(sys_futex, "WAKE uaddr %p nrwake 0x%x bitset 0x%x",
779 		    args->uaddr, args->val, args->val3);
780 
781 		error = futex_get(args->uaddr, NULL, &f,
782 		    flags | FUTEX_DONTCREATE);
783 		if (error) {
784 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
785 			return (error);
786 		}
787 
788 		if (f == NULL) {
789 			td->td_retval[0] = 0;
790 
791 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
792 			return (error);
793 		}
794 		td->td_retval[0] = futex_wake(f, args->val, args->val3);
795 		futex_put(f, NULL);
796 		break;
797 
798 	case LINUX_FUTEX_CMP_REQUEUE:
799 		LIN_SDT_PROBE5(futex, linux_sys_futex, debug_cmp_requeue,
800 		    args->uaddr, args->val, args->val3, args->uaddr2,
801 		    args->timeout);
802 		LINUX_CTR5(sys_futex, "CMP_REQUEUE uaddr %p "
803 		    "nrwake 0x%x uval 0x%x uaddr2 %p nrequeue 0x%x",
804 		    args->uaddr, args->val, args->val3, args->uaddr2,
805 		    args->timeout);
806 
807 		/*
808 		 * Linux allows this, we would not, it is an incorrect
809 		 * usage of declared ABI, so return EINVAL.
810 		 */
811 		if (args->uaddr == args->uaddr2) {
812 			LIN_SDT_PROBE0(futex, linux_sys_futex,
813 			    invalid_cmp_requeue_use);
814 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
815 			return (EINVAL);
816 		}
817 
818 		error = futex_get(args->uaddr, NULL, &f, flags);
819 		if (error) {
820 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
821 			return (error);
822 		}
823 
824 		/*
825 		 * To avoid deadlocks return EINVAL if second futex
826 		 * exists at this time.
827 		 *
828 		 * Glibc fall back to FUTEX_WAKE in case of any error
829 		 * returned by FUTEX_CMP_REQUEUE.
830 		 */
831 		error = futex_get(args->uaddr2, NULL, &f2,
832 		    flags | FUTEX_DONTEXISTS);
833 		if (error) {
834 			futex_put(f, NULL);
835 
836 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
837 			return (error);
838 		}
839 		error = copyin(args->uaddr, &val, sizeof(val));
840 		if (error) {
841 			LIN_SDT_PROBE1(futex, linux_sys_futex, copyin_error,
842 			    error);
843 			LINUX_CTR1(sys_futex, "CMP_REQUEUE copyin failed %d",
844 			    error);
845 			futex_put(f2, NULL);
846 			futex_put(f, NULL);
847 
848 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
849 			return (error);
850 		}
851 		if (val != args->val3) {
852 			LIN_SDT_PROBE2(futex, linux_sys_futex,
853 			    debug_cmp_requeue_value_neq, args->val, val);
854 			LINUX_CTR2(sys_futex, "CMP_REQUEUE val 0x%x != uval 0x%x",
855 			    args->val, val);
856 			futex_put(f2, NULL);
857 			futex_put(f, NULL);
858 
859 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, EAGAIN);
860 			return (EAGAIN);
861 		}
862 
863 		nrwake = (int)(unsigned long)args->timeout;
864 		td->td_retval[0] = futex_requeue(f, args->val, f2, nrwake);
865 		futex_put(f2, NULL);
866 		futex_put(f, NULL);
867 		break;
868 
869 	case LINUX_FUTEX_WAKE_OP:
870 		LIN_SDT_PROBE5(futex, linux_sys_futex, debug_wake_op,
871 		    args->uaddr, args->op, args->val, args->uaddr2, args->val3);
872 		LINUX_CTR5(sys_futex, "WAKE_OP "
873 		    "uaddr %p nrwake 0x%x uaddr2 %p op 0x%x nrwake2 0x%x",
874 		    args->uaddr, args->val, args->uaddr2, args->val3,
875 		    args->timeout);
876 
877 		error = futex_get(args->uaddr, NULL, &f, flags);
878 		if (error) {
879 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
880 			return (error);
881 		}
882 
883 		if (args->uaddr != args->uaddr2)
884 			error = futex_get(args->uaddr2, NULL, &f2, flags);
885 		if (error) {
886 			futex_put(f, NULL);
887 
888 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
889 			return (error);
890 		}
891 
892 		/*
893 		 * This function returns positive number as results and
894 		 * negative as errors
895 		 */
896 		op_ret = futex_atomic_op(td, args->val3, args->uaddr2);
897 
898 		LINUX_CTR2(sys_futex, "WAKE_OP atomic_op uaddr %p ret 0x%x",
899 		    args->uaddr, op_ret);
900 
901 		if (op_ret < 0) {
902 			/* XXX: We don't handle the EFAULT yet. */
903 			if (op_ret != -EFAULT) {
904 				if (f2 != NULL)
905 					futex_put(f2, NULL);
906 				futex_put(f, NULL);
907 
908 				LIN_SDT_PROBE1(futex, linux_sys_futex, return,
909 				    -op_ret);
910 				return (-op_ret);
911 			} else {
912 				LIN_SDT_PROBE0(futex, linux_sys_futex,
913 				    unhandled_efault);
914 			}
915 			if (f2 != NULL)
916 				futex_put(f2, NULL);
917 			futex_put(f, NULL);
918 
919 			LIN_SDT_PROBE1(futex, linux_sys_futex, return, EFAULT);
920 			return (EFAULT);
921 		}
922 
923 		ret = futex_wake(f, args->val, args->val3);
924 
925 		if (op_ret > 0) {
926 			op_ret = 0;
927 			nrwake = (int)(unsigned long)args->timeout;
928 
929 			if (f2 != NULL)
930 				op_ret += futex_wake(f2, nrwake, args->val3);
931 			else
932 				op_ret += futex_wake(f, nrwake, args->val3);
933 			ret += op_ret;
934 
935 		}
936 		if (f2 != NULL)
937 			futex_put(f2, NULL);
938 		futex_put(f, NULL);
939 		td->td_retval[0] = ret;
940 		break;
941 
942 	case LINUX_FUTEX_LOCK_PI:
943 		/* not yet implemented */
944 		linux_msg(td,
945 			  "linux_sys_futex: "
946 			  "op LINUX_FUTEX_LOCK_PI not implemented\n");
947 		LIN_SDT_PROBE0(futex, linux_sys_futex, unimplemented_lock_pi);
948 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
949 		return (ENOSYS);
950 
951 	case LINUX_FUTEX_UNLOCK_PI:
952 		/* not yet implemented */
953 		linux_msg(td,
954 			  "linux_sys_futex: "
955 			  "op LINUX_FUTEX_UNLOCK_PI not implemented\n");
956 		LIN_SDT_PROBE0(futex, linux_sys_futex, unimplemented_unlock_pi);
957 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
958 		return (ENOSYS);
959 
960 	case LINUX_FUTEX_TRYLOCK_PI:
961 		/* not yet implemented */
962 		linux_msg(td,
963 			  "linux_sys_futex: "
964 			  "op LINUX_FUTEX_TRYLOCK_PI not implemented\n");
965 		LIN_SDT_PROBE0(futex, linux_sys_futex,
966 		    unimplemented_trylock_pi);
967 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
968 		return (ENOSYS);
969 
970 	case LINUX_FUTEX_REQUEUE:
971 
972 		/*
973 		 * Glibc does not use this operation since version 2.3.3,
974 		 * as it is racy and replaced by FUTEX_CMP_REQUEUE operation.
975 		 * Glibc versions prior to 2.3.3 fall back to FUTEX_WAKE when
976 		 * FUTEX_REQUEUE returned EINVAL.
977 		 */
978 		em = em_find(td->td_proc, EMUL_DONTLOCK);
979 		if ((em->flags & LINUX_XDEPR_REQUEUEOP) == 0) {
980 			linux_msg(td,
981 				  "linux_sys_futex: "
982 				  "unsupported futex_requeue op\n");
983 			em->flags |= LINUX_XDEPR_REQUEUEOP;
984 			LIN_SDT_PROBE0(futex, linux_sys_futex,
985 			    deprecated_requeue);
986 		}
987 
988 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, EINVAL);
989 		return (EINVAL);
990 
991 	case LINUX_FUTEX_WAIT_REQUEUE_PI:
992 		/* not yet implemented */
993 		linux_msg(td,
994 			  "linux_sys_futex: "
995 			  "op FUTEX_WAIT_REQUEUE_PI not implemented\n");
996 		LIN_SDT_PROBE0(futex, linux_sys_futex,
997 		    unimplemented_wait_requeue_pi);
998 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
999 		return (ENOSYS);
1000 
1001 	case LINUX_FUTEX_CMP_REQUEUE_PI:
1002 		/* not yet implemented */
1003 		linux_msg(td,
1004 			    "linux_sys_futex: "
1005 			    "op LINUX_FUTEX_CMP_REQUEUE_PI not implemented\n");
1006 		LIN_SDT_PROBE0(futex, linux_sys_futex,
1007 		    unimplemented_cmp_requeue_pi);
1008 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
1009 		return (ENOSYS);
1010 
1011 	default:
1012 		linux_msg(td,
1013 			  "linux_sys_futex: unknown op %d\n", args->op);
1014 		LIN_SDT_PROBE1(futex, linux_sys_futex, unknown_operation,
1015 		    args->op);
1016 		LIN_SDT_PROBE1(futex, linux_sys_futex, return, ENOSYS);
1017 		return (ENOSYS);
1018 	}
1019 
1020 	LIN_SDT_PROBE1(futex, linux_sys_futex, return, error);
1021 	return (error);
1022 }
1023 
1024 int
1025 linux_set_robust_list(struct thread *td, struct linux_set_robust_list_args *args)
1026 {
1027 	struct linux_emuldata *em;
1028 
1029 	LIN_SDT_PROBE2(futex, linux_set_robust_list, entry, td, args);
1030 
1031 	if (args->len != sizeof(struct linux_robust_list_head)) {
1032 		LIN_SDT_PROBE0(futex, linux_set_robust_list, size_error);
1033 		LIN_SDT_PROBE1(futex, linux_set_robust_list, return, EINVAL);
1034 		return (EINVAL);
1035 	}
1036 
1037 	em = em_find(td->td_proc, EMUL_DOLOCK);
1038 	em->robust_futexes = args->head;
1039 	EMUL_UNLOCK(&emul_lock);
1040 
1041 	LIN_SDT_PROBE1(futex, linux_set_robust_list, return, 0);
1042 	return (0);
1043 }
1044 
1045 int
1046 linux_get_robust_list(struct thread *td, struct linux_get_robust_list_args *args)
1047 {
1048 	struct linux_emuldata *em;
1049 	struct linux_robust_list_head *head;
1050 	l_size_t len = sizeof(struct linux_robust_list_head);
1051 	int error = 0;
1052 
1053 	LIN_SDT_PROBE2(futex, linux_get_robust_list, entry, td, args);
1054 
1055 	if (!args->pid) {
1056 		em = em_find(td->td_proc, EMUL_DONTLOCK);
1057 		head = em->robust_futexes;
1058 	} else {
1059 		struct proc *p;
1060 
1061 		p = pfind(args->pid);
1062 		if (p == NULL) {
1063 			LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
1064 			    ESRCH);
1065 			return (ESRCH);
1066 		}
1067 
1068 		em = em_find(p, EMUL_DONTLOCK);
1069 		/* XXX: ptrace? */
1070 		if (priv_check(td, PRIV_CRED_SETUID) ||
1071 		    priv_check(td, PRIV_CRED_SETEUID) ||
1072 		    p_candebug(td, p)) {
1073 			PROC_UNLOCK(p);
1074 
1075 			LIN_SDT_PROBE1(futex, linux_get_robust_list, return,
1076 			    EPERM);
1077 			return (EPERM);
1078 		}
1079 		head = em->robust_futexes;
1080 
1081 		PROC_UNLOCK(p);
1082 	}
1083 
1084 	error = copyout(&len, args->len, sizeof(l_size_t));
1085 	if (error) {
1086 		LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
1087 		    error);
1088 		LIN_SDT_PROBE1(futex, linux_get_robust_list, return, EFAULT);
1089 		return (EFAULT);
1090 	}
1091 
1092 	error = copyout(head, args->head, sizeof(struct linux_robust_list_head));
1093 	if (error) {
1094 		LIN_SDT_PROBE1(futex, linux_get_robust_list, copyout_error,
1095 		    error);
1096 	}
1097 
1098 	LIN_SDT_PROBE1(futex, linux_get_robust_list, return, error);
1099 	return (error);
1100 }
1101 
1102 static int
1103 handle_futex_death(struct proc *p, uint32_t *uaddr, int pi)
1104 {
1105 	uint32_t uval, nval, mval;
1106 	struct futex *f;
1107 	int error;
1108 
1109 	LIN_SDT_PROBE3(futex, handle_futex_death, entry, p, uaddr, pi);
1110 
1111 retry:
1112 	error = copyin(uaddr, &uval, 4);
1113 	if (error) {
1114 		LIN_SDT_PROBE1(futex, handle_futex_death, copyin_error, error);
1115 		LIN_SDT_PROBE1(futex, handle_futex_death, return, EFAULT);
1116 		return (EFAULT);
1117 	}
1118 	if ((uval & FUTEX_TID_MASK) == p->p_pid) {
1119 		mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
1120 		nval = casuword32(uaddr, uval, mval);
1121 
1122 		if (nval == -1) {
1123 			LIN_SDT_PROBE1(futex, handle_futex_death, return,
1124 			    EFAULT);
1125 			return (EFAULT);
1126 		}
1127 
1128 		if (nval != uval)
1129 			goto retry;
1130 
1131 		if (!pi && (uval & FUTEX_WAITERS)) {
1132 			error = futex_get(uaddr, NULL, &f,
1133 			    FUTEX_DONTCREATE | FUTEX_SHARED);
1134 			if (error) {
1135 				LIN_SDT_PROBE1(futex, handle_futex_death,
1136 				    return, error);
1137 				return (error);
1138 			}
1139 			if (f != NULL) {
1140 				futex_wake(f, 1, FUTEX_BITSET_MATCH_ANY);
1141 				futex_put(f, NULL);
1142 			}
1143 		}
1144 	}
1145 
1146 	LIN_SDT_PROBE1(futex, handle_futex_death, return, 0);
1147 	return (0);
1148 }
1149 
1150 static int
1151 fetch_robust_entry(struct linux_robust_list **entry,
1152     struct linux_robust_list **head, int *pi)
1153 {
1154 	l_ulong uentry;
1155 	int error;
1156 
1157 	LIN_SDT_PROBE3(futex, fetch_robust_entry, entry, entry, head, pi);
1158 
1159 	error = copyin((const void *)head, &uentry, sizeof(l_ulong));
1160 	if (error) {
1161 		LIN_SDT_PROBE1(futex, fetch_robust_entry, copyin_error, error);
1162 		LIN_SDT_PROBE1(futex, fetch_robust_entry, return, EFAULT);
1163 		return (EFAULT);
1164 	}
1165 
1166 	*entry = (void *)(uentry & ~1UL);
1167 	*pi = uentry & 1;
1168 
1169 	LIN_SDT_PROBE1(futex, fetch_robust_entry, return, 0);
1170 	return (0);
1171 }
1172 
1173 /* This walks the list of robust futexes releasing them. */
1174 void
1175 release_futexes(struct proc *p)
1176 {
1177 	struct linux_robust_list_head *head = NULL;
1178 	struct linux_robust_list *entry, *next_entry, *pending;
1179 	unsigned int limit = 2048, pi, next_pi, pip;
1180 	struct linux_emuldata *em;
1181 	l_long futex_offset;
1182 	int rc, error;
1183 
1184 	LIN_SDT_PROBE1(futex, release_futexes, entry, p);
1185 
1186 	em = em_find(p, EMUL_DONTLOCK);
1187 	head = em->robust_futexes;
1188 
1189 	if (head == NULL) {
1190 		LIN_SDT_PROBE0(futex, release_futexes, return);
1191 		return;
1192 	}
1193 
1194 	if (fetch_robust_entry(&entry, PTRIN(&head->list.next), &pi)) {
1195 		LIN_SDT_PROBE0(futex, release_futexes, return);
1196 		return;
1197 	}
1198 
1199 	error = copyin(&head->futex_offset, &futex_offset,
1200 	    sizeof(futex_offset));
1201 	if (error) {
1202 		LIN_SDT_PROBE1(futex, release_futexes, copyin_error, error);
1203 		LIN_SDT_PROBE0(futex, release_futexes, return);
1204 		return;
1205 	}
1206 
1207 	if (fetch_robust_entry(&pending, PTRIN(&head->pending_list), &pip)) {
1208 		LIN_SDT_PROBE0(futex, release_futexes, return);
1209 		return;
1210 	}
1211 
1212 	while (entry != &head->list) {
1213 		rc = fetch_robust_entry(&next_entry, PTRIN(&entry->next), &next_pi);
1214 
1215 		if (entry != pending)
1216 			if (handle_futex_death(p,
1217 			    (uint32_t *)((caddr_t)entry + futex_offset), pi)) {
1218 				LIN_SDT_PROBE0(futex, release_futexes, return);
1219 				return;
1220 			}
1221 		if (rc) {
1222 			LIN_SDT_PROBE0(futex, release_futexes, return);
1223 			return;
1224 		}
1225 
1226 		entry = next_entry;
1227 		pi = next_pi;
1228 
1229 		if (!--limit)
1230 			break;
1231 
1232 		sched_relinquish(curthread);
1233 	}
1234 
1235 	if (pending)
1236 		handle_futex_death(p, (uint32_t *)((caddr_t)pending + futex_offset), pip);
1237 
1238 	LIN_SDT_PROBE0(futex, release_futexes, return);
1239 }
1240