1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 #include <sys/param.h>
31 #include <sys/types.h>
32 #include <sys/sysmacros.h>
33 #include <sys/systm.h>
34 #include <sys/cred.h>
35 #include <sys/user.h>
36 #include <sys/errno.h>
37 #include <sys/file.h>
38 #include <sys/proc.h>
39 #include <sys/prsystm.h>
40 #include <sys/kmem.h>
41 #include <sys/sobject.h>
42 #include <sys/fault.h>
43 #include <sys/procfs.h>
44 #include <sys/watchpoint.h>
45 #include <sys/time.h>
46 #include <sys/cmn_err.h>
47 #include <sys/machlock.h>
48 #include <sys/debug.h>
49 #include <sys/synch.h>
50 #include <sys/synch32.h>
51 #include <sys/mman.h>
52 #include <sys/class.h>
53 #include <sys/schedctl.h>
54 #include <sys/sleepq.h>
55 #include <sys/policy.h>
56 #include <sys/tnf_probe.h>
57 #include <sys/lwpchan_impl.h>
58 #include <sys/turnstile.h>
59 #include <sys/atomic.h>
60 #include <sys/lwp_timer_impl.h>
61 #include <sys/lwp_upimutex_impl.h>
62 #include <vm/as.h>
63 #include <sys/sdt.h>
64
65 static kthread_t *lwpsobj_owner(caddr_t);
66 static void lwp_unsleep(kthread_t *t);
67 static void lwp_change_pri(kthread_t *t, pri_t pri, pri_t *t_prip);
68 static void lwp_mutex_cleanup(lwpchan_entry_t *ent, uint16_t lockflg);
69 static void lwp_mutex_unregister(void *uaddr);
70 static void set_owner_pid(lwp_mutex_t *, uintptr_t, pid_t);
71 static int iswanted(kthread_t *, lwpchan_t *);
72
73 extern int lwp_cond_signal(lwp_cond_t *cv);
74
75 /*
76 * Maximum number of user prio inheritance locks that can be held by a thread.
77 * Used to limit kmem for each thread. This is a per-thread limit that
78 * can be administered on a system wide basis (using /etc/system).
79 *
80 * Also, when a limit, say maxlwps is added for numbers of lwps within a
81 * process, the per-thread limit automatically becomes a process-wide limit
82 * of maximum number of held upi locks within a process:
83 * maxheldupimx = maxnestupimx * maxlwps;
84 */
85 static uint32_t maxnestupimx = 2000;
86
87 /*
88 * The sobj_ops vector exports a set of functions needed when a thread
89 * is asleep on a synchronization object of this type.
90 */
91 static sobj_ops_t lwp_sobj_ops = {
92 SOBJ_USER, lwpsobj_owner, lwp_unsleep, lwp_change_pri
93 };
94
95 static kthread_t *lwpsobj_pi_owner(upimutex_t *up);
96
97 static sobj_ops_t lwp_sobj_pi_ops = {
98 SOBJ_USER_PI, lwpsobj_pi_owner, turnstile_unsleep,
99 turnstile_change_pri
100 };
101
102 static sleepq_head_t lwpsleepq[NSLEEPQ];
103 upib_t upimutextab[UPIMUTEX_TABSIZE];
104
105 #define LWPCHAN_LOCK_SHIFT 10 /* 1024 locks for each pool */
106 #define LWPCHAN_LOCK_SIZE (1 << LWPCHAN_LOCK_SHIFT)
107
108 /*
109 * We know that both lc_wchan and lc_wchan0 are addresses that most
110 * likely are 8-byte aligned, so we shift off the low-order 3 bits.
111 * 'pool' is either 0 or 1.
112 */
113 #define LWPCHAN_LOCK_HASH(X, pool) \
114 (((((X) >> 3) ^ ((X) >> (LWPCHAN_LOCK_SHIFT + 3))) & \
115 (LWPCHAN_LOCK_SIZE - 1)) + ((pool)? LWPCHAN_LOCK_SIZE : 0))
116
117 static kmutex_t lwpchanlock[2 * LWPCHAN_LOCK_SIZE];
118
119 /*
120 * Is this a POSIX threads user-level lock requiring priority inheritance?
121 */
122 #define UPIMUTEX(type) ((type) & LOCK_PRIO_INHERIT)
123
124 static sleepq_head_t *
lwpsqhash(lwpchan_t * lwpchan)125 lwpsqhash(lwpchan_t *lwpchan)
126 {
127 uint_t x = (uintptr_t)lwpchan->lc_wchan ^ (uintptr_t)lwpchan->lc_wchan0;
128 return (&lwpsleepq[SQHASHINDEX(x)]);
129 }
130
131 /*
132 * Lock an lwpchan.
133 * Keep this in sync with lwpchan_unlock(), below.
134 */
135 static void
lwpchan_lock(lwpchan_t * lwpchan,int pool)136 lwpchan_lock(lwpchan_t *lwpchan, int pool)
137 {
138 uint_t x = (uintptr_t)lwpchan->lc_wchan ^ (uintptr_t)lwpchan->lc_wchan0;
139 mutex_enter(&lwpchanlock[LWPCHAN_LOCK_HASH(x, pool)]);
140 }
141
142 /*
143 * Unlock an lwpchan.
144 * Keep this in sync with lwpchan_lock(), above.
145 */
146 static void
lwpchan_unlock(lwpchan_t * lwpchan,int pool)147 lwpchan_unlock(lwpchan_t *lwpchan, int pool)
148 {
149 uint_t x = (uintptr_t)lwpchan->lc_wchan ^ (uintptr_t)lwpchan->lc_wchan0;
150 mutex_exit(&lwpchanlock[LWPCHAN_LOCK_HASH(x, pool)]);
151 }
152
153 /*
154 * Delete mappings from the lwpchan cache for pages that are being
155 * unmapped by as_unmap(). Given a range of addresses, "start" to "end",
156 * all mappings within the range are deleted from the lwpchan cache.
157 */
158 void
lwpchan_delete_mapping(proc_t * p,caddr_t start,caddr_t end)159 lwpchan_delete_mapping(proc_t *p, caddr_t start, caddr_t end)
160 {
161 lwpchan_data_t *lcp;
162 lwpchan_hashbucket_t *hashbucket;
163 lwpchan_hashbucket_t *endbucket;
164 lwpchan_entry_t *ent;
165 lwpchan_entry_t **prev;
166 caddr_t addr;
167
168 mutex_enter(&p->p_lcp_lock);
169 lcp = p->p_lcp;
170 hashbucket = lcp->lwpchan_cache;
171 endbucket = hashbucket + lcp->lwpchan_size;
172 for (; hashbucket < endbucket; hashbucket++) {
173 if (hashbucket->lwpchan_chain == NULL)
174 continue;
175 mutex_enter(&hashbucket->lwpchan_lock);
176 prev = &hashbucket->lwpchan_chain;
177 /* check entire chain */
178 while ((ent = *prev) != NULL) {
179 addr = ent->lwpchan_addr;
180 if (start <= addr && addr < end) {
181 *prev = ent->lwpchan_next;
182 /*
183 * We do this only for the obsolete type
184 * USYNC_PROCESS_ROBUST. Otherwise robust
185 * locks do not draw ELOCKUNMAPPED or
186 * EOWNERDEAD due to being unmapped.
187 */
188 if (ent->lwpchan_pool == LWPCHAN_MPPOOL &&
189 (ent->lwpchan_type & USYNC_PROCESS_ROBUST))
190 lwp_mutex_cleanup(ent, LOCK_UNMAPPED);
191 /*
192 * If there is a user-level robust lock
193 * registration, mark it as invalid.
194 */
195 if ((addr = ent->lwpchan_uaddr) != NULL)
196 lwp_mutex_unregister(addr);
197 kmem_free(ent, sizeof (*ent));
198 atomic_dec_32(&lcp->lwpchan_entries);
199 } else {
200 prev = &ent->lwpchan_next;
201 }
202 }
203 mutex_exit(&hashbucket->lwpchan_lock);
204 }
205 mutex_exit(&p->p_lcp_lock);
206 }
207
208 /*
209 * Given an lwpchan cache pointer and a process virtual address,
210 * return a pointer to the corresponding lwpchan hash bucket.
211 */
212 static lwpchan_hashbucket_t *
lwpchan_bucket(lwpchan_data_t * lcp,uintptr_t addr)213 lwpchan_bucket(lwpchan_data_t *lcp, uintptr_t addr)
214 {
215 uint_t i;
216
217 /*
218 * All user-level sync object addresses are 8-byte aligned.
219 * Ignore the lowest 3 bits of the address and use the
220 * higher-order 2*lwpchan_bits bits for the hash index.
221 */
222 addr >>= 3;
223 i = (addr ^ (addr >> lcp->lwpchan_bits)) & lcp->lwpchan_mask;
224 return (lcp->lwpchan_cache + i);
225 }
226
227 /*
228 * (Re)allocate the per-process lwpchan cache.
229 */
230 static void
lwpchan_alloc_cache(proc_t * p,uint_t bits)231 lwpchan_alloc_cache(proc_t *p, uint_t bits)
232 {
233 lwpchan_data_t *lcp;
234 lwpchan_data_t *old_lcp;
235 lwpchan_hashbucket_t *hashbucket;
236 lwpchan_hashbucket_t *endbucket;
237 lwpchan_hashbucket_t *newbucket;
238 lwpchan_entry_t *ent;
239 lwpchan_entry_t *next;
240 uint_t count;
241
242 ASSERT(bits >= LWPCHAN_INITIAL_BITS && bits <= LWPCHAN_MAX_BITS);
243
244 lcp = kmem_alloc(sizeof (lwpchan_data_t), KM_SLEEP);
245 lcp->lwpchan_bits = bits;
246 lcp->lwpchan_size = 1 << lcp->lwpchan_bits;
247 lcp->lwpchan_mask = lcp->lwpchan_size - 1;
248 lcp->lwpchan_entries = 0;
249 lcp->lwpchan_cache = kmem_zalloc(lcp->lwpchan_size *
250 sizeof (lwpchan_hashbucket_t), KM_SLEEP);
251 lcp->lwpchan_next_data = NULL;
252
253 mutex_enter(&p->p_lcp_lock);
254 if ((old_lcp = p->p_lcp) != NULL) {
255 if (old_lcp->lwpchan_bits >= bits) {
256 /* someone beat us to it */
257 mutex_exit(&p->p_lcp_lock);
258 kmem_free(lcp->lwpchan_cache, lcp->lwpchan_size *
259 sizeof (lwpchan_hashbucket_t));
260 kmem_free(lcp, sizeof (lwpchan_data_t));
261 return;
262 }
263 /*
264 * Acquire all of the old hash table locks.
265 */
266 hashbucket = old_lcp->lwpchan_cache;
267 endbucket = hashbucket + old_lcp->lwpchan_size;
268 for (; hashbucket < endbucket; hashbucket++)
269 mutex_enter(&hashbucket->lwpchan_lock);
270 /*
271 * Move all of the old hash table entries to the
272 * new hash table. The new hash table has not yet
273 * been installed so we don't need any of its locks.
274 */
275 count = 0;
276 hashbucket = old_lcp->lwpchan_cache;
277 for (; hashbucket < endbucket; hashbucket++) {
278 ent = hashbucket->lwpchan_chain;
279 while (ent != NULL) {
280 next = ent->lwpchan_next;
281 newbucket = lwpchan_bucket(lcp,
282 (uintptr_t)ent->lwpchan_addr);
283 ent->lwpchan_next = newbucket->lwpchan_chain;
284 newbucket->lwpchan_chain = ent;
285 ent = next;
286 count++;
287 }
288 hashbucket->lwpchan_chain = NULL;
289 }
290 lcp->lwpchan_entries = count;
291 }
292
293 /*
294 * Retire the old hash table. We can't actually kmem_free() it
295 * now because someone may still have a pointer to it. Instead,
296 * we link it onto the new hash table's list of retired hash tables.
297 * The new hash table is double the size of the previous one, so
298 * the total size of all retired hash tables is less than the size
299 * of the new one. exit() and exec() free the retired hash tables
300 * (see lwpchan_destroy_cache(), below).
301 */
302 lcp->lwpchan_next_data = old_lcp;
303
304 /*
305 * As soon as we store the new lcp, future locking operations will
306 * use it. Therefore, we must ensure that all the state we've just
307 * established reaches global visibility before the new lcp does.
308 */
309 membar_producer();
310 p->p_lcp = lcp;
311
312 if (old_lcp != NULL) {
313 /*
314 * Release all of the old hash table locks.
315 */
316 hashbucket = old_lcp->lwpchan_cache;
317 for (; hashbucket < endbucket; hashbucket++)
318 mutex_exit(&hashbucket->lwpchan_lock);
319 }
320 mutex_exit(&p->p_lcp_lock);
321 }
322
323 /*
324 * Deallocate the lwpchan cache, and any dynamically allocated mappings.
325 * Called when the process exits or execs. All lwps except one have
326 * exited so we need no locks here.
327 */
328 void
lwpchan_destroy_cache(int exec)329 lwpchan_destroy_cache(int exec)
330 {
331 proc_t *p = curproc;
332 lwpchan_hashbucket_t *hashbucket;
333 lwpchan_hashbucket_t *endbucket;
334 lwpchan_data_t *lcp;
335 lwpchan_entry_t *ent;
336 lwpchan_entry_t *next;
337 uint16_t lockflg;
338
339 lcp = p->p_lcp;
340 p->p_lcp = NULL;
341
342 lockflg = exec? LOCK_UNMAPPED : LOCK_OWNERDEAD;
343 hashbucket = lcp->lwpchan_cache;
344 endbucket = hashbucket + lcp->lwpchan_size;
345 for (; hashbucket < endbucket; hashbucket++) {
346 ent = hashbucket->lwpchan_chain;
347 hashbucket->lwpchan_chain = NULL;
348 while (ent != NULL) {
349 next = ent->lwpchan_next;
350 if (ent->lwpchan_pool == LWPCHAN_MPPOOL &&
351 (ent->lwpchan_type & (USYNC_PROCESS | LOCK_ROBUST))
352 == (USYNC_PROCESS | LOCK_ROBUST))
353 lwp_mutex_cleanup(ent, lockflg);
354 kmem_free(ent, sizeof (*ent));
355 ent = next;
356 }
357 }
358
359 while (lcp != NULL) {
360 lwpchan_data_t *next_lcp = lcp->lwpchan_next_data;
361 kmem_free(lcp->lwpchan_cache, lcp->lwpchan_size *
362 sizeof (lwpchan_hashbucket_t));
363 kmem_free(lcp, sizeof (lwpchan_data_t));
364 lcp = next_lcp;
365 }
366 }
367
368 /*
369 * Return zero when there is an entry in the lwpchan cache for the
370 * given process virtual address and non-zero when there is not.
371 * The returned non-zero value is the current length of the
372 * hash chain plus one. The caller holds the hash bucket lock.
373 */
374 static uint_t
lwpchan_cache_mapping(caddr_t addr,int type,int pool,lwpchan_t * lwpchan,lwpchan_hashbucket_t * hashbucket)375 lwpchan_cache_mapping(caddr_t addr, int type, int pool, lwpchan_t *lwpchan,
376 lwpchan_hashbucket_t *hashbucket)
377 {
378 lwpchan_entry_t *ent;
379 uint_t count = 1;
380
381 for (ent = hashbucket->lwpchan_chain; ent; ent = ent->lwpchan_next) {
382 if (ent->lwpchan_addr == addr) {
383 if (ent->lwpchan_type != type ||
384 ent->lwpchan_pool != pool) {
385 /*
386 * This shouldn't happen, but might if the
387 * process reuses its memory for different
388 * types of sync objects. We test first
389 * to avoid grabbing the memory cache line.
390 */
391 ent->lwpchan_type = (uint16_t)type;
392 ent->lwpchan_pool = (uint16_t)pool;
393 }
394 *lwpchan = ent->lwpchan_lwpchan;
395 return (0);
396 }
397 count++;
398 }
399 return (count);
400 }
401
402 /*
403 * Return the cached lwpchan mapping if cached, otherwise insert
404 * a virtual address to lwpchan mapping into the cache.
405 */
406 static int
lwpchan_get_mapping(struct as * as,caddr_t addr,caddr_t uaddr,int type,lwpchan_t * lwpchan,int pool)407 lwpchan_get_mapping(struct as *as, caddr_t addr, caddr_t uaddr,
408 int type, lwpchan_t *lwpchan, int pool)
409 {
410 proc_t *p = curproc;
411 lwpchan_data_t *lcp;
412 lwpchan_hashbucket_t *hashbucket;
413 lwpchan_entry_t *ent;
414 memid_t memid;
415 uint_t count;
416 uint_t bits;
417
418 top:
419 /* initialize the lwpchan cache, if necesary */
420 if ((lcp = p->p_lcp) == NULL) {
421 lwpchan_alloc_cache(p, LWPCHAN_INITIAL_BITS);
422 goto top;
423 }
424 hashbucket = lwpchan_bucket(lcp, (uintptr_t)addr);
425 mutex_enter(&hashbucket->lwpchan_lock);
426 if (lcp != p->p_lcp) {
427 /* someone resized the lwpchan cache; start over */
428 mutex_exit(&hashbucket->lwpchan_lock);
429 goto top;
430 }
431 if (lwpchan_cache_mapping(addr, type, pool, lwpchan, hashbucket) == 0) {
432 /* it's in the cache */
433 mutex_exit(&hashbucket->lwpchan_lock);
434 return (1);
435 }
436 mutex_exit(&hashbucket->lwpchan_lock);
437 if (as_getmemid(as, addr, &memid) != 0)
438 return (0);
439 lwpchan->lc_wchan0 = (caddr_t)(uintptr_t)memid.val[0];
440 lwpchan->lc_wchan = (caddr_t)(uintptr_t)memid.val[1];
441 ent = kmem_alloc(sizeof (lwpchan_entry_t), KM_SLEEP);
442 mutex_enter(&hashbucket->lwpchan_lock);
443 if (lcp != p->p_lcp) {
444 /* someone resized the lwpchan cache; start over */
445 mutex_exit(&hashbucket->lwpchan_lock);
446 kmem_free(ent, sizeof (*ent));
447 goto top;
448 }
449 count = lwpchan_cache_mapping(addr, type, pool, lwpchan, hashbucket);
450 if (count == 0) {
451 /* someone else added this entry to the cache */
452 mutex_exit(&hashbucket->lwpchan_lock);
453 kmem_free(ent, sizeof (*ent));
454 return (1);
455 }
456 if (count > lcp->lwpchan_bits + 2 && /* larger table, longer chains */
457 (bits = lcp->lwpchan_bits) < LWPCHAN_MAX_BITS) {
458 /* hash chain too long; reallocate the hash table */
459 mutex_exit(&hashbucket->lwpchan_lock);
460 kmem_free(ent, sizeof (*ent));
461 lwpchan_alloc_cache(p, bits + 1);
462 goto top;
463 }
464 ent->lwpchan_addr = addr;
465 ent->lwpchan_uaddr = uaddr;
466 ent->lwpchan_type = (uint16_t)type;
467 ent->lwpchan_pool = (uint16_t)pool;
468 ent->lwpchan_lwpchan = *lwpchan;
469 ent->lwpchan_next = hashbucket->lwpchan_chain;
470 hashbucket->lwpchan_chain = ent;
471 atomic_inc_32(&lcp->lwpchan_entries);
472 mutex_exit(&hashbucket->lwpchan_lock);
473 return (1);
474 }
475
476 /*
477 * Return a unique pair of identifiers that corresponds to a
478 * synchronization object's virtual address. Process-shared
479 * sync objects usually get vnode/offset from as_getmemid().
480 */
481 static int
get_lwpchan(struct as * as,caddr_t addr,int type,lwpchan_t * lwpchan,int pool)482 get_lwpchan(struct as *as, caddr_t addr, int type, lwpchan_t *lwpchan, int pool)
483 {
484 /*
485 * If the lwp synch object is defined to be process-private,
486 * we just make the first field of the lwpchan be 'as' and
487 * the second field be the synch object's virtual address.
488 * (segvn_getmemid() does the same for MAP_PRIVATE mappings.)
489 * The lwpchan cache is used only for process-shared objects.
490 */
491 if (!(type & USYNC_PROCESS)) {
492 lwpchan->lc_wchan0 = (caddr_t)as;
493 lwpchan->lc_wchan = addr;
494 return (1);
495 }
496
497 return (lwpchan_get_mapping(as, addr, NULL, type, lwpchan, pool));
498 }
499
500 static void
lwp_block(lwpchan_t * lwpchan)501 lwp_block(lwpchan_t *lwpchan)
502 {
503 kthread_t *t = curthread;
504 klwp_t *lwp = ttolwp(t);
505 sleepq_head_t *sqh;
506
507 thread_lock(t);
508 t->t_flag |= T_WAKEABLE;
509 t->t_lwpchan = *lwpchan;
510 t->t_sobj_ops = &lwp_sobj_ops;
511 t->t_release = 0;
512 sqh = lwpsqhash(lwpchan);
513 disp_lock_enter_high(&sqh->sq_lock);
514 CL_SLEEP(t);
515 DTRACE_SCHED(sleep);
516 THREAD_SLEEP(t, &sqh->sq_lock);
517 sleepq_insert(&sqh->sq_queue, t);
518 thread_unlock(t);
519 lwp->lwp_asleep = 1;
520 lwp->lwp_sysabort = 0;
521 lwp->lwp_ru.nvcsw++;
522 (void) new_mstate(curthread, LMS_SLEEP);
523 }
524
525 static kthread_t *
lwpsobj_pi_owner(upimutex_t * up)526 lwpsobj_pi_owner(upimutex_t *up)
527 {
528 return (up->upi_owner);
529 }
530
531 static struct upimutex *
upi_get(upib_t * upibp,lwpchan_t * lcp)532 upi_get(upib_t *upibp, lwpchan_t *lcp)
533 {
534 struct upimutex *upip;
535
536 for (upip = upibp->upib_first; upip != NULL;
537 upip = upip->upi_nextchain) {
538 if (upip->upi_lwpchan.lc_wchan0 == lcp->lc_wchan0 &&
539 upip->upi_lwpchan.lc_wchan == lcp->lc_wchan)
540 break;
541 }
542 return (upip);
543 }
544
545 static void
upi_chain_add(upib_t * upibp,struct upimutex * upimutex)546 upi_chain_add(upib_t *upibp, struct upimutex *upimutex)
547 {
548 ASSERT(MUTEX_HELD(&upibp->upib_lock));
549
550 /*
551 * Insert upimutex at front of list. Maybe a bit unfair
552 * but assume that not many lwpchans hash to the same
553 * upimutextab bucket, i.e. the list of upimutexes from
554 * upib_first is not too long.
555 */
556 upimutex->upi_nextchain = upibp->upib_first;
557 upibp->upib_first = upimutex;
558 }
559
560 static void
upi_chain_del(upib_t * upibp,struct upimutex * upimutex)561 upi_chain_del(upib_t *upibp, struct upimutex *upimutex)
562 {
563 struct upimutex **prev;
564
565 ASSERT(MUTEX_HELD(&upibp->upib_lock));
566
567 prev = &upibp->upib_first;
568 while (*prev != upimutex) {
569 prev = &(*prev)->upi_nextchain;
570 }
571 *prev = upimutex->upi_nextchain;
572 upimutex->upi_nextchain = NULL;
573 }
574
575 /*
576 * Add upimutex to chain of upimutexes held by curthread.
577 * Returns number of upimutexes held by curthread.
578 */
579 static uint32_t
upi_mylist_add(struct upimutex * upimutex)580 upi_mylist_add(struct upimutex *upimutex)
581 {
582 kthread_t *t = curthread;
583
584 /*
585 * Insert upimutex at front of list of upimutexes owned by t. This
586 * would match typical LIFO order in which nested locks are acquired
587 * and released.
588 */
589 upimutex->upi_nextowned = t->t_upimutex;
590 t->t_upimutex = upimutex;
591 t->t_nupinest++;
592 ASSERT(t->t_nupinest > 0);
593 return (t->t_nupinest);
594 }
595
596 /*
597 * Delete upimutex from list of upimutexes owned by curthread.
598 */
599 static void
upi_mylist_del(struct upimutex * upimutex)600 upi_mylist_del(struct upimutex *upimutex)
601 {
602 kthread_t *t = curthread;
603 struct upimutex **prev;
604
605 /*
606 * Since the order in which nested locks are acquired and released,
607 * is typically LIFO, and typical nesting levels are not too deep, the
608 * following should not be expensive in the general case.
609 */
610 prev = &t->t_upimutex;
611 while (*prev != upimutex) {
612 prev = &(*prev)->upi_nextowned;
613 }
614 *prev = upimutex->upi_nextowned;
615 upimutex->upi_nextowned = NULL;
616 ASSERT(t->t_nupinest > 0);
617 t->t_nupinest--;
618 }
619
620 /*
621 * Returns true if upimutex is owned. Should be called only when upim points
622 * to kmem which cannot disappear from underneath.
623 */
624 static int
upi_owned(upimutex_t * upim)625 upi_owned(upimutex_t *upim)
626 {
627 return (upim->upi_owner == curthread);
628 }
629
630 /*
631 * Returns pointer to kernel object (upimutex_t *) if lp is owned.
632 */
633 static struct upimutex *
lwp_upimutex_owned(lwp_mutex_t * lp,uint8_t type)634 lwp_upimutex_owned(lwp_mutex_t *lp, uint8_t type)
635 {
636 lwpchan_t lwpchan;
637 upib_t *upibp;
638 struct upimutex *upimutex;
639
640 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
641 &lwpchan, LWPCHAN_MPPOOL))
642 return (NULL);
643
644 upibp = &UPI_CHAIN(lwpchan);
645 mutex_enter(&upibp->upib_lock);
646 upimutex = upi_get(upibp, &lwpchan);
647 if (upimutex == NULL || upimutex->upi_owner != curthread) {
648 mutex_exit(&upibp->upib_lock);
649 return (NULL);
650 }
651 mutex_exit(&upibp->upib_lock);
652 return (upimutex);
653 }
654
655 /*
656 * Unlocks upimutex, waking up waiters if any. upimutex kmem is freed if
657 * no lock hand-off occurrs.
658 */
659 static void
upimutex_unlock(struct upimutex * upimutex,uint16_t flag)660 upimutex_unlock(struct upimutex *upimutex, uint16_t flag)
661 {
662 turnstile_t *ts;
663 upib_t *upibp;
664 kthread_t *newowner;
665
666 upi_mylist_del(upimutex);
667 upibp = upimutex->upi_upibp;
668 mutex_enter(&upibp->upib_lock);
669 if (upimutex->upi_waiter != 0) { /* if waiters */
670 ts = turnstile_lookup(upimutex);
671 if (ts != NULL && !(flag & LOCK_NOTRECOVERABLE)) {
672 /* hand-off lock to highest prio waiter */
673 newowner = ts->ts_sleepq[TS_WRITER_Q].sq_first;
674 upimutex->upi_owner = newowner;
675 if (ts->ts_waiters == 1)
676 upimutex->upi_waiter = 0;
677 turnstile_wakeup(ts, TS_WRITER_Q, 1, newowner);
678 mutex_exit(&upibp->upib_lock);
679 return;
680 } else if (ts != NULL) {
681 /* LOCK_NOTRECOVERABLE: wakeup all */
682 turnstile_wakeup(ts, TS_WRITER_Q, ts->ts_waiters, NULL);
683 } else {
684 /*
685 * Misleading w bit. Waiters might have been
686 * interrupted. No need to clear the w bit (upimutex
687 * will soon be freed). Re-calculate PI from existing
688 * waiters.
689 */
690 turnstile_exit(upimutex);
691 turnstile_pi_recalc();
692 }
693 }
694 /*
695 * no waiters, or LOCK_NOTRECOVERABLE.
696 * remove from the bucket chain of upi mutexes.
697 * de-allocate kernel memory (upimutex).
698 */
699 upi_chain_del(upimutex->upi_upibp, upimutex);
700 mutex_exit(&upibp->upib_lock);
701 kmem_free(upimutex, sizeof (upimutex_t));
702 }
703
704 static int
lwp_upimutex_lock(lwp_mutex_t * lp,uint8_t type,int try,lwp_timer_t * lwptp)705 lwp_upimutex_lock(lwp_mutex_t *lp, uint8_t type, int try, lwp_timer_t *lwptp)
706 {
707 label_t ljb;
708 int error = 0;
709 lwpchan_t lwpchan;
710 uint16_t flag;
711 upib_t *upibp;
712 volatile struct upimutex *upimutex = NULL;
713 turnstile_t *ts;
714 uint32_t nupinest;
715 volatile int upilocked = 0;
716
717 if (on_fault(&ljb)) {
718 if (upilocked)
719 upimutex_unlock((upimutex_t *)upimutex, 0);
720 error = EFAULT;
721 goto out;
722 }
723 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
724 &lwpchan, LWPCHAN_MPPOOL)) {
725 error = EFAULT;
726 goto out;
727 }
728 upibp = &UPI_CHAIN(lwpchan);
729 retry:
730 mutex_enter(&upibp->upib_lock);
731 upimutex = upi_get(upibp, &lwpchan);
732 if (upimutex == NULL) {
733 /* lock available since lwpchan has no upimutex */
734 upimutex = kmem_zalloc(sizeof (upimutex_t), KM_SLEEP);
735 upi_chain_add(upibp, (upimutex_t *)upimutex);
736 upimutex->upi_owner = curthread; /* grab lock */
737 upimutex->upi_upibp = upibp;
738 upimutex->upi_vaddr = lp;
739 upimutex->upi_lwpchan = lwpchan;
740 mutex_exit(&upibp->upib_lock);
741 nupinest = upi_mylist_add((upimutex_t *)upimutex);
742 upilocked = 1;
743 fuword16_noerr(&lp->mutex_flag, &flag);
744 if (nupinest > maxnestupimx &&
745 secpolicy_resource(CRED()) != 0) {
746 upimutex_unlock((upimutex_t *)upimutex, flag);
747 error = ENOMEM;
748 goto out;
749 }
750 if (flag & LOCK_NOTRECOVERABLE) {
751 /*
752 * Since the setting of LOCK_NOTRECOVERABLE
753 * was done under the high-level upi mutex,
754 * in lwp_upimutex_unlock(), this flag needs to
755 * be checked while holding the upi mutex.
756 * If set, this thread should return without
757 * the lock held, and with the right error code.
758 */
759 upimutex_unlock((upimutex_t *)upimutex, flag);
760 upilocked = 0;
761 error = ENOTRECOVERABLE;
762 } else if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
763 if (flag & LOCK_OWNERDEAD)
764 error = EOWNERDEAD;
765 else if (type & USYNC_PROCESS_ROBUST)
766 error = ELOCKUNMAPPED;
767 else
768 error = EOWNERDEAD;
769 }
770 goto out;
771 }
772 /*
773 * If a upimutex object exists, it must have an owner.
774 * This is due to lock hand-off, and release of upimutex when no
775 * waiters are present at unlock time,
776 */
777 ASSERT(upimutex->upi_owner != NULL);
778 if (upimutex->upi_owner == curthread) {
779 /*
780 * The user wrapper can check if the mutex type is
781 * ERRORCHECK: if not, it should stall at user-level.
782 * If so, it should return the error code.
783 */
784 mutex_exit(&upibp->upib_lock);
785 error = EDEADLK;
786 goto out;
787 }
788 if (try == UPIMUTEX_TRY) {
789 mutex_exit(&upibp->upib_lock);
790 error = EBUSY;
791 goto out;
792 }
793 /*
794 * Block for the lock.
795 */
796 if ((error = lwptp->lwpt_time_error) != 0) {
797 /*
798 * The SUSV3 Posix spec is very clear that we
799 * should get no error from validating the
800 * timer until we would actually sleep.
801 */
802 mutex_exit(&upibp->upib_lock);
803 goto out;
804 }
805 if (lwptp->lwpt_tsp != NULL) {
806 /*
807 * Unlike the protocol for other lwp timedwait operations,
808 * we must drop t_delay_lock before going to sleep in
809 * turnstile_block() for a upi mutex.
810 * See the comments below and in turnstile.c
811 */
812 mutex_enter(&curthread->t_delay_lock);
813 (void) lwp_timer_enqueue(lwptp);
814 mutex_exit(&curthread->t_delay_lock);
815 }
816 /*
817 * Now, set the waiter bit and block for the lock in turnstile_block().
818 * No need to preserve the previous wbit since a lock try is not
819 * attempted after setting the wait bit. Wait bit is set under
820 * the upib_lock, which is not released until the turnstile lock
821 * is acquired. Say, the upimutex is L:
822 *
823 * 1. upib_lock is held so the waiter does not have to retry L after
824 * setting the wait bit: since the owner has to grab the upib_lock
825 * to unlock L, it will certainly see the wait bit set.
826 * 2. upib_lock is not released until the turnstile lock is acquired.
827 * This is the key to preventing a missed wake-up. Otherwise, the
828 * owner could acquire the upib_lock, and the tc_lock, to call
829 * turnstile_wakeup(). All this, before the waiter gets tc_lock
830 * to sleep in turnstile_block(). turnstile_wakeup() will then not
831 * find this waiter, resulting in the missed wakeup.
832 * 3. The upib_lock, being a kernel mutex, cannot be released while
833 * holding the tc_lock (since mutex_exit() could need to acquire
834 * the same tc_lock)...and so is held when calling turnstile_block().
835 * The address of upib_lock is passed to turnstile_block() which
836 * releases it after releasing all turnstile locks, and before going
837 * to sleep in swtch().
838 * 4. The waiter value cannot be a count of waiters, because a waiter
839 * can be interrupted. The interrupt occurs under the tc_lock, at
840 * which point, the upib_lock cannot be locked, to decrement waiter
841 * count. So, just treat the waiter state as a bit, not a count.
842 */
843 ts = turnstile_lookup((upimutex_t *)upimutex);
844 upimutex->upi_waiter = 1;
845 error = turnstile_block(ts, TS_WRITER_Q, (upimutex_t *)upimutex,
846 &lwp_sobj_pi_ops, &upibp->upib_lock, lwptp);
847 /*
848 * Hand-off implies that we wakeup holding the lock, except when:
849 * - deadlock is detected
850 * - lock is not recoverable
851 * - we got an interrupt or timeout
852 * If we wake up due to an interrupt or timeout, we may
853 * or may not be holding the lock due to mutex hand-off.
854 * Use lwp_upimutex_owned() to check if we do hold the lock.
855 */
856 if (error != 0) {
857 if ((error == EINTR || error == ETIME) &&
858 (upimutex = lwp_upimutex_owned(lp, type))) {
859 /*
860 * Unlock and return - the re-startable syscall will
861 * try the lock again if we got EINTR.
862 */
863 (void) upi_mylist_add((upimutex_t *)upimutex);
864 upimutex_unlock((upimutex_t *)upimutex, 0);
865 }
866 /*
867 * The only other possible error is EDEADLK. If so, upimutex
868 * is valid, since its owner is deadlocked with curthread.
869 */
870 ASSERT(error == EINTR || error == ETIME ||
871 (error == EDEADLK && !upi_owned((upimutex_t *)upimutex)));
872 ASSERT(!lwp_upimutex_owned(lp, type));
873 goto out;
874 }
875 if (lwp_upimutex_owned(lp, type)) {
876 ASSERT(lwp_upimutex_owned(lp, type) == upimutex);
877 nupinest = upi_mylist_add((upimutex_t *)upimutex);
878 upilocked = 1;
879 }
880 /*
881 * Now, need to read the user-level lp->mutex_flag to do the following:
882 *
883 * - if lock is held, check if EOWNERDEAD or ELOCKUNMAPPED
884 * should be returned.
885 * - if lock isn't held, check if ENOTRECOVERABLE should
886 * be returned.
887 *
888 * Now, either lp->mutex_flag is readable or it's not. If not
889 * readable, the on_fault path will cause a return with EFAULT
890 * as it should. If it is readable, the state of the flag
891 * encodes the robustness state of the lock:
892 *
893 * If the upimutex is locked here, the flag's LOCK_OWNERDEAD
894 * or LOCK_UNMAPPED setting will influence the return code
895 * appropriately. If the upimutex is not locked here, this
896 * could be due to a spurious wake-up or a NOTRECOVERABLE
897 * event. The flag's setting can be used to distinguish
898 * between these two events.
899 */
900 fuword16_noerr(&lp->mutex_flag, &flag);
901 if (upilocked) {
902 /*
903 * If the thread wakes up from turnstile_block with the lock
904 * held, the flag could not be set to LOCK_NOTRECOVERABLE,
905 * since it would not have been handed-off the lock.
906 * So, no need to check for this case.
907 */
908 if (nupinest > maxnestupimx &&
909 secpolicy_resource(CRED()) != 0) {
910 upimutex_unlock((upimutex_t *)upimutex, flag);
911 upilocked = 0;
912 error = ENOMEM;
913 } else if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
914 if (flag & LOCK_OWNERDEAD)
915 error = EOWNERDEAD;
916 else if (type & USYNC_PROCESS_ROBUST)
917 error = ELOCKUNMAPPED;
918 else
919 error = EOWNERDEAD;
920 }
921 } else {
922 /*
923 * Wake-up without the upimutex held. Either this is a
924 * spurious wake-up (due to signals, forkall(), whatever), or
925 * it is a LOCK_NOTRECOVERABLE robustness event. The setting
926 * of the mutex flag can be used to distinguish between the
927 * two events.
928 */
929 if (flag & LOCK_NOTRECOVERABLE) {
930 error = ENOTRECOVERABLE;
931 } else {
932 /*
933 * Here, the flag could be set to LOCK_OWNERDEAD or
934 * not. In both cases, this is a spurious wakeup,
935 * since the upi lock is not held, but the thread
936 * has returned from turnstile_block().
937 *
938 * The user flag could be LOCK_OWNERDEAD if, at the
939 * same time as curthread having been woken up
940 * spuriously, the owner (say Tdead) has died, marked
941 * the mutex flag accordingly, and handed off the lock
942 * to some other waiter (say Tnew). curthread just
943 * happened to read the flag while Tnew has yet to deal
944 * with the owner-dead event.
945 *
946 * In this event, curthread should retry the lock.
947 * If Tnew is able to cleanup the lock, curthread
948 * will eventually get the lock with a zero error code,
949 * If Tnew is unable to cleanup, its eventual call to
950 * unlock the lock will result in the mutex flag being
951 * set to LOCK_NOTRECOVERABLE, and the wake-up of
952 * all waiters, including curthread, which will then
953 * eventually return ENOTRECOVERABLE due to the above
954 * check.
955 *
956 * Of course, if the user-flag is not set with
957 * LOCK_OWNERDEAD, retrying is the thing to do, since
958 * this is definitely a spurious wakeup.
959 */
960 goto retry;
961 }
962 }
963
964 out:
965 no_fault();
966 return (error);
967 }
968
969
970 static int
lwp_upimutex_unlock(lwp_mutex_t * lp,uint8_t type)971 lwp_upimutex_unlock(lwp_mutex_t *lp, uint8_t type)
972 {
973 label_t ljb;
974 int error = 0;
975 lwpchan_t lwpchan;
976 uint16_t flag;
977 upib_t *upibp;
978 volatile struct upimutex *upimutex = NULL;
979 volatile int upilocked = 0;
980
981 if (on_fault(&ljb)) {
982 if (upilocked)
983 upimutex_unlock((upimutex_t *)upimutex, 0);
984 error = EFAULT;
985 goto out;
986 }
987 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
988 &lwpchan, LWPCHAN_MPPOOL)) {
989 error = EFAULT;
990 goto out;
991 }
992 upibp = &UPI_CHAIN(lwpchan);
993 mutex_enter(&upibp->upib_lock);
994 upimutex = upi_get(upibp, &lwpchan);
995 /*
996 * If the lock is not held, or the owner is not curthread, return
997 * error. The user-level wrapper can return this error or stall,
998 * depending on whether mutex is of ERRORCHECK type or not.
999 */
1000 if (upimutex == NULL || upimutex->upi_owner != curthread) {
1001 mutex_exit(&upibp->upib_lock);
1002 error = EPERM;
1003 goto out;
1004 }
1005 mutex_exit(&upibp->upib_lock); /* release for user memory access */
1006 upilocked = 1;
1007 fuword16_noerr(&lp->mutex_flag, &flag);
1008 if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
1009 /*
1010 * transition mutex to the LOCK_NOTRECOVERABLE state.
1011 */
1012 flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
1013 flag |= LOCK_NOTRECOVERABLE;
1014 suword16_noerr(&lp->mutex_flag, flag);
1015 }
1016 set_owner_pid(lp, 0, 0);
1017 upimutex_unlock((upimutex_t *)upimutex, flag);
1018 upilocked = 0;
1019 out:
1020 no_fault();
1021 return (error);
1022 }
1023
1024 /*
1025 * Set the owner and ownerpid fields of a user-level mutex.
1026 */
1027 static void
set_owner_pid(lwp_mutex_t * lp,uintptr_t owner,pid_t pid)1028 set_owner_pid(lwp_mutex_t *lp, uintptr_t owner, pid_t pid)
1029 {
1030 union {
1031 uint64_t word64;
1032 uint32_t word32[2];
1033 } un;
1034
1035 un.word64 = (uint64_t)owner;
1036
1037 suword32_noerr(&lp->mutex_ownerpid, pid);
1038 #if defined(_LP64)
1039 if (((uintptr_t)lp & (_LONG_LONG_ALIGNMENT - 1)) == 0) { /* aligned */
1040 suword64_noerr(&lp->mutex_owner, un.word64);
1041 return;
1042 }
1043 #endif
1044 /* mutex is unaligned or we are running on a 32-bit kernel */
1045 suword32_noerr((uint32_t *)&lp->mutex_owner, un.word32[0]);
1046 suword32_noerr((uint32_t *)&lp->mutex_owner + 1, un.word32[1]);
1047 }
1048
1049 /*
1050 * Clear the contents of a user-level mutex; return the flags.
1051 * Used only by upi_dead() and lwp_mutex_cleanup(), below.
1052 */
1053 static uint16_t
lwp_clear_mutex(lwp_mutex_t * lp,uint16_t lockflg)1054 lwp_clear_mutex(lwp_mutex_t *lp, uint16_t lockflg)
1055 {
1056 uint16_t flag;
1057
1058 fuword16_noerr(&lp->mutex_flag, &flag);
1059 if ((flag &
1060 (LOCK_OWNERDEAD | LOCK_UNMAPPED | LOCK_NOTRECOVERABLE)) == 0) {
1061 flag |= lockflg;
1062 suword16_noerr(&lp->mutex_flag, flag);
1063 }
1064 set_owner_pid(lp, 0, 0);
1065 suword8_noerr(&lp->mutex_rcount, 0);
1066
1067 return (flag);
1068 }
1069
1070 /*
1071 * Mark user mutex state, corresponding to kernel upimutex,
1072 * as LOCK_UNMAPPED or LOCK_OWNERDEAD, as appropriate
1073 */
1074 static int
upi_dead(upimutex_t * upip,uint16_t lockflg)1075 upi_dead(upimutex_t *upip, uint16_t lockflg)
1076 {
1077 label_t ljb;
1078 int error = 0;
1079 lwp_mutex_t *lp;
1080
1081 if (on_fault(&ljb)) {
1082 error = EFAULT;
1083 goto out;
1084 }
1085
1086 lp = upip->upi_vaddr;
1087 (void) lwp_clear_mutex(lp, lockflg);
1088 suword8_noerr(&lp->mutex_lockw, 0);
1089 out:
1090 no_fault();
1091 return (error);
1092 }
1093
1094 /*
1095 * Unlock all upimutexes held by curthread, since curthread is dying.
1096 * For each upimutex, attempt to mark its corresponding user mutex object as
1097 * dead.
1098 */
1099 void
upimutex_cleanup()1100 upimutex_cleanup()
1101 {
1102 kthread_t *t = curthread;
1103 uint16_t lockflg = (ttoproc(t)->p_proc_flag & P_PR_EXEC)?
1104 LOCK_UNMAPPED : LOCK_OWNERDEAD;
1105 struct upimutex *upip;
1106
1107 while ((upip = t->t_upimutex) != NULL) {
1108 if (upi_dead(upip, lockflg) != 0) {
1109 /*
1110 * If the user object associated with this upimutex is
1111 * unmapped, unlock upimutex with the
1112 * LOCK_NOTRECOVERABLE flag, so that all waiters are
1113 * woken up. Since user object is unmapped, it could
1114 * not be marked as dead or notrecoverable.
1115 * The waiters will now all wake up and return
1116 * ENOTRECOVERABLE, since they would find that the lock
1117 * has not been handed-off to them.
1118 * See lwp_upimutex_lock().
1119 */
1120 upimutex_unlock(upip, LOCK_NOTRECOVERABLE);
1121 } else {
1122 /*
1123 * The user object has been updated as dead.
1124 * Unlock the upimutex: if no waiters, upip kmem will
1125 * be freed. If there is a waiter, the lock will be
1126 * handed off. If exit() is in progress, each existing
1127 * waiter will successively get the lock, as owners
1128 * die, and each new owner will call this routine as
1129 * it dies. The last owner will free kmem, since
1130 * it will find the upimutex has no waiters. So,
1131 * eventually, the kmem is guaranteed to be freed.
1132 */
1133 upimutex_unlock(upip, 0);
1134 }
1135 /*
1136 * Note that the call to upimutex_unlock() above will delete
1137 * upimutex from the t_upimutexes chain. And so the
1138 * while loop will eventually terminate.
1139 */
1140 }
1141 }
1142
1143 int
lwp_mutex_timedlock(lwp_mutex_t * lp,timespec_t * tsp,uintptr_t owner)1144 lwp_mutex_timedlock(lwp_mutex_t *lp, timespec_t *tsp, uintptr_t owner)
1145 {
1146 kthread_t *t = curthread;
1147 klwp_t *lwp = ttolwp(t);
1148 proc_t *p = ttoproc(t);
1149 lwp_timer_t lwpt;
1150 caddr_t timedwait;
1151 int error = 0;
1152 int time_error;
1153 clock_t tim = -1;
1154 uchar_t waiters;
1155 volatile int locked = 0;
1156 volatile int watched = 0;
1157 label_t ljb;
1158 volatile uint8_t type = 0;
1159 lwpchan_t lwpchan;
1160 sleepq_head_t *sqh;
1161 uint16_t flag;
1162 int imm_timeout = 0;
1163
1164 if ((caddr_t)lp >= p->p_as->a_userlimit)
1165 return (set_errno(EFAULT));
1166
1167 /*
1168 * Put the lwp in an orderly state for debugging,
1169 * in case we are stopped while sleeping, below.
1170 */
1171 prstop(PR_REQUESTED, 0);
1172
1173 timedwait = (caddr_t)tsp;
1174 if ((time_error = lwp_timer_copyin(&lwpt, tsp)) == 0 &&
1175 lwpt.lwpt_imm_timeout) {
1176 imm_timeout = 1;
1177 timedwait = NULL;
1178 }
1179
1180 /*
1181 * Although LMS_USER_LOCK implies "asleep waiting for user-mode lock",
1182 * this micro state is really a run state. If the thread indeed blocks,
1183 * this state becomes valid. If not, the state is converted back to
1184 * LMS_SYSTEM. So, it is OK to set the mstate here, instead of just
1185 * when blocking.
1186 */
1187 (void) new_mstate(t, LMS_USER_LOCK);
1188 if (on_fault(&ljb)) {
1189 if (locked)
1190 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
1191 error = EFAULT;
1192 goto out;
1193 }
1194 /*
1195 * Force Copy-on-write if necessary and ensure that the
1196 * synchronization object resides in read/write memory.
1197 * Cause an EFAULT return now if this is not so.
1198 */
1199 fuword8_noerr(&lp->mutex_type, (uint8_t *)&type);
1200 suword8_noerr(&lp->mutex_type, type);
1201 if (UPIMUTEX(type)) {
1202 no_fault();
1203 error = lwp_upimutex_lock(lp, type, UPIMUTEX_BLOCK, &lwpt);
1204 if (error == 0 || error == EOWNERDEAD || error == ELOCKUNMAPPED)
1205 set_owner_pid(lp, owner,
1206 (type & USYNC_PROCESS)? p->p_pid : 0);
1207 if (tsp && !time_error) /* copyout the residual time left */
1208 error = lwp_timer_copyout(&lwpt, error);
1209 if (error)
1210 return (set_errno(error));
1211 return (0);
1212 }
1213 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
1214 &lwpchan, LWPCHAN_MPPOOL)) {
1215 error = EFAULT;
1216 goto out;
1217 }
1218 lwpchan_lock(&lwpchan, LWPCHAN_MPPOOL);
1219 locked = 1;
1220 if (type & LOCK_ROBUST) {
1221 fuword16_noerr(&lp->mutex_flag, &flag);
1222 if (flag & LOCK_NOTRECOVERABLE) {
1223 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
1224 error = ENOTRECOVERABLE;
1225 goto out;
1226 }
1227 }
1228 fuword8_noerr(&lp->mutex_waiters, &waiters);
1229 suword8_noerr(&lp->mutex_waiters, 1);
1230
1231 /*
1232 * If watchpoints are set, they need to be restored, since
1233 * atomic accesses of memory such as the call to ulock_try()
1234 * below cannot be watched.
1235 */
1236
1237 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
1238
1239 while (!ulock_try(&lp->mutex_lockw)) {
1240 if (time_error) {
1241 /*
1242 * The SUSV3 Posix spec is very clear that we
1243 * should get no error from validating the
1244 * timer until we would actually sleep.
1245 */
1246 error = time_error;
1247 break;
1248 }
1249
1250 if (watched) {
1251 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
1252 watched = 0;
1253 }
1254
1255 if (timedwait) {
1256 /*
1257 * If we successfully queue the timeout,
1258 * then don't drop t_delay_lock until
1259 * we are on the sleep queue (below).
1260 */
1261 mutex_enter(&t->t_delay_lock);
1262 if (lwp_timer_enqueue(&lwpt) != 0) {
1263 mutex_exit(&t->t_delay_lock);
1264 imm_timeout = 1;
1265 timedwait = NULL;
1266 }
1267 }
1268 lwp_block(&lwpchan);
1269 /*
1270 * Nothing should happen to cause the lwp to go to
1271 * sleep again until after it returns from swtch().
1272 */
1273 if (timedwait)
1274 mutex_exit(&t->t_delay_lock);
1275 locked = 0;
1276 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
1277 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) || imm_timeout)
1278 setrun(t);
1279 swtch();
1280 t->t_flag &= ~T_WAKEABLE;
1281 if (timedwait)
1282 tim = lwp_timer_dequeue(&lwpt);
1283 setallwatch();
1284 if (ISSIG(t, FORREAL) || lwp->lwp_sysabort || MUSTRETURN(p, t))
1285 error = EINTR;
1286 else if (imm_timeout || (timedwait && tim == -1))
1287 error = ETIME;
1288 if (error) {
1289 lwp->lwp_asleep = 0;
1290 lwp->lwp_sysabort = 0;
1291 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp),
1292 S_WRITE);
1293
1294 /*
1295 * Need to re-compute waiters bit. The waiters field in
1296 * the lock is not reliable. Either of two things could
1297 * have occurred: no lwp may have called lwp_release()
1298 * for me but I have woken up due to a signal or
1299 * timeout. In this case, the waiter bit is incorrect
1300 * since it is still set to 1, set above.
1301 * OR an lwp_release() did occur for some other lwp on
1302 * the same lwpchan. In this case, the waiter bit is
1303 * correct. But which event occurred, one can't tell.
1304 * So, recompute.
1305 */
1306 lwpchan_lock(&lwpchan, LWPCHAN_MPPOOL);
1307 locked = 1;
1308 sqh = lwpsqhash(&lwpchan);
1309 disp_lock_enter(&sqh->sq_lock);
1310 waiters = iswanted(sqh->sq_queue.sq_first, &lwpchan);
1311 disp_lock_exit(&sqh->sq_lock);
1312 break;
1313 }
1314 lwp->lwp_asleep = 0;
1315 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp),
1316 S_WRITE);
1317 lwpchan_lock(&lwpchan, LWPCHAN_MPPOOL);
1318 locked = 1;
1319 fuword8_noerr(&lp->mutex_waiters, &waiters);
1320 suword8_noerr(&lp->mutex_waiters, 1);
1321 if (type & LOCK_ROBUST) {
1322 fuword16_noerr(&lp->mutex_flag, &flag);
1323 if (flag & LOCK_NOTRECOVERABLE) {
1324 error = ENOTRECOVERABLE;
1325 break;
1326 }
1327 }
1328 }
1329
1330 if (t->t_mstate == LMS_USER_LOCK)
1331 (void) new_mstate(t, LMS_SYSTEM);
1332
1333 if (error == 0) {
1334 set_owner_pid(lp, owner, (type & USYNC_PROCESS)? p->p_pid : 0);
1335 if (type & LOCK_ROBUST) {
1336 fuword16_noerr(&lp->mutex_flag, &flag);
1337 if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
1338 if (flag & LOCK_OWNERDEAD)
1339 error = EOWNERDEAD;
1340 else if (type & USYNC_PROCESS_ROBUST)
1341 error = ELOCKUNMAPPED;
1342 else
1343 error = EOWNERDEAD;
1344 }
1345 }
1346 }
1347 suword8_noerr(&lp->mutex_waiters, waiters);
1348 locked = 0;
1349 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
1350 out:
1351 no_fault();
1352 if (watched)
1353 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
1354 if (tsp && !time_error) /* copyout the residual time left */
1355 error = lwp_timer_copyout(&lwpt, error);
1356 if (error)
1357 return (set_errno(error));
1358 return (0);
1359 }
1360
1361 static int
iswanted(kthread_t * t,lwpchan_t * lwpchan)1362 iswanted(kthread_t *t, lwpchan_t *lwpchan)
1363 {
1364 /*
1365 * The caller holds the dispatcher lock on the sleep queue.
1366 */
1367 while (t != NULL) {
1368 if (t->t_lwpchan.lc_wchan0 == lwpchan->lc_wchan0 &&
1369 t->t_lwpchan.lc_wchan == lwpchan->lc_wchan)
1370 return (1);
1371 t = t->t_link;
1372 }
1373 return (0);
1374 }
1375
1376 /*
1377 * Return the highest priority thread sleeping on this lwpchan.
1378 */
1379 static kthread_t *
lwp_queue_waiter(lwpchan_t * lwpchan)1380 lwp_queue_waiter(lwpchan_t *lwpchan)
1381 {
1382 sleepq_head_t *sqh;
1383 kthread_t *tp;
1384
1385 sqh = lwpsqhash(lwpchan);
1386 disp_lock_enter(&sqh->sq_lock); /* lock the sleep queue */
1387 for (tp = sqh->sq_queue.sq_first; tp != NULL; tp = tp->t_link) {
1388 if (tp->t_lwpchan.lc_wchan0 == lwpchan->lc_wchan0 &&
1389 tp->t_lwpchan.lc_wchan == lwpchan->lc_wchan)
1390 break;
1391 }
1392 disp_lock_exit(&sqh->sq_lock);
1393 return (tp);
1394 }
1395
1396 static int
lwp_release(lwpchan_t * lwpchan,uchar_t * waiters,int sync_type)1397 lwp_release(lwpchan_t *lwpchan, uchar_t *waiters, int sync_type)
1398 {
1399 sleepq_head_t *sqh;
1400 kthread_t *tp;
1401 kthread_t **tpp;
1402
1403 sqh = lwpsqhash(lwpchan);
1404 disp_lock_enter(&sqh->sq_lock); /* lock the sleep queue */
1405 tpp = &sqh->sq_queue.sq_first;
1406 while ((tp = *tpp) != NULL) {
1407 if (tp->t_lwpchan.lc_wchan0 == lwpchan->lc_wchan0 &&
1408 tp->t_lwpchan.lc_wchan == lwpchan->lc_wchan) {
1409 /*
1410 * The following is typically false. It could be true
1411 * only if lwp_release() is called from
1412 * lwp_mutex_wakeup() after reading the waiters field
1413 * from memory in which the lwp lock used to be, but has
1414 * since been re-used to hold a lwp cv or lwp semaphore.
1415 * The thread "tp" found to match the lwp lock's wchan
1416 * is actually sleeping for the cv or semaphore which
1417 * now has the same wchan. In this case, lwp_release()
1418 * should return failure.
1419 */
1420 if (sync_type != (tp->t_flag & T_WAITCVSEM)) {
1421 ASSERT(sync_type == 0);
1422 /*
1423 * assert that this can happen only for mutexes
1424 * i.e. sync_type == 0, for correctly written
1425 * user programs.
1426 */
1427 disp_lock_exit(&sqh->sq_lock);
1428 return (0);
1429 }
1430 *waiters = iswanted(tp->t_link, lwpchan);
1431 sleepq_unlink(tpp, tp);
1432 DTRACE_SCHED1(wakeup, kthread_t *, tp);
1433 tp->t_wchan0 = NULL;
1434 tp->t_wchan = NULL;
1435 tp->t_sobj_ops = NULL;
1436 tp->t_release = 1;
1437 THREAD_TRANSITION(tp); /* drops sleepq lock */
1438 CL_WAKEUP(tp);
1439 thread_unlock(tp); /* drop run queue lock */
1440 return (1);
1441 }
1442 tpp = &tp->t_link;
1443 }
1444 *waiters = 0;
1445 disp_lock_exit(&sqh->sq_lock);
1446 return (0);
1447 }
1448
1449 static void
lwp_release_all(lwpchan_t * lwpchan)1450 lwp_release_all(lwpchan_t *lwpchan)
1451 {
1452 sleepq_head_t *sqh;
1453 kthread_t *tp;
1454 kthread_t **tpp;
1455
1456 sqh = lwpsqhash(lwpchan);
1457 disp_lock_enter(&sqh->sq_lock); /* lock sleep q queue */
1458 tpp = &sqh->sq_queue.sq_first;
1459 while ((tp = *tpp) != NULL) {
1460 if (tp->t_lwpchan.lc_wchan0 == lwpchan->lc_wchan0 &&
1461 tp->t_lwpchan.lc_wchan == lwpchan->lc_wchan) {
1462 sleepq_unlink(tpp, tp);
1463 DTRACE_SCHED1(wakeup, kthread_t *, tp);
1464 tp->t_wchan0 = NULL;
1465 tp->t_wchan = NULL;
1466 tp->t_sobj_ops = NULL;
1467 CL_WAKEUP(tp);
1468 thread_unlock_high(tp); /* release run queue lock */
1469 } else {
1470 tpp = &tp->t_link;
1471 }
1472 }
1473 disp_lock_exit(&sqh->sq_lock); /* drop sleep q lock */
1474 }
1475
1476 /*
1477 * unblock a lwp that is trying to acquire this mutex. the blocked
1478 * lwp resumes and retries to acquire the lock.
1479 */
1480 int
lwp_mutex_wakeup(lwp_mutex_t * lp,int release_all)1481 lwp_mutex_wakeup(lwp_mutex_t *lp, int release_all)
1482 {
1483 proc_t *p = ttoproc(curthread);
1484 lwpchan_t lwpchan;
1485 uchar_t waiters;
1486 volatile int locked = 0;
1487 volatile int watched = 0;
1488 volatile uint8_t type = 0;
1489 label_t ljb;
1490 int error = 0;
1491
1492 if ((caddr_t)lp >= p->p_as->a_userlimit)
1493 return (set_errno(EFAULT));
1494
1495 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
1496
1497 if (on_fault(&ljb)) {
1498 if (locked)
1499 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
1500 error = EFAULT;
1501 goto out;
1502 }
1503 /*
1504 * Force Copy-on-write if necessary and ensure that the
1505 * synchronization object resides in read/write memory.
1506 * Cause an EFAULT return now if this is not so.
1507 */
1508 fuword8_noerr(&lp->mutex_type, (uint8_t *)&type);
1509 suword8_noerr(&lp->mutex_type, type);
1510 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
1511 &lwpchan, LWPCHAN_MPPOOL)) {
1512 error = EFAULT;
1513 goto out;
1514 }
1515 lwpchan_lock(&lwpchan, LWPCHAN_MPPOOL);
1516 locked = 1;
1517 /*
1518 * Always wake up an lwp (if any) waiting on lwpchan. The woken lwp will
1519 * re-try the lock in lwp_mutex_timedlock(). The call to lwp_release()
1520 * may fail. If it fails, do not write into the waiter bit.
1521 * The call to lwp_release() might fail due to one of three reasons:
1522 *
1523 * 1. due to the thread which set the waiter bit not actually
1524 * sleeping since it got the lock on the re-try. The waiter
1525 * bit will then be correctly updated by that thread. This
1526 * window may be closed by reading the wait bit again here
1527 * and not calling lwp_release() at all if it is zero.
1528 * 2. the thread which set the waiter bit and went to sleep
1529 * was woken up by a signal. This time, the waiter recomputes
1530 * the wait bit in the return with EINTR code.
1531 * 3. the waiter bit read by lwp_mutex_wakeup() was in
1532 * memory that has been re-used after the lock was dropped.
1533 * In this case, writing into the waiter bit would cause data
1534 * corruption.
1535 */
1536 if (release_all)
1537 lwp_release_all(&lwpchan);
1538 else if (lwp_release(&lwpchan, &waiters, 0))
1539 suword8_noerr(&lp->mutex_waiters, waiters);
1540 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
1541 out:
1542 no_fault();
1543 if (watched)
1544 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
1545 if (error)
1546 return (set_errno(error));
1547 return (0);
1548 }
1549
1550 /*
1551 * lwp_cond_wait() has four arguments, a pointer to a condition variable,
1552 * a pointer to a mutex, a pointer to a timespec for a timed wait and
1553 * a flag telling the kernel whether or not to honor the kernel/user
1554 * schedctl parking protocol (see schedctl_is_park() in schedctl.c).
1555 * The kernel puts the lwp to sleep on a unique pair of caddr_t's called an
1556 * lwpchan, returned by get_lwpchan(). If the timespec pointer is non-NULL,
1557 * it is used an an in/out parameter. On entry, it contains the relative
1558 * time until timeout. On exit, we copyout the residual time left to it.
1559 */
1560 int
lwp_cond_wait(lwp_cond_t * cv,lwp_mutex_t * mp,timespec_t * tsp,int check_park)1561 lwp_cond_wait(lwp_cond_t *cv, lwp_mutex_t *mp, timespec_t *tsp, int check_park)
1562 {
1563 kthread_t *t = curthread;
1564 klwp_t *lwp = ttolwp(t);
1565 proc_t *p = ttoproc(t);
1566 lwp_timer_t lwpt;
1567 lwpchan_t cv_lwpchan;
1568 lwpchan_t m_lwpchan;
1569 caddr_t timedwait;
1570 volatile uint16_t type = 0;
1571 volatile uint8_t mtype = 0;
1572 uchar_t waiters;
1573 volatile int error;
1574 clock_t tim = -1;
1575 volatile int locked = 0;
1576 volatile int m_locked = 0;
1577 volatile int cvwatched = 0;
1578 volatile int mpwatched = 0;
1579 label_t ljb;
1580 volatile int no_lwpchan = 1;
1581 int imm_timeout = 0;
1582 int imm_unpark = 0;
1583
1584 if ((caddr_t)cv >= p->p_as->a_userlimit ||
1585 (caddr_t)mp >= p->p_as->a_userlimit)
1586 return (set_errno(EFAULT));
1587
1588 /*
1589 * Put the lwp in an orderly state for debugging,
1590 * in case we are stopped while sleeping, below.
1591 */
1592 prstop(PR_REQUESTED, 0);
1593
1594 timedwait = (caddr_t)tsp;
1595 if ((error = lwp_timer_copyin(&lwpt, tsp)) != 0)
1596 return (set_errno(error));
1597 if (lwpt.lwpt_imm_timeout) {
1598 imm_timeout = 1;
1599 timedwait = NULL;
1600 }
1601
1602 (void) new_mstate(t, LMS_USER_LOCK);
1603
1604 if (on_fault(&ljb)) {
1605 if (no_lwpchan) {
1606 error = EFAULT;
1607 goto out;
1608 }
1609 if (m_locked) {
1610 m_locked = 0;
1611 lwpchan_unlock(&m_lwpchan, LWPCHAN_MPPOOL);
1612 }
1613 if (locked) {
1614 locked = 0;
1615 lwpchan_unlock(&cv_lwpchan, LWPCHAN_CVPOOL);
1616 }
1617 /*
1618 * set up another on_fault() for a possible fault
1619 * on the user lock accessed at "efault"
1620 */
1621 if (on_fault(&ljb)) {
1622 if (m_locked) {
1623 m_locked = 0;
1624 lwpchan_unlock(&m_lwpchan, LWPCHAN_MPPOOL);
1625 }
1626 goto out;
1627 }
1628 error = EFAULT;
1629 goto efault;
1630 }
1631
1632 /*
1633 * Force Copy-on-write if necessary and ensure that the
1634 * synchronization object resides in read/write memory.
1635 * Cause an EFAULT return now if this is not so.
1636 */
1637 fuword8_noerr(&mp->mutex_type, (uint8_t *)&mtype);
1638 suword8_noerr(&mp->mutex_type, mtype);
1639 if (UPIMUTEX(mtype) == 0) {
1640 /* convert user level mutex, "mp", to a unique lwpchan */
1641 /* check if mtype is ok to use below, instead of type from cv */
1642 if (!get_lwpchan(p->p_as, (caddr_t)mp, mtype,
1643 &m_lwpchan, LWPCHAN_MPPOOL)) {
1644 error = EFAULT;
1645 goto out;
1646 }
1647 }
1648 fuword16_noerr(&cv->cond_type, (uint16_t *)&type);
1649 suword16_noerr(&cv->cond_type, type);
1650 /* convert user level condition variable, "cv", to a unique lwpchan */
1651 if (!get_lwpchan(p->p_as, (caddr_t)cv, type,
1652 &cv_lwpchan, LWPCHAN_CVPOOL)) {
1653 error = EFAULT;
1654 goto out;
1655 }
1656 no_lwpchan = 0;
1657 cvwatched = watch_disable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1658 if (UPIMUTEX(mtype) == 0)
1659 mpwatched = watch_disable_addr((caddr_t)mp, sizeof (*mp),
1660 S_WRITE);
1661
1662 /*
1663 * lwpchan_lock ensures that the calling lwp is put to sleep atomically
1664 * with respect to a possible wakeup which is a result of either
1665 * an lwp_cond_signal() or an lwp_cond_broadcast().
1666 *
1667 * What's misleading, is that the lwp is put to sleep after the
1668 * condition variable's mutex is released. This is OK as long as
1669 * the release operation is also done while holding lwpchan_lock.
1670 * The lwp is then put to sleep when the possibility of pagefaulting
1671 * or sleeping is completely eliminated.
1672 */
1673 lwpchan_lock(&cv_lwpchan, LWPCHAN_CVPOOL);
1674 locked = 1;
1675 if (UPIMUTEX(mtype) == 0) {
1676 lwpchan_lock(&m_lwpchan, LWPCHAN_MPPOOL);
1677 m_locked = 1;
1678 suword8_noerr(&cv->cond_waiters_kernel, 1);
1679 /*
1680 * unlock the condition variable's mutex. (pagefaults are
1681 * possible here.)
1682 */
1683 set_owner_pid(mp, 0, 0);
1684 ulock_clear(&mp->mutex_lockw);
1685 fuword8_noerr(&mp->mutex_waiters, &waiters);
1686 if (waiters != 0) {
1687 /*
1688 * Given the locking of lwpchan_lock around the release
1689 * of the mutex and checking for waiters, the following
1690 * call to lwp_release() can fail ONLY if the lock
1691 * acquirer is interrupted after setting the waiter bit,
1692 * calling lwp_block() and releasing lwpchan_lock.
1693 * In this case, it could get pulled off the lwp sleep
1694 * q (via setrun()) before the following call to
1695 * lwp_release() occurs. In this case, the lock
1696 * requestor will update the waiter bit correctly by
1697 * re-evaluating it.
1698 */
1699 if (lwp_release(&m_lwpchan, &waiters, 0))
1700 suword8_noerr(&mp->mutex_waiters, waiters);
1701 }
1702 m_locked = 0;
1703 lwpchan_unlock(&m_lwpchan, LWPCHAN_MPPOOL);
1704 } else {
1705 suword8_noerr(&cv->cond_waiters_kernel, 1);
1706 error = lwp_upimutex_unlock(mp, mtype);
1707 if (error) { /* if the upimutex unlock failed */
1708 locked = 0;
1709 lwpchan_unlock(&cv_lwpchan, LWPCHAN_CVPOOL);
1710 goto out;
1711 }
1712 }
1713 no_fault();
1714
1715 if (mpwatched) {
1716 watch_enable_addr((caddr_t)mp, sizeof (*mp), S_WRITE);
1717 mpwatched = 0;
1718 }
1719 if (cvwatched) {
1720 watch_enable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1721 cvwatched = 0;
1722 }
1723
1724 if (check_park && (!schedctl_is_park() || t->t_unpark)) {
1725 /*
1726 * We received a signal at user-level before calling here
1727 * or another thread wants us to return immediately
1728 * with EINTR. See lwp_unpark().
1729 */
1730 imm_unpark = 1;
1731 t->t_unpark = 0;
1732 timedwait = NULL;
1733 } else if (timedwait) {
1734 /*
1735 * If we successfully queue the timeout,
1736 * then don't drop t_delay_lock until
1737 * we are on the sleep queue (below).
1738 */
1739 mutex_enter(&t->t_delay_lock);
1740 if (lwp_timer_enqueue(&lwpt) != 0) {
1741 mutex_exit(&t->t_delay_lock);
1742 imm_timeout = 1;
1743 timedwait = NULL;
1744 }
1745 }
1746 t->t_flag |= T_WAITCVSEM;
1747 lwp_block(&cv_lwpchan);
1748 /*
1749 * Nothing should happen to cause the lwp to go to sleep
1750 * until after it returns from swtch().
1751 */
1752 if (timedwait)
1753 mutex_exit(&t->t_delay_lock);
1754 locked = 0;
1755 lwpchan_unlock(&cv_lwpchan, LWPCHAN_CVPOOL);
1756 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) ||
1757 (imm_timeout | imm_unpark))
1758 setrun(t);
1759 swtch();
1760 t->t_flag &= ~(T_WAITCVSEM | T_WAKEABLE);
1761 if (timedwait)
1762 tim = lwp_timer_dequeue(&lwpt);
1763 if (ISSIG(t, FORREAL) || lwp->lwp_sysabort ||
1764 MUSTRETURN(p, t) || imm_unpark)
1765 error = EINTR;
1766 else if (imm_timeout || (timedwait && tim == -1))
1767 error = ETIME;
1768 lwp->lwp_asleep = 0;
1769 lwp->lwp_sysabort = 0;
1770 setallwatch();
1771
1772 if (t->t_mstate == LMS_USER_LOCK)
1773 (void) new_mstate(t, LMS_SYSTEM);
1774
1775 if (tsp && check_park) /* copyout the residual time left */
1776 error = lwp_timer_copyout(&lwpt, error);
1777
1778 /* the mutex is reacquired by the caller on return to user level */
1779 if (error) {
1780 /*
1781 * If we were concurrently lwp_cond_signal()d and we
1782 * received a UNIX signal or got a timeout, then perform
1783 * another lwp_cond_signal() to avoid consuming the wakeup.
1784 */
1785 if (t->t_release)
1786 (void) lwp_cond_signal(cv);
1787 return (set_errno(error));
1788 }
1789 return (0);
1790
1791 efault:
1792 /*
1793 * make sure that the user level lock is dropped before
1794 * returning to caller, since the caller always re-acquires it.
1795 */
1796 if (UPIMUTEX(mtype) == 0) {
1797 lwpchan_lock(&m_lwpchan, LWPCHAN_MPPOOL);
1798 m_locked = 1;
1799 set_owner_pid(mp, 0, 0);
1800 ulock_clear(&mp->mutex_lockw);
1801 fuword8_noerr(&mp->mutex_waiters, &waiters);
1802 if (waiters != 0) {
1803 /*
1804 * See comment above on lock clearing and lwp_release()
1805 * success/failure.
1806 */
1807 if (lwp_release(&m_lwpchan, &waiters, 0))
1808 suword8_noerr(&mp->mutex_waiters, waiters);
1809 }
1810 m_locked = 0;
1811 lwpchan_unlock(&m_lwpchan, LWPCHAN_MPPOOL);
1812 } else {
1813 (void) lwp_upimutex_unlock(mp, mtype);
1814 }
1815 out:
1816 no_fault();
1817 if (mpwatched)
1818 watch_enable_addr((caddr_t)mp, sizeof (*mp), S_WRITE);
1819 if (cvwatched)
1820 watch_enable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1821 if (t->t_mstate == LMS_USER_LOCK)
1822 (void) new_mstate(t, LMS_SYSTEM);
1823 return (set_errno(error));
1824 }
1825
1826 /*
1827 * wakeup one lwp that's blocked on this condition variable.
1828 */
1829 int
lwp_cond_signal(lwp_cond_t * cv)1830 lwp_cond_signal(lwp_cond_t *cv)
1831 {
1832 proc_t *p = ttoproc(curthread);
1833 lwpchan_t lwpchan;
1834 uchar_t waiters;
1835 volatile uint16_t type = 0;
1836 volatile int locked = 0;
1837 volatile int watched = 0;
1838 label_t ljb;
1839 int error = 0;
1840
1841 if ((caddr_t)cv >= p->p_as->a_userlimit)
1842 return (set_errno(EFAULT));
1843
1844 watched = watch_disable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1845
1846 if (on_fault(&ljb)) {
1847 if (locked)
1848 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
1849 error = EFAULT;
1850 goto out;
1851 }
1852 /*
1853 * Force Copy-on-write if necessary and ensure that the
1854 * synchronization object resides in read/write memory.
1855 * Cause an EFAULT return now if this is not so.
1856 */
1857 fuword16_noerr(&cv->cond_type, (uint16_t *)&type);
1858 suword16_noerr(&cv->cond_type, type);
1859 if (!get_lwpchan(curproc->p_as, (caddr_t)cv, type,
1860 &lwpchan, LWPCHAN_CVPOOL)) {
1861 error = EFAULT;
1862 goto out;
1863 }
1864 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
1865 locked = 1;
1866 fuword8_noerr(&cv->cond_waiters_kernel, &waiters);
1867 if (waiters != 0) {
1868 /*
1869 * The following call to lwp_release() might fail but it is
1870 * OK to write into the waiters bit below, since the memory
1871 * could not have been re-used or unmapped (for correctly
1872 * written user programs) as in the case of lwp_mutex_wakeup().
1873 * For an incorrect program, we should not care about data
1874 * corruption since this is just one instance of other places
1875 * where corruption can occur for such a program. Of course
1876 * if the memory is unmapped, normal fault recovery occurs.
1877 */
1878 (void) lwp_release(&lwpchan, &waiters, T_WAITCVSEM);
1879 suword8_noerr(&cv->cond_waiters_kernel, waiters);
1880 }
1881 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
1882 out:
1883 no_fault();
1884 if (watched)
1885 watch_enable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1886 if (error)
1887 return (set_errno(error));
1888 return (0);
1889 }
1890
1891 /*
1892 * wakeup every lwp that's blocked on this condition variable.
1893 */
1894 int
lwp_cond_broadcast(lwp_cond_t * cv)1895 lwp_cond_broadcast(lwp_cond_t *cv)
1896 {
1897 proc_t *p = ttoproc(curthread);
1898 lwpchan_t lwpchan;
1899 volatile uint16_t type = 0;
1900 volatile int locked = 0;
1901 volatile int watched = 0;
1902 label_t ljb;
1903 uchar_t waiters;
1904 int error = 0;
1905
1906 if ((caddr_t)cv >= p->p_as->a_userlimit)
1907 return (set_errno(EFAULT));
1908
1909 watched = watch_disable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1910
1911 if (on_fault(&ljb)) {
1912 if (locked)
1913 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
1914 error = EFAULT;
1915 goto out;
1916 }
1917 /*
1918 * Force Copy-on-write if necessary and ensure that the
1919 * synchronization object resides in read/write memory.
1920 * Cause an EFAULT return now if this is not so.
1921 */
1922 fuword16_noerr(&cv->cond_type, (uint16_t *)&type);
1923 suword16_noerr(&cv->cond_type, type);
1924 if (!get_lwpchan(curproc->p_as, (caddr_t)cv, type,
1925 &lwpchan, LWPCHAN_CVPOOL)) {
1926 error = EFAULT;
1927 goto out;
1928 }
1929 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
1930 locked = 1;
1931 fuword8_noerr(&cv->cond_waiters_kernel, &waiters);
1932 if (waiters != 0) {
1933 lwp_release_all(&lwpchan);
1934 suword8_noerr(&cv->cond_waiters_kernel, 0);
1935 }
1936 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
1937 out:
1938 no_fault();
1939 if (watched)
1940 watch_enable_addr((caddr_t)cv, sizeof (*cv), S_WRITE);
1941 if (error)
1942 return (set_errno(error));
1943 return (0);
1944 }
1945
1946 int
lwp_sema_trywait(lwp_sema_t * sp)1947 lwp_sema_trywait(lwp_sema_t *sp)
1948 {
1949 kthread_t *t = curthread;
1950 proc_t *p = ttoproc(t);
1951 label_t ljb;
1952 volatile int locked = 0;
1953 volatile int watched = 0;
1954 volatile uint16_t type = 0;
1955 int count;
1956 lwpchan_t lwpchan;
1957 uchar_t waiters;
1958 int error = 0;
1959
1960 if ((caddr_t)sp >= p->p_as->a_userlimit)
1961 return (set_errno(EFAULT));
1962
1963 watched = watch_disable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
1964
1965 if (on_fault(&ljb)) {
1966 if (locked)
1967 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
1968 error = EFAULT;
1969 goto out;
1970 }
1971 /*
1972 * Force Copy-on-write if necessary and ensure that the
1973 * synchronization object resides in read/write memory.
1974 * Cause an EFAULT return now if this is not so.
1975 */
1976 fuword16_noerr((void *)&sp->sema_type, (uint16_t *)&type);
1977 suword16_noerr((void *)&sp->sema_type, type);
1978 if (!get_lwpchan(p->p_as, (caddr_t)sp, type,
1979 &lwpchan, LWPCHAN_CVPOOL)) {
1980 error = EFAULT;
1981 goto out;
1982 }
1983 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
1984 locked = 1;
1985 fuword32_noerr((void *)&sp->sema_count, (uint32_t *)&count);
1986 if (count == 0)
1987 error = EBUSY;
1988 else
1989 suword32_noerr((void *)&sp->sema_count, --count);
1990 if (count != 0) {
1991 fuword8_noerr(&sp->sema_waiters, &waiters);
1992 if (waiters != 0) {
1993 (void) lwp_release(&lwpchan, &waiters, T_WAITCVSEM);
1994 suword8_noerr(&sp->sema_waiters, waiters);
1995 }
1996 }
1997 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
1998 out:
1999 no_fault();
2000 if (watched)
2001 watch_enable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
2002 if (error)
2003 return (set_errno(error));
2004 return (0);
2005 }
2006
2007 /*
2008 * See lwp_cond_wait(), above, for an explanation of the 'check_park' argument.
2009 */
2010 int
lwp_sema_timedwait(lwp_sema_t * sp,timespec_t * tsp,int check_park)2011 lwp_sema_timedwait(lwp_sema_t *sp, timespec_t *tsp, int check_park)
2012 {
2013 kthread_t *t = curthread;
2014 klwp_t *lwp = ttolwp(t);
2015 proc_t *p = ttoproc(t);
2016 lwp_timer_t lwpt;
2017 caddr_t timedwait;
2018 clock_t tim = -1;
2019 label_t ljb;
2020 volatile int locked = 0;
2021 volatile int watched = 0;
2022 volatile uint16_t type = 0;
2023 int count;
2024 lwpchan_t lwpchan;
2025 uchar_t waiters;
2026 int error = 0;
2027 int time_error;
2028 int imm_timeout = 0;
2029 int imm_unpark = 0;
2030
2031 if ((caddr_t)sp >= p->p_as->a_userlimit)
2032 return (set_errno(EFAULT));
2033
2034 /*
2035 * Put the lwp in an orderly state for debugging,
2036 * in case we are stopped while sleeping, below.
2037 */
2038 prstop(PR_REQUESTED, 0);
2039
2040 timedwait = (caddr_t)tsp;
2041 if ((time_error = lwp_timer_copyin(&lwpt, tsp)) == 0 &&
2042 lwpt.lwpt_imm_timeout) {
2043 imm_timeout = 1;
2044 timedwait = NULL;
2045 }
2046
2047 watched = watch_disable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
2048
2049 if (on_fault(&ljb)) {
2050 if (locked)
2051 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2052 error = EFAULT;
2053 goto out;
2054 }
2055 /*
2056 * Force Copy-on-write if necessary and ensure that the
2057 * synchronization object resides in read/write memory.
2058 * Cause an EFAULT return now if this is not so.
2059 */
2060 fuword16_noerr((void *)&sp->sema_type, (uint16_t *)&type);
2061 suword16_noerr((void *)&sp->sema_type, type);
2062 if (!get_lwpchan(p->p_as, (caddr_t)sp, type,
2063 &lwpchan, LWPCHAN_CVPOOL)) {
2064 error = EFAULT;
2065 goto out;
2066 }
2067 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
2068 locked = 1;
2069 fuword32_noerr((void *)&sp->sema_count, (uint32_t *)&count);
2070 while (error == 0 && count == 0) {
2071 if (time_error) {
2072 /*
2073 * The SUSV3 Posix spec is very clear that we
2074 * should get no error from validating the
2075 * timer until we would actually sleep.
2076 */
2077 error = time_error;
2078 break;
2079 }
2080 suword8_noerr(&sp->sema_waiters, 1);
2081 if (watched)
2082 watch_enable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
2083 if (check_park && (!schedctl_is_park() || t->t_unpark)) {
2084 /*
2085 * We received a signal at user-level before calling
2086 * here or another thread wants us to return
2087 * immediately with EINTR. See lwp_unpark().
2088 */
2089 imm_unpark = 1;
2090 t->t_unpark = 0;
2091 timedwait = NULL;
2092 } else if (timedwait) {
2093 /*
2094 * If we successfully queue the timeout,
2095 * then don't drop t_delay_lock until
2096 * we are on the sleep queue (below).
2097 */
2098 mutex_enter(&t->t_delay_lock);
2099 if (lwp_timer_enqueue(&lwpt) != 0) {
2100 mutex_exit(&t->t_delay_lock);
2101 imm_timeout = 1;
2102 timedwait = NULL;
2103 }
2104 }
2105 t->t_flag |= T_WAITCVSEM;
2106 lwp_block(&lwpchan);
2107 /*
2108 * Nothing should happen to cause the lwp to sleep
2109 * again until after it returns from swtch().
2110 */
2111 if (timedwait)
2112 mutex_exit(&t->t_delay_lock);
2113 locked = 0;
2114 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2115 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) ||
2116 (imm_timeout | imm_unpark))
2117 setrun(t);
2118 swtch();
2119 t->t_flag &= ~(T_WAITCVSEM | T_WAKEABLE);
2120 if (timedwait)
2121 tim = lwp_timer_dequeue(&lwpt);
2122 setallwatch();
2123 if (ISSIG(t, FORREAL) || lwp->lwp_sysabort ||
2124 MUSTRETURN(p, t) || imm_unpark)
2125 error = EINTR;
2126 else if (imm_timeout || (timedwait && tim == -1))
2127 error = ETIME;
2128 lwp->lwp_asleep = 0;
2129 lwp->lwp_sysabort = 0;
2130 watched = watch_disable_addr((caddr_t)sp,
2131 sizeof (*sp), S_WRITE);
2132 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
2133 locked = 1;
2134 fuword32_noerr((void *)&sp->sema_count, (uint32_t *)&count);
2135 }
2136 if (error == 0)
2137 suword32_noerr((void *)&sp->sema_count, --count);
2138 if (count != 0) {
2139 (void) lwp_release(&lwpchan, &waiters, T_WAITCVSEM);
2140 suword8_noerr(&sp->sema_waiters, waiters);
2141 }
2142 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2143 out:
2144 no_fault();
2145 if (watched)
2146 watch_enable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
2147 if (tsp && check_park && !time_error)
2148 error = lwp_timer_copyout(&lwpt, error);
2149 if (error)
2150 return (set_errno(error));
2151 return (0);
2152 }
2153
2154 int
lwp_sema_post(lwp_sema_t * sp)2155 lwp_sema_post(lwp_sema_t *sp)
2156 {
2157 proc_t *p = ttoproc(curthread);
2158 label_t ljb;
2159 volatile int locked = 0;
2160 volatile int watched = 0;
2161 volatile uint16_t type = 0;
2162 int count;
2163 lwpchan_t lwpchan;
2164 uchar_t waiters;
2165 int error = 0;
2166
2167 if ((caddr_t)sp >= p->p_as->a_userlimit)
2168 return (set_errno(EFAULT));
2169
2170 watched = watch_disable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
2171
2172 if (on_fault(&ljb)) {
2173 if (locked)
2174 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2175 error = EFAULT;
2176 goto out;
2177 }
2178 /*
2179 * Force Copy-on-write if necessary and ensure that the
2180 * synchronization object resides in read/write memory.
2181 * Cause an EFAULT return now if this is not so.
2182 */
2183 fuword16_noerr(&sp->sema_type, (uint16_t *)&type);
2184 suword16_noerr(&sp->sema_type, type);
2185 if (!get_lwpchan(curproc->p_as, (caddr_t)sp, type,
2186 &lwpchan, LWPCHAN_CVPOOL)) {
2187 error = EFAULT;
2188 goto out;
2189 }
2190 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
2191 locked = 1;
2192 fuword32_noerr(&sp->sema_count, (uint32_t *)&count);
2193 if (count == _SEM_VALUE_MAX)
2194 error = EOVERFLOW;
2195 else
2196 suword32_noerr(&sp->sema_count, ++count);
2197 if (count == 1) {
2198 fuword8_noerr(&sp->sema_waiters, &waiters);
2199 if (waiters) {
2200 (void) lwp_release(&lwpchan, &waiters, T_WAITCVSEM);
2201 suword8_noerr(&sp->sema_waiters, waiters);
2202 }
2203 }
2204 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2205 out:
2206 no_fault();
2207 if (watched)
2208 watch_enable_addr((caddr_t)sp, sizeof (*sp), S_WRITE);
2209 if (error)
2210 return (set_errno(error));
2211 return (0);
2212 }
2213
2214 #define TRW_WANT_WRITE 0x1
2215 #define TRW_LOCK_GRANTED 0x2
2216
2217 #define READ_LOCK 0
2218 #define WRITE_LOCK 1
2219 #define TRY_FLAG 0x10
2220 #define READ_LOCK_TRY (READ_LOCK | TRY_FLAG)
2221 #define WRITE_LOCK_TRY (WRITE_LOCK | TRY_FLAG)
2222
2223 /*
2224 * Release one writer or one or more readers. Compute the rwstate word to
2225 * reflect the new state of the queue. For a safe hand-off we copy the new
2226 * rwstate value back to userland before we wake any of the new lock holders.
2227 *
2228 * Note that sleepq_insert() implements a prioritized FIFO (with writers
2229 * being given precedence over readers of the same priority).
2230 *
2231 * If the first thread is a reader we scan the queue releasing all readers
2232 * until we hit a writer or the end of the queue. If the first thread is a
2233 * writer we still need to check for another writer.
2234 */
2235 void
lwp_rwlock_release(lwpchan_t * lwpchan,lwp_rwlock_t * rw)2236 lwp_rwlock_release(lwpchan_t *lwpchan, lwp_rwlock_t *rw)
2237 {
2238 sleepq_head_t *sqh;
2239 kthread_t *tp;
2240 kthread_t **tpp;
2241 kthread_t *tpnext;
2242 kthread_t *wakelist = NULL;
2243 uint32_t rwstate = 0;
2244 int wcount = 0;
2245 int rcount = 0;
2246
2247 sqh = lwpsqhash(lwpchan);
2248 disp_lock_enter(&sqh->sq_lock);
2249 tpp = &sqh->sq_queue.sq_first;
2250 while ((tp = *tpp) != NULL) {
2251 if (tp->t_lwpchan.lc_wchan0 == lwpchan->lc_wchan0 &&
2252 tp->t_lwpchan.lc_wchan == lwpchan->lc_wchan) {
2253 if (tp->t_writer & TRW_WANT_WRITE) {
2254 if ((wcount++ == 0) && (rcount == 0)) {
2255 rwstate |= URW_WRITE_LOCKED;
2256
2257 /* Just one writer to wake. */
2258 sleepq_unlink(tpp, tp);
2259 wakelist = tp;
2260
2261 /* tpp already set for next thread. */
2262 continue;
2263 } else {
2264 rwstate |= URW_HAS_WAITERS;
2265 /* We need look no further. */
2266 break;
2267 }
2268 } else {
2269 rcount++;
2270 if (wcount == 0) {
2271 rwstate++;
2272
2273 /* Add reader to wake list. */
2274 sleepq_unlink(tpp, tp);
2275 tp->t_link = wakelist;
2276 wakelist = tp;
2277
2278 /* tpp already set for next thread. */
2279 continue;
2280 } else {
2281 rwstate |= URW_HAS_WAITERS;
2282 /* We need look no further. */
2283 break;
2284 }
2285 }
2286 }
2287 tpp = &tp->t_link;
2288 }
2289
2290 /* Copy the new rwstate back to userland. */
2291 suword32_noerr(&rw->rwlock_readers, rwstate);
2292
2293 /* Wake the new lock holder(s) up. */
2294 tp = wakelist;
2295 while (tp != NULL) {
2296 DTRACE_SCHED1(wakeup, kthread_t *, tp);
2297 tp->t_wchan0 = NULL;
2298 tp->t_wchan = NULL;
2299 tp->t_sobj_ops = NULL;
2300 tp->t_writer |= TRW_LOCK_GRANTED;
2301 tpnext = tp->t_link;
2302 tp->t_link = NULL;
2303 CL_WAKEUP(tp);
2304 thread_unlock_high(tp);
2305 tp = tpnext;
2306 }
2307
2308 disp_lock_exit(&sqh->sq_lock);
2309 }
2310
2311 /*
2312 * We enter here holding the user-level mutex, which we must release before
2313 * returning or blocking. Based on lwp_cond_wait().
2314 */
2315 static int
lwp_rwlock_lock(lwp_rwlock_t * rw,timespec_t * tsp,int rd_wr)2316 lwp_rwlock_lock(lwp_rwlock_t *rw, timespec_t *tsp, int rd_wr)
2317 {
2318 lwp_mutex_t *mp = NULL;
2319 kthread_t *t = curthread;
2320 kthread_t *tp;
2321 klwp_t *lwp = ttolwp(t);
2322 proc_t *p = ttoproc(t);
2323 lwp_timer_t lwpt;
2324 lwpchan_t lwpchan;
2325 lwpchan_t mlwpchan;
2326 caddr_t timedwait;
2327 volatile uint16_t type = 0;
2328 volatile uint8_t mtype = 0;
2329 uchar_t mwaiters;
2330 volatile int error = 0;
2331 int time_error;
2332 clock_t tim = -1;
2333 volatile int locked = 0;
2334 volatile int mlocked = 0;
2335 volatile int watched = 0;
2336 volatile int mwatched = 0;
2337 label_t ljb;
2338 volatile int no_lwpchan = 1;
2339 int imm_timeout = 0;
2340 int try_flag;
2341 uint32_t rwstate;
2342 int acquired = 0;
2343
2344 /* We only check rw because the mutex is included in it. */
2345 if ((caddr_t)rw >= p->p_as->a_userlimit)
2346 return (set_errno(EFAULT));
2347
2348 /*
2349 * Put the lwp in an orderly state for debugging,
2350 * in case we are stopped while sleeping, below.
2351 */
2352 prstop(PR_REQUESTED, 0);
2353
2354 /* We must only report this error if we are about to sleep (later). */
2355 timedwait = (caddr_t)tsp;
2356 if ((time_error = lwp_timer_copyin(&lwpt, tsp)) == 0 &&
2357 lwpt.lwpt_imm_timeout) {
2358 imm_timeout = 1;
2359 timedwait = NULL;
2360 }
2361
2362 (void) new_mstate(t, LMS_USER_LOCK);
2363
2364 if (on_fault(&ljb)) {
2365 if (no_lwpchan) {
2366 error = EFAULT;
2367 goto out_nodrop;
2368 }
2369 if (mlocked) {
2370 mlocked = 0;
2371 lwpchan_unlock(&mlwpchan, LWPCHAN_MPPOOL);
2372 }
2373 if (locked) {
2374 locked = 0;
2375 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2376 }
2377 /*
2378 * Set up another on_fault() for a possible fault
2379 * on the user lock accessed at "out_drop".
2380 */
2381 if (on_fault(&ljb)) {
2382 if (mlocked) {
2383 mlocked = 0;
2384 lwpchan_unlock(&mlwpchan, LWPCHAN_MPPOOL);
2385 }
2386 error = EFAULT;
2387 goto out_nodrop;
2388 }
2389 error = EFAULT;
2390 goto out_nodrop;
2391 }
2392
2393 /* Process rd_wr (including sanity check). */
2394 try_flag = (rd_wr & TRY_FLAG);
2395 rd_wr &= ~TRY_FLAG;
2396 if ((rd_wr != READ_LOCK) && (rd_wr != WRITE_LOCK)) {
2397 error = EINVAL;
2398 goto out_nodrop;
2399 }
2400
2401 /*
2402 * Force Copy-on-write if necessary and ensure that the
2403 * synchronization object resides in read/write memory.
2404 * Cause an EFAULT return now if this is not so.
2405 */
2406 mp = &rw->mutex;
2407 fuword8_noerr(&mp->mutex_type, (uint8_t *)&mtype);
2408 fuword16_noerr(&rw->rwlock_type, (uint16_t *)&type);
2409 suword8_noerr(&mp->mutex_type, mtype);
2410 suword16_noerr(&rw->rwlock_type, type);
2411
2412 /* We can only continue for simple USYNC_PROCESS locks. */
2413 if ((mtype != USYNC_PROCESS) || (type != USYNC_PROCESS)) {
2414 error = EINVAL;
2415 goto out_nodrop;
2416 }
2417
2418 /* Convert user level mutex, "mp", to a unique lwpchan. */
2419 if (!get_lwpchan(p->p_as, (caddr_t)mp, mtype,
2420 &mlwpchan, LWPCHAN_MPPOOL)) {
2421 error = EFAULT;
2422 goto out_nodrop;
2423 }
2424
2425 /* Convert user level rwlock, "rw", to a unique lwpchan. */
2426 if (!get_lwpchan(p->p_as, (caddr_t)rw, type,
2427 &lwpchan, LWPCHAN_CVPOOL)) {
2428 error = EFAULT;
2429 goto out_nodrop;
2430 }
2431
2432 no_lwpchan = 0;
2433 watched = watch_disable_addr((caddr_t)rw, sizeof (*rw), S_WRITE);
2434 mwatched = watch_disable_addr((caddr_t)mp, sizeof (*mp), S_WRITE);
2435
2436 /*
2437 * lwpchan_lock() ensures that the calling LWP is put to sleep
2438 * atomically with respect to a possible wakeup which is a result
2439 * of lwp_rwlock_unlock().
2440 *
2441 * What's misleading is that the LWP is put to sleep after the
2442 * rwlock's mutex is released. This is OK as long as the release
2443 * operation is also done while holding mlwpchan. The LWP is then
2444 * put to sleep when the possibility of pagefaulting or sleeping
2445 * has been completely eliminated.
2446 */
2447 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
2448 locked = 1;
2449 lwpchan_lock(&mlwpchan, LWPCHAN_MPPOOL);
2450 mlocked = 1;
2451
2452 /*
2453 * Fetch the current rwlock state.
2454 *
2455 * The possibility of spurious wake-ups or killed waiters means
2456 * rwstate's URW_HAS_WAITERS bit may indicate false positives.
2457 * We only fix these if they are important to us.
2458 *
2459 * Although various error states can be observed here (e.g. the lock
2460 * is not held, but there are waiters) we assume these are applicaton
2461 * errors and so we take no corrective action.
2462 */
2463 fuword32_noerr(&rw->rwlock_readers, &rwstate);
2464 /*
2465 * We cannot legitimately get here from user-level
2466 * without URW_HAS_WAITERS being set.
2467 * Set it now to guard against user-level error.
2468 */
2469 rwstate |= URW_HAS_WAITERS;
2470
2471 /*
2472 * We can try only if the lock isn't held by a writer.
2473 */
2474 if (!(rwstate & URW_WRITE_LOCKED)) {
2475 tp = lwp_queue_waiter(&lwpchan);
2476 if (tp == NULL) {
2477 /*
2478 * Hmmm, rwstate indicates waiters but there are
2479 * none queued. This could just be the result of a
2480 * spurious wakeup, so let's ignore it.
2481 *
2482 * We now have a chance to acquire the lock
2483 * uncontended, but this is the last chance for
2484 * a writer to acquire the lock without blocking.
2485 */
2486 if (rd_wr == READ_LOCK) {
2487 rwstate++;
2488 acquired = 1;
2489 } else if ((rwstate & URW_READERS_MASK) == 0) {
2490 rwstate |= URW_WRITE_LOCKED;
2491 acquired = 1;
2492 }
2493 } else if (rd_wr == READ_LOCK) {
2494 /*
2495 * This is the last chance for a reader to acquire
2496 * the lock now, but it can only do so if there is
2497 * no writer of equal or greater priority at the
2498 * head of the queue .
2499 *
2500 * It is also just possible that there is a reader
2501 * at the head of the queue. This may be the result
2502 * of a spurious wakeup or an application failure.
2503 * In this case we only acquire the lock if we have
2504 * equal or greater priority. It is not our job to
2505 * release spurious waiters.
2506 */
2507 pri_t our_pri = DISP_PRIO(t);
2508 pri_t his_pri = DISP_PRIO(tp);
2509
2510 if ((our_pri > his_pri) || ((our_pri == his_pri) &&
2511 !(tp->t_writer & TRW_WANT_WRITE))) {
2512 rwstate++;
2513 acquired = 1;
2514 }
2515 }
2516 }
2517
2518 if (acquired || try_flag || time_error) {
2519 /*
2520 * We're not going to block this time.
2521 */
2522 suword32_noerr(&rw->rwlock_readers, rwstate);
2523 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2524 locked = 0;
2525
2526 if (acquired) {
2527 /*
2528 * Got the lock!
2529 */
2530 error = 0;
2531
2532 } else if (try_flag) {
2533 /*
2534 * We didn't get the lock and we're about to block.
2535 * If we're doing a trylock, return EBUSY instead.
2536 */
2537 error = EBUSY;
2538
2539 } else if (time_error) {
2540 /*
2541 * The SUSV3 POSIX spec is very clear that we should
2542 * get no error from validating the timer (above)
2543 * until we would actually sleep.
2544 */
2545 error = time_error;
2546 }
2547
2548 goto out_drop;
2549 }
2550
2551 /*
2552 * We're about to block, so indicate what kind of waiter we are.
2553 */
2554 t->t_writer = 0;
2555 if (rd_wr == WRITE_LOCK)
2556 t->t_writer = TRW_WANT_WRITE;
2557 suword32_noerr(&rw->rwlock_readers, rwstate);
2558
2559 /*
2560 * Unlock the rwlock's mutex (pagefaults are possible here).
2561 */
2562 set_owner_pid(mp, 0, 0);
2563 ulock_clear(&mp->mutex_lockw);
2564 fuword8_noerr(&mp->mutex_waiters, &mwaiters);
2565 if (mwaiters != 0) {
2566 /*
2567 * Given the locking of mlwpchan around the release of
2568 * the mutex and checking for waiters, the following
2569 * call to lwp_release() can fail ONLY if the lock
2570 * acquirer is interrupted after setting the waiter bit,
2571 * calling lwp_block() and releasing mlwpchan.
2572 * In this case, it could get pulled off the LWP sleep
2573 * queue (via setrun()) before the following call to
2574 * lwp_release() occurs, and the lock requestor will
2575 * update the waiter bit correctly by re-evaluating it.
2576 */
2577 if (lwp_release(&mlwpchan, &mwaiters, 0))
2578 suword8_noerr(&mp->mutex_waiters, mwaiters);
2579 }
2580 lwpchan_unlock(&mlwpchan, LWPCHAN_MPPOOL);
2581 mlocked = 0;
2582 no_fault();
2583
2584 if (mwatched) {
2585 watch_enable_addr((caddr_t)mp, sizeof (*mp), S_WRITE);
2586 mwatched = 0;
2587 }
2588 if (watched) {
2589 watch_enable_addr((caddr_t)rw, sizeof (*rw), S_WRITE);
2590 watched = 0;
2591 }
2592
2593 if (timedwait) {
2594 /*
2595 * If we successfully queue the timeout,
2596 * then don't drop t_delay_lock until
2597 * we are on the sleep queue (below).
2598 */
2599 mutex_enter(&t->t_delay_lock);
2600 if (lwp_timer_enqueue(&lwpt) != 0) {
2601 mutex_exit(&t->t_delay_lock);
2602 imm_timeout = 1;
2603 timedwait = NULL;
2604 }
2605 }
2606 t->t_flag |= T_WAITCVSEM;
2607 lwp_block(&lwpchan);
2608
2609 /*
2610 * Nothing should happen to cause the LWp to go to sleep until after
2611 * it returns from swtch().
2612 */
2613 if (timedwait)
2614 mutex_exit(&t->t_delay_lock);
2615 locked = 0;
2616 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2617 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) || imm_timeout)
2618 setrun(t);
2619 swtch();
2620
2621 /*
2622 * We're back, but we need to work out why. Were we interrupted? Did
2623 * we timeout? Were we granted the lock?
2624 */
2625 error = EAGAIN;
2626 acquired = (t->t_writer & TRW_LOCK_GRANTED);
2627 t->t_writer = 0;
2628 t->t_flag &= ~(T_WAITCVSEM | T_WAKEABLE);
2629 if (timedwait)
2630 tim = lwp_timer_dequeue(&lwpt);
2631 if (ISSIG(t, FORREAL) || lwp->lwp_sysabort || MUSTRETURN(p, t))
2632 error = EINTR;
2633 else if (imm_timeout || (timedwait && tim == -1))
2634 error = ETIME;
2635 lwp->lwp_asleep = 0;
2636 lwp->lwp_sysabort = 0;
2637 setallwatch();
2638
2639 /*
2640 * If we were granted the lock we don't care about EINTR or ETIME.
2641 */
2642 if (acquired)
2643 error = 0;
2644
2645 if (t->t_mstate == LMS_USER_LOCK)
2646 (void) new_mstate(t, LMS_SYSTEM);
2647
2648 if (error)
2649 return (set_errno(error));
2650 return (0);
2651
2652 out_drop:
2653 /*
2654 * Make sure that the user level lock is dropped before returning
2655 * to the caller.
2656 */
2657 if (!mlocked) {
2658 lwpchan_lock(&mlwpchan, LWPCHAN_MPPOOL);
2659 mlocked = 1;
2660 }
2661 set_owner_pid(mp, 0, 0);
2662 ulock_clear(&mp->mutex_lockw);
2663 fuword8_noerr(&mp->mutex_waiters, &mwaiters);
2664 if (mwaiters != 0) {
2665 /*
2666 * See comment above on lock clearing and lwp_release()
2667 * success/failure.
2668 */
2669 if (lwp_release(&mlwpchan, &mwaiters, 0))
2670 suword8_noerr(&mp->mutex_waiters, mwaiters);
2671 }
2672 lwpchan_unlock(&mlwpchan, LWPCHAN_MPPOOL);
2673 mlocked = 0;
2674
2675 out_nodrop:
2676 no_fault();
2677 if (mwatched)
2678 watch_enable_addr((caddr_t)mp, sizeof (*mp), S_WRITE);
2679 if (watched)
2680 watch_enable_addr((caddr_t)rw, sizeof (*rw), S_WRITE);
2681 if (t->t_mstate == LMS_USER_LOCK)
2682 (void) new_mstate(t, LMS_SYSTEM);
2683 if (error)
2684 return (set_errno(error));
2685 return (0);
2686 }
2687
2688 /*
2689 * We enter here holding the user-level mutex but, unlike lwp_rwlock_lock(),
2690 * we never drop the lock.
2691 */
2692 static int
lwp_rwlock_unlock(lwp_rwlock_t * rw)2693 lwp_rwlock_unlock(lwp_rwlock_t *rw)
2694 {
2695 kthread_t *t = curthread;
2696 proc_t *p = ttoproc(t);
2697 lwpchan_t lwpchan;
2698 volatile uint16_t type = 0;
2699 volatile int error = 0;
2700 volatile int locked = 0;
2701 volatile int watched = 0;
2702 label_t ljb;
2703 volatile int no_lwpchan = 1;
2704 uint32_t rwstate;
2705
2706 /* We only check rw because the mutex is included in it. */
2707 if ((caddr_t)rw >= p->p_as->a_userlimit)
2708 return (set_errno(EFAULT));
2709
2710 if (on_fault(&ljb)) {
2711 if (no_lwpchan) {
2712 error = EFAULT;
2713 goto out_nodrop;
2714 }
2715 if (locked) {
2716 locked = 0;
2717 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2718 }
2719 error = EFAULT;
2720 goto out_nodrop;
2721 }
2722
2723 /*
2724 * Force Copy-on-write if necessary and ensure that the
2725 * synchronization object resides in read/write memory.
2726 * Cause an EFAULT return now if this is not so.
2727 */
2728 fuword16_noerr(&rw->rwlock_type, (uint16_t *)&type);
2729 suword16_noerr(&rw->rwlock_type, type);
2730
2731 /* We can only continue for simple USYNC_PROCESS locks. */
2732 if (type != USYNC_PROCESS) {
2733 error = EINVAL;
2734 goto out_nodrop;
2735 }
2736
2737 /* Convert user level rwlock, "rw", to a unique lwpchan. */
2738 if (!get_lwpchan(p->p_as, (caddr_t)rw, type,
2739 &lwpchan, LWPCHAN_CVPOOL)) {
2740 error = EFAULT;
2741 goto out_nodrop;
2742 }
2743
2744 no_lwpchan = 0;
2745 watched = watch_disable_addr((caddr_t)rw, sizeof (*rw), S_WRITE);
2746
2747 lwpchan_lock(&lwpchan, LWPCHAN_CVPOOL);
2748 locked = 1;
2749
2750 /*
2751 * We can resolve multiple readers (except the last reader) here.
2752 * For the last reader or a writer we need lwp_rwlock_release(),
2753 * to which we also delegate the task of copying the new rwstate
2754 * back to userland (see the comment there).
2755 */
2756 fuword32_noerr(&rw->rwlock_readers, &rwstate);
2757 if (rwstate & URW_WRITE_LOCKED)
2758 lwp_rwlock_release(&lwpchan, rw);
2759 else if ((rwstate & URW_READERS_MASK) > 0) {
2760 rwstate--;
2761 if ((rwstate & URW_READERS_MASK) == 0)
2762 lwp_rwlock_release(&lwpchan, rw);
2763 else
2764 suword32_noerr(&rw->rwlock_readers, rwstate);
2765 }
2766
2767 lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
2768 locked = 0;
2769 error = 0;
2770
2771 out_nodrop:
2772 no_fault();
2773 if (watched)
2774 watch_enable_addr((caddr_t)rw, sizeof (*rw), S_WRITE);
2775 if (error)
2776 return (set_errno(error));
2777 return (0);
2778 }
2779
2780 int
lwp_rwlock_sys(int subcode,lwp_rwlock_t * rwlp,timespec_t * tsp)2781 lwp_rwlock_sys(int subcode, lwp_rwlock_t *rwlp, timespec_t *tsp)
2782 {
2783 switch (subcode) {
2784 case 0:
2785 return (lwp_rwlock_lock(rwlp, tsp, READ_LOCK));
2786 case 1:
2787 return (lwp_rwlock_lock(rwlp, tsp, WRITE_LOCK));
2788 case 2:
2789 return (lwp_rwlock_lock(rwlp, NULL, READ_LOCK_TRY));
2790 case 3:
2791 return (lwp_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY));
2792 case 4:
2793 return (lwp_rwlock_unlock(rwlp));
2794 }
2795 return (set_errno(EINVAL));
2796 }
2797
2798 /*
2799 * Return the owner of the user-level s-object.
2800 * Since we can't really do this, return NULL.
2801 */
2802 /* ARGSUSED */
2803 static kthread_t *
lwpsobj_owner(caddr_t sobj)2804 lwpsobj_owner(caddr_t sobj)
2805 {
2806 return ((kthread_t *)NULL);
2807 }
2808
2809 /*
2810 * Wake up a thread asleep on a user-level synchronization
2811 * object.
2812 */
2813 static void
lwp_unsleep(kthread_t * t)2814 lwp_unsleep(kthread_t *t)
2815 {
2816 ASSERT(THREAD_LOCK_HELD(t));
2817 if (t->t_wchan0 != NULL) {
2818 sleepq_head_t *sqh;
2819 sleepq_t *sqp = t->t_sleepq;
2820
2821 if (sqp != NULL) {
2822 sqh = lwpsqhash(&t->t_lwpchan);
2823 ASSERT(&sqh->sq_queue == sqp);
2824 sleepq_unsleep(t);
2825 disp_lock_exit_high(&sqh->sq_lock);
2826 CL_SETRUN(t);
2827 return;
2828 }
2829 }
2830 panic("lwp_unsleep: thread %p not on sleepq", (void *)t);
2831 }
2832
2833 /*
2834 * Change the priority of a thread asleep on a user-level
2835 * synchronization object. To maintain proper priority order,
2836 * we:
2837 * o dequeue the thread.
2838 * o change its priority.
2839 * o re-enqueue the thread.
2840 * Assumption: the thread is locked on entry.
2841 */
2842 static void
lwp_change_pri(kthread_t * t,pri_t pri,pri_t * t_prip)2843 lwp_change_pri(kthread_t *t, pri_t pri, pri_t *t_prip)
2844 {
2845 ASSERT(THREAD_LOCK_HELD(t));
2846 if (t->t_wchan0 != NULL) {
2847 sleepq_t *sqp = t->t_sleepq;
2848
2849 sleepq_dequeue(t);
2850 *t_prip = pri;
2851 sleepq_insert(sqp, t);
2852 } else
2853 panic("lwp_change_pri: %p not on a sleep queue", (void *)t);
2854 }
2855
2856 /*
2857 * Clean up a left-over process-shared robust mutex
2858 */
2859 static void
lwp_mutex_cleanup(lwpchan_entry_t * ent,uint16_t lockflg)2860 lwp_mutex_cleanup(lwpchan_entry_t *ent, uint16_t lockflg)
2861 {
2862 uint16_t flag;
2863 uchar_t waiters;
2864 label_t ljb;
2865 pid_t owner_pid;
2866 lwp_mutex_t *lp;
2867 volatile int locked = 0;
2868 volatile int watched = 0;
2869 volatile struct upimutex *upimutex = NULL;
2870 volatile int upilocked = 0;
2871
2872 if ((ent->lwpchan_type & (USYNC_PROCESS | LOCK_ROBUST))
2873 != (USYNC_PROCESS | LOCK_ROBUST))
2874 return;
2875
2876 lp = (lwp_mutex_t *)ent->lwpchan_addr;
2877 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
2878 if (on_fault(&ljb)) {
2879 if (locked)
2880 lwpchan_unlock(&ent->lwpchan_lwpchan, LWPCHAN_MPPOOL);
2881 if (upilocked)
2882 upimutex_unlock((upimutex_t *)upimutex, 0);
2883 goto out;
2884 }
2885
2886 fuword32_noerr(&lp->mutex_ownerpid, (uint32_t *)&owner_pid);
2887
2888 if (UPIMUTEX(ent->lwpchan_type)) {
2889 lwpchan_t lwpchan = ent->lwpchan_lwpchan;
2890 upib_t *upibp = &UPI_CHAIN(lwpchan);
2891
2892 if (owner_pid != curproc->p_pid)
2893 goto out;
2894 mutex_enter(&upibp->upib_lock);
2895 upimutex = upi_get(upibp, &lwpchan);
2896 if (upimutex == NULL || upimutex->upi_owner != curthread) {
2897 mutex_exit(&upibp->upib_lock);
2898 goto out;
2899 }
2900 mutex_exit(&upibp->upib_lock);
2901 upilocked = 1;
2902 flag = lwp_clear_mutex(lp, lockflg);
2903 suword8_noerr(&lp->mutex_lockw, 0);
2904 upimutex_unlock((upimutex_t *)upimutex, flag);
2905 } else {
2906 lwpchan_lock(&ent->lwpchan_lwpchan, LWPCHAN_MPPOOL);
2907 locked = 1;
2908 /*
2909 * Clear the spinners count because one of our
2910 * threads could have been spinning for this lock
2911 * at user level when the process was suddenly killed.
2912 * There is no harm in this since user-level libc code
2913 * will adapt to the sudden change in the spinner count.
2914 */
2915 suword8_noerr(&lp->mutex_spinners, 0);
2916 if (owner_pid != curproc->p_pid) {
2917 /*
2918 * We are not the owner. There may or may not be one.
2919 * If there are waiters, we wake up one or all of them.
2920 * It doesn't hurt to wake them up in error since
2921 * they will just retry the lock and go to sleep
2922 * again if necessary.
2923 */
2924 fuword8_noerr(&lp->mutex_waiters, &waiters);
2925 if (waiters != 0) { /* there are waiters */
2926 fuword16_noerr(&lp->mutex_flag, &flag);
2927 if (flag & LOCK_NOTRECOVERABLE) {
2928 lwp_release_all(&ent->lwpchan_lwpchan);
2929 suword8_noerr(&lp->mutex_waiters, 0);
2930 } else if (lwp_release(&ent->lwpchan_lwpchan,
2931 &waiters, 0)) {
2932 suword8_noerr(&lp->mutex_waiters,
2933 waiters);
2934 }
2935 }
2936 } else {
2937 /*
2938 * We are the owner. Release it.
2939 */
2940 (void) lwp_clear_mutex(lp, lockflg);
2941 ulock_clear(&lp->mutex_lockw);
2942 fuword8_noerr(&lp->mutex_waiters, &waiters);
2943 if (waiters &&
2944 lwp_release(&ent->lwpchan_lwpchan, &waiters, 0))
2945 suword8_noerr(&lp->mutex_waiters, waiters);
2946 }
2947 lwpchan_unlock(&ent->lwpchan_lwpchan, LWPCHAN_MPPOOL);
2948 }
2949 out:
2950 no_fault();
2951 if (watched)
2952 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
2953 }
2954
2955 /*
2956 * Register a process-shared robust mutex in the lwpchan cache.
2957 */
2958 int
lwp_mutex_register(lwp_mutex_t * lp,caddr_t uaddr)2959 lwp_mutex_register(lwp_mutex_t *lp, caddr_t uaddr)
2960 {
2961 int error = 0;
2962 volatile int watched;
2963 label_t ljb;
2964 uint8_t type;
2965 lwpchan_t lwpchan;
2966
2967 if ((caddr_t)lp >= (caddr_t)USERLIMIT)
2968 return (set_errno(EFAULT));
2969
2970 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
2971
2972 if (on_fault(&ljb)) {
2973 error = EFAULT;
2974 } else {
2975 /*
2976 * Force Copy-on-write if necessary and ensure that the
2977 * synchronization object resides in read/write memory.
2978 * Cause an EFAULT return now if this is not so.
2979 */
2980 fuword8_noerr(&lp->mutex_type, &type);
2981 suword8_noerr(&lp->mutex_type, type);
2982 if ((type & (USYNC_PROCESS|LOCK_ROBUST))
2983 != (USYNC_PROCESS|LOCK_ROBUST)) {
2984 error = EINVAL;
2985 } else if (!lwpchan_get_mapping(curproc->p_as, (caddr_t)lp,
2986 uaddr, type, &lwpchan, LWPCHAN_MPPOOL)) {
2987 error = EFAULT;
2988 }
2989 }
2990 no_fault();
2991 if (watched)
2992 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
2993 if (error)
2994 return (set_errno(error));
2995 return (0);
2996 }
2997
2998 /*
2999 * There is a user-level robust lock registration in libc.
3000 * Mark it as invalid by storing -1 into the location of the pointer.
3001 */
3002 static void
lwp_mutex_unregister(void * uaddr)3003 lwp_mutex_unregister(void *uaddr)
3004 {
3005 if (get_udatamodel() == DATAMODEL_NATIVE) {
3006 (void) sulword(uaddr, (ulong_t)-1);
3007 #ifdef _SYSCALL32_IMPL
3008 } else {
3009 (void) suword32(uaddr, (uint32_t)-1);
3010 #endif
3011 }
3012 }
3013
3014 int
lwp_mutex_trylock(lwp_mutex_t * lp,uintptr_t owner)3015 lwp_mutex_trylock(lwp_mutex_t *lp, uintptr_t owner)
3016 {
3017 kthread_t *t = curthread;
3018 proc_t *p = ttoproc(t);
3019 int error = 0;
3020 volatile int locked = 0;
3021 volatile int watched = 0;
3022 label_t ljb;
3023 volatile uint8_t type = 0;
3024 uint16_t flag;
3025 lwpchan_t lwpchan;
3026
3027 if ((caddr_t)lp >= p->p_as->a_userlimit)
3028 return (set_errno(EFAULT));
3029
3030 (void) new_mstate(t, LMS_USER_LOCK);
3031
3032 if (on_fault(&ljb)) {
3033 if (locked)
3034 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
3035 error = EFAULT;
3036 goto out;
3037 }
3038 /*
3039 * Force Copy-on-write if necessary and ensure that the
3040 * synchronization object resides in read/write memory.
3041 * Cause an EFAULT return now if this is not so.
3042 */
3043 fuword8_noerr(&lp->mutex_type, (uint8_t *)&type);
3044 suword8_noerr(&lp->mutex_type, type);
3045 if (UPIMUTEX(type)) {
3046 no_fault();
3047 error = lwp_upimutex_lock(lp, type, UPIMUTEX_TRY, NULL);
3048 if (error == 0 || error == EOWNERDEAD || error == ELOCKUNMAPPED)
3049 set_owner_pid(lp, owner,
3050 (type & USYNC_PROCESS)? p->p_pid : 0);
3051 if (error)
3052 return (set_errno(error));
3053 return (0);
3054 }
3055 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
3056 &lwpchan, LWPCHAN_MPPOOL)) {
3057 error = EFAULT;
3058 goto out;
3059 }
3060 lwpchan_lock(&lwpchan, LWPCHAN_MPPOOL);
3061 locked = 1;
3062 if (type & LOCK_ROBUST) {
3063 fuword16_noerr(&lp->mutex_flag, &flag);
3064 if (flag & LOCK_NOTRECOVERABLE) {
3065 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
3066 error = ENOTRECOVERABLE;
3067 goto out;
3068 }
3069 }
3070
3071 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
3072
3073 if (!ulock_try(&lp->mutex_lockw))
3074 error = EBUSY;
3075 else {
3076 set_owner_pid(lp, owner, (type & USYNC_PROCESS)? p->p_pid : 0);
3077 if (type & LOCK_ROBUST) {
3078 fuword16_noerr(&lp->mutex_flag, &flag);
3079 if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
3080 if (flag & LOCK_OWNERDEAD)
3081 error = EOWNERDEAD;
3082 else if (type & USYNC_PROCESS_ROBUST)
3083 error = ELOCKUNMAPPED;
3084 else
3085 error = EOWNERDEAD;
3086 }
3087 }
3088 }
3089 locked = 0;
3090 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
3091 out:
3092
3093 if (t->t_mstate == LMS_USER_LOCK)
3094 (void) new_mstate(t, LMS_SYSTEM);
3095
3096 no_fault();
3097 if (watched)
3098 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
3099 if (error)
3100 return (set_errno(error));
3101 return (0);
3102 }
3103
3104 /*
3105 * unlock the mutex and unblock lwps that is trying to acquire this mutex.
3106 * the blocked lwp resumes and retries to acquire the lock.
3107 */
3108 int
lwp_mutex_unlock(lwp_mutex_t * lp)3109 lwp_mutex_unlock(lwp_mutex_t *lp)
3110 {
3111 proc_t *p = ttoproc(curthread);
3112 lwpchan_t lwpchan;
3113 uchar_t waiters;
3114 volatile int locked = 0;
3115 volatile int watched = 0;
3116 volatile uint8_t type = 0;
3117 label_t ljb;
3118 uint16_t flag;
3119 int error = 0;
3120
3121 if ((caddr_t)lp >= p->p_as->a_userlimit)
3122 return (set_errno(EFAULT));
3123
3124 if (on_fault(&ljb)) {
3125 if (locked)
3126 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
3127 error = EFAULT;
3128 goto out;
3129 }
3130
3131 /*
3132 * Force Copy-on-write if necessary and ensure that the
3133 * synchronization object resides in read/write memory.
3134 * Cause an EFAULT return now if this is not so.
3135 */
3136 fuword8_noerr(&lp->mutex_type, (uint8_t *)&type);
3137 suword8_noerr(&lp->mutex_type, type);
3138
3139 if (UPIMUTEX(type)) {
3140 no_fault();
3141 error = lwp_upimutex_unlock(lp, type);
3142 if (error)
3143 return (set_errno(error));
3144 return (0);
3145 }
3146
3147 watched = watch_disable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
3148
3149 if (!get_lwpchan(curproc->p_as, (caddr_t)lp, type,
3150 &lwpchan, LWPCHAN_MPPOOL)) {
3151 error = EFAULT;
3152 goto out;
3153 }
3154 lwpchan_lock(&lwpchan, LWPCHAN_MPPOOL);
3155 locked = 1;
3156 if (type & LOCK_ROBUST) {
3157 fuword16_noerr(&lp->mutex_flag, &flag);
3158 if (flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
3159 flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
3160 flag |= LOCK_NOTRECOVERABLE;
3161 suword16_noerr(&lp->mutex_flag, flag);
3162 }
3163 }
3164 set_owner_pid(lp, 0, 0);
3165 ulock_clear(&lp->mutex_lockw);
3166 /*
3167 * Always wake up an lwp (if any) waiting on lwpchan. The woken lwp will
3168 * re-try the lock in lwp_mutex_timedlock(). The call to lwp_release()
3169 * may fail. If it fails, do not write into the waiter bit.
3170 * The call to lwp_release() might fail due to one of three reasons:
3171 *
3172 * 1. due to the thread which set the waiter bit not actually
3173 * sleeping since it got the lock on the re-try. The waiter
3174 * bit will then be correctly updated by that thread. This
3175 * window may be closed by reading the wait bit again here
3176 * and not calling lwp_release() at all if it is zero.
3177 * 2. the thread which set the waiter bit and went to sleep
3178 * was woken up by a signal. This time, the waiter recomputes
3179 * the wait bit in the return with EINTR code.
3180 * 3. the waiter bit read by lwp_mutex_wakeup() was in
3181 * memory that has been re-used after the lock was dropped.
3182 * In this case, writing into the waiter bit would cause data
3183 * corruption.
3184 */
3185 fuword8_noerr(&lp->mutex_waiters, &waiters);
3186 if (waiters) {
3187 if ((type & LOCK_ROBUST) &&
3188 (flag & LOCK_NOTRECOVERABLE)) {
3189 lwp_release_all(&lwpchan);
3190 suword8_noerr(&lp->mutex_waiters, 0);
3191 } else if (lwp_release(&lwpchan, &waiters, 0)) {
3192 suword8_noerr(&lp->mutex_waiters, waiters);
3193 }
3194 }
3195
3196 lwpchan_unlock(&lwpchan, LWPCHAN_MPPOOL);
3197 out:
3198 no_fault();
3199 if (watched)
3200 watch_enable_addr((caddr_t)lp, sizeof (*lp), S_WRITE);
3201 if (error)
3202 return (set_errno(error));
3203 return (0);
3204 }
3205