1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2012 by Delphix. All rights reserved.
29 * Copyright 2019 Joyent, Inc.
30 */
31
32 #include <sys/thread.h>
33 #include <sys/proc.h>
34 #include <sys/debug.h>
35 #include <sys/cmn_err.h>
36 #include <sys/systm.h>
37 #include <sys/sobject.h>
38 #include <sys/sleepq.h>
39 #include <sys/cpuvar.h>
40 #include <sys/condvar.h>
41 #include <sys/condvar_impl.h>
42 #include <sys/schedctl.h>
43 #include <sys/procfs.h>
44 #include <sys/sdt.h>
45 #include <sys/callo.h>
46
47 /*
48 * CV_MAX_WAITERS is the maximum number of waiters we track; once
49 * the number becomes higher than that, we look at the sleepq to
50 * see whether there are *really* any waiters.
51 */
52 #define CV_MAX_WAITERS 1024 /* must be power of 2 */
53 #define CV_WAITERS_MASK (CV_MAX_WAITERS - 1)
54
55 /*
56 * Threads don't "own" condition variables.
57 */
58 /* ARGSUSED */
59 static kthread_t *
cv_owner(void * cvp)60 cv_owner(void *cvp)
61 {
62 return (NULL);
63 }
64
65 /*
66 * Unsleep a thread that's blocked on a condition variable.
67 */
68 static void
cv_unsleep(kthread_t * t)69 cv_unsleep(kthread_t *t)
70 {
71 condvar_impl_t *cvp = (condvar_impl_t *)t->t_wchan;
72 sleepq_head_t *sqh = SQHASH(cvp);
73
74 ASSERT(THREAD_LOCK_HELD(t));
75
76 if (cvp == NULL)
77 panic("cv_unsleep: thread %p not on sleepq %p",
78 (void *)t, (void *)sqh);
79 DTRACE_SCHED1(wakeup, kthread_t *, t);
80 sleepq_unsleep(t);
81 if (cvp->cv_waiters != CV_MAX_WAITERS)
82 cvp->cv_waiters--;
83 disp_lock_exit_high(&sqh->sq_lock);
84 CL_SETRUN(t);
85 }
86
87 /*
88 * Change the priority of a thread that's blocked on a condition variable.
89 */
90 static void
cv_change_pri(kthread_t * t,pri_t pri,pri_t * t_prip)91 cv_change_pri(kthread_t *t, pri_t pri, pri_t *t_prip)
92 {
93 condvar_impl_t *cvp = (condvar_impl_t *)t->t_wchan;
94 sleepq_t *sqp = t->t_sleepq;
95
96 ASSERT(THREAD_LOCK_HELD(t));
97 ASSERT(&SQHASH(cvp)->sq_queue == sqp);
98
99 if (cvp == NULL)
100 panic("cv_change_pri: %p not on sleep queue", (void *)t);
101 sleepq_dequeue(t);
102 *t_prip = pri;
103 sleepq_insert(sqp, t);
104 }
105
106 /*
107 * The sobj_ops vector exports a set of functions needed when a thread
108 * is asleep on a synchronization object of this type.
109 */
110 static sobj_ops_t cv_sobj_ops = {
111 SOBJ_CV, cv_owner, cv_unsleep, cv_change_pri
112 };
113
114 /* ARGSUSED */
115 void
cv_init(kcondvar_t * cvp,char * name,kcv_type_t type,void * arg)116 cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg)
117 {
118 ((condvar_impl_t *)cvp)->cv_waiters = 0;
119 }
120
121 /*
122 * cv_destroy is not currently needed, but is part of the DDI.
123 * This is in case cv_init ever needs to allocate something for a cv.
124 */
125 /* ARGSUSED */
126 void
cv_destroy(kcondvar_t * cvp)127 cv_destroy(kcondvar_t *cvp)
128 {
129 ASSERT((((condvar_impl_t *)cvp)->cv_waiters & CV_WAITERS_MASK) == 0);
130 }
131
132 /*
133 * The cv_block() function blocks a thread on a condition variable
134 * by putting it in a hashed sleep queue associated with the
135 * synchronization object.
136 *
137 * Threads are taken off the hashed sleep queues via calls to
138 * cv_signal(), cv_broadcast(), or cv_unsleep().
139 */
140 static void
cv_block(condvar_impl_t * cvp)141 cv_block(condvar_impl_t *cvp)
142 {
143 kthread_t *t = curthread;
144 klwp_t *lwp = ttolwp(t);
145 sleepq_head_t *sqh;
146
147 ASSERT(THREAD_LOCK_HELD(t));
148 ASSERT(t != CPU->cpu_idle_thread);
149 ASSERT(CPU_ON_INTR(CPU) == 0);
150 ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
151 ASSERT(t->t_state == TS_ONPROC);
152
153 t->t_schedflag &= ~TS_SIGNALLED;
154 CL_SLEEP(t); /* assign kernel priority */
155 t->t_wchan = (caddr_t)cvp;
156 t->t_sobj_ops = &cv_sobj_ops;
157 DTRACE_SCHED(sleep);
158
159 /*
160 * The check for t_intr is to avoid doing the
161 * account for an interrupt thread on the still-pinned
162 * lwp's statistics.
163 */
164 if (lwp != NULL && t->t_intr == NULL) {
165 lwp->lwp_ru.nvcsw++;
166 (void) new_mstate(t, LMS_SLEEP);
167 }
168
169 sqh = SQHASH(cvp);
170 disp_lock_enter_high(&sqh->sq_lock);
171 if (cvp->cv_waiters < CV_MAX_WAITERS)
172 cvp->cv_waiters++;
173 ASSERT(cvp->cv_waiters <= CV_MAX_WAITERS);
174 THREAD_SLEEP(t, &sqh->sq_lock);
175 sleepq_insert(&sqh->sq_queue, t);
176 /*
177 * THREAD_SLEEP() moves curthread->t_lockp to point to the
178 * lock sqh->sq_lock. This lock is later released by the caller
179 * when it calls thread_unlock() on curthread.
180 */
181 }
182
183 #define cv_block_sig(t, cvp) \
184 { (t)->t_flag |= T_WAKEABLE; cv_block(cvp); }
185
186 /*
187 * Block on the indicated condition variable and release the
188 * associated kmutex while blocked.
189 */
190 void
cv_wait(kcondvar_t * cvp,kmutex_t * mp)191 cv_wait(kcondvar_t *cvp, kmutex_t *mp)
192 {
193 if (panicstr)
194 return;
195 ASSERT(!quiesce_active);
196
197 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
198 thread_lock(curthread); /* lock the thread */
199 cv_block((condvar_impl_t *)cvp);
200 thread_unlock_nopreempt(curthread); /* unlock the waiters field */
201 mutex_exit(mp);
202 swtch();
203 mutex_enter(mp);
204 }
205
206 static void
cv_wakeup(void * arg)207 cv_wakeup(void *arg)
208 {
209 kthread_t *t = arg;
210
211 /*
212 * This mutex is acquired and released in order to make sure that
213 * the wakeup does not happen before the block itself happens.
214 */
215 mutex_enter(&t->t_wait_mutex);
216 mutex_exit(&t->t_wait_mutex);
217 setrun(t);
218 }
219
220 /*
221 * Same as cv_wait except the thread will unblock at 'tim'
222 * (an absolute time) if it hasn't already unblocked.
223 *
224 * Returns the amount of time left from the original 'tim' value
225 * when it was unblocked.
226 */
227 clock_t
cv_timedwait(kcondvar_t * cvp,kmutex_t * mp,clock_t tim)228 cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim)
229 {
230 hrtime_t hrtim;
231 clock_t now = ddi_get_lbolt();
232
233 if (tim <= now)
234 return (-1);
235
236 hrtim = TICK_TO_NSEC(tim - now);
237 return (cv_timedwait_hires(cvp, mp, hrtim, nsec_per_tick, 0));
238 }
239
240 /*
241 * Same as cv_timedwait() except that the third argument is a relative
242 * timeout value, as opposed to an absolute one. There is also a fourth
243 * argument that specifies how accurately the timeout must be implemented.
244 */
245 clock_t
cv_reltimedwait(kcondvar_t * cvp,kmutex_t * mp,clock_t delta,time_res_t res)246 cv_reltimedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t delta, time_res_t res)
247 {
248 hrtime_t exp;
249
250 ASSERT(TIME_RES_VALID(res));
251
252 if (delta <= 0)
253 return (-1);
254
255 if ((exp = TICK_TO_NSEC(delta)) < 0)
256 exp = CY_INFINITY;
257
258 return (cv_timedwait_hires(cvp, mp, exp, time_res[res], 0));
259 }
260
261 clock_t
cv_timedwait_hires(kcondvar_t * cvp,kmutex_t * mp,hrtime_t tim,hrtime_t res,int flag)262 cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
263 hrtime_t res, int flag)
264 {
265 kthread_t *t = curthread;
266 callout_id_t id;
267 clock_t timeleft;
268 hrtime_t limit;
269 int signalled;
270
271 if (panicstr)
272 return (-1);
273 ASSERT(!quiesce_active);
274
275 limit = (flag & CALLOUT_FLAG_ABSOLUTE) ? gethrtime() : 0;
276 if (tim <= limit)
277 return (-1);
278 mutex_enter(&t->t_wait_mutex);
279 id = timeout_generic(CALLOUT_REALTIME, (void (*)(void *))cv_wakeup, t,
280 tim, res, flag);
281 thread_lock(t); /* lock the thread */
282 cv_block((condvar_impl_t *)cvp);
283 thread_unlock_nopreempt(t);
284 mutex_exit(&t->t_wait_mutex);
285 mutex_exit(mp);
286 swtch();
287 signalled = (t->t_schedflag & TS_SIGNALLED);
288 /*
289 * Get the time left. untimeout() returns -1 if the timeout has
290 * occured or the time remaining. If the time remaining is zero,
291 * the timeout has occured between when we were awoken and
292 * we called untimeout. We will treat this as if the timeout
293 * has occured and set timeleft to -1.
294 */
295 timeleft = untimeout_default(id, 0);
296 mutex_enter(mp);
297 if (timeleft <= 0) {
298 timeleft = -1;
299 if (signalled) /* avoid consuming the cv_signal() */
300 cv_signal(cvp);
301 }
302 return (timeleft);
303 }
304
305 int
cv_wait_sig(kcondvar_t * cvp,kmutex_t * mp)306 cv_wait_sig(kcondvar_t *cvp, kmutex_t *mp)
307 {
308 kthread_t *t = curthread;
309 proc_t *p = ttoproc(t);
310 klwp_t *lwp = ttolwp(t);
311 int cancel_pending;
312 int rval = 1;
313 int signalled = 0;
314
315 if (panicstr)
316 return (rval);
317 ASSERT(!quiesce_active);
318
319 /*
320 * Threads in system processes don't process signals. This is
321 * true both for standard threads of system processes and for
322 * interrupt threads which have borrowed their pinned thread's LWP.
323 */
324 if (lwp == NULL || (p->p_flag & SSYS)) {
325 cv_wait(cvp, mp);
326 return (rval);
327 }
328 ASSERT(t->t_intr == NULL);
329
330 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
331 cancel_pending = schedctl_cancel_pending();
332 lwp->lwp_asleep = 1;
333 lwp->lwp_sysabort = 0;
334 thread_lock(t);
335 cv_block_sig(t, (condvar_impl_t *)cvp);
336 thread_unlock_nopreempt(t);
337 mutex_exit(mp);
338 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) || cancel_pending)
339 setrun(t);
340 /* ASSERT(no locks are held) */
341 swtch();
342 signalled = (t->t_schedflag & TS_SIGNALLED);
343 t->t_flag &= ~T_WAKEABLE;
344 mutex_enter(mp);
345 if (ISSIG_PENDING(t, lwp, p)) {
346 mutex_exit(mp);
347 if (issig(FORREAL))
348 rval = 0;
349 mutex_enter(mp);
350 }
351 if (lwp->lwp_sysabort || MUSTRETURN(p, t))
352 rval = 0;
353 if (rval != 0 && cancel_pending) {
354 schedctl_cancel_eintr();
355 rval = 0;
356 }
357 lwp->lwp_asleep = 0;
358 lwp->lwp_sysabort = 0;
359 if (rval == 0 && signalled) /* avoid consuming the cv_signal() */
360 cv_signal(cvp);
361 return (rval);
362 }
363
364 static clock_t
cv_timedwait_sig_hires(kcondvar_t * cvp,kmutex_t * mp,hrtime_t tim,hrtime_t res,int flag)365 cv_timedwait_sig_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
366 hrtime_t res, int flag)
367 {
368 kthread_t *t = curthread;
369 proc_t *p = ttoproc(t);
370 klwp_t *lwp = ttolwp(t);
371 int cancel_pending = 0;
372 callout_id_t id;
373 clock_t rval = 1;
374 hrtime_t limit;
375 int signalled = 0;
376
377 if (panicstr)
378 return (rval);
379 ASSERT(!quiesce_active);
380
381 /*
382 * Threads in system processes don't process signals. This is
383 * true both for standard threads of system processes and for
384 * interrupt threads which have borrowed their pinned thread's LWP.
385 */
386 if (lwp == NULL || (p->p_flag & SSYS))
387 return (cv_timedwait_hires(cvp, mp, tim, res, flag));
388 ASSERT(t->t_intr == NULL);
389
390 /*
391 * If tim is less than or equal to current hrtime, then the timeout
392 * has already occured. So just check to see if there is a signal
393 * pending. If so return 0 indicating that there is a signal pending.
394 * Else return -1 indicating that the timeout occured. No need to
395 * wait on anything.
396 */
397 limit = (flag & CALLOUT_FLAG_ABSOLUTE) ? gethrtime() : 0;
398 if (tim <= limit) {
399 lwp->lwp_asleep = 1;
400 lwp->lwp_sysabort = 0;
401 rval = -1;
402 goto out;
403 }
404
405 /*
406 * Set the timeout and wait.
407 */
408 cancel_pending = schedctl_cancel_pending();
409 mutex_enter(&t->t_wait_mutex);
410 id = timeout_generic(CALLOUT_REALTIME, (void (*)(void *))cv_wakeup, t,
411 tim, res, flag);
412 lwp->lwp_asleep = 1;
413 lwp->lwp_sysabort = 0;
414 thread_lock(t);
415 cv_block_sig(t, (condvar_impl_t *)cvp);
416 thread_unlock_nopreempt(t);
417 mutex_exit(&t->t_wait_mutex);
418 mutex_exit(mp);
419 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) || cancel_pending)
420 setrun(t);
421 /* ASSERT(no locks are held) */
422 swtch();
423 signalled = (t->t_schedflag & TS_SIGNALLED);
424 t->t_flag &= ~T_WAKEABLE;
425
426 /*
427 * Untimeout the thread. untimeout() returns -1 if the timeout has
428 * occured or the time remaining. If the time remaining is zero,
429 * the timeout has occured between when we were awoken and
430 * we called untimeout. We will treat this as if the timeout
431 * has occured and set rval to -1.
432 */
433 rval = untimeout_default(id, 0);
434 mutex_enter(mp);
435 if (rval <= 0)
436 rval = -1;
437
438 /*
439 * Check to see if a signal is pending. If so, regardless of whether
440 * or not we were awoken due to the signal, the signal is now pending
441 * and a return of 0 has the highest priority.
442 */
443 out:
444 if (ISSIG_PENDING(t, lwp, p)) {
445 mutex_exit(mp);
446 if (issig(FORREAL))
447 rval = 0;
448 mutex_enter(mp);
449 }
450 if (lwp->lwp_sysabort || MUSTRETURN(p, t))
451 rval = 0;
452 if (rval != 0 && cancel_pending) {
453 schedctl_cancel_eintr();
454 rval = 0;
455 }
456 lwp->lwp_asleep = 0;
457 lwp->lwp_sysabort = 0;
458 if (rval <= 0 && signalled) /* avoid consuming the cv_signal() */
459 cv_signal(cvp);
460 return (rval);
461 }
462
463 /*
464 * Returns:
465 * Function result in order of precedence:
466 * 0 if a signal was received
467 * -1 if timeout occured
468 * >0 if awakened via cv_signal() or cv_broadcast().
469 * (returns time remaining)
470 *
471 * cv_timedwait_sig() is now part of the DDI.
472 *
473 * This function is now just a wrapper for cv_timedwait_sig_hires().
474 */
475 clock_t
cv_timedwait_sig(kcondvar_t * cvp,kmutex_t * mp,clock_t tim)476 cv_timedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t tim)
477 {
478 hrtime_t hrtim;
479
480 hrtim = TICK_TO_NSEC(tim - ddi_get_lbolt());
481 return (cv_timedwait_sig_hires(cvp, mp, hrtim, nsec_per_tick, 0));
482 }
483
484 /*
485 * Wait until the specified time.
486 * If tim == -1, waits without timeout using cv_wait_sig_swap().
487 */
488 int
cv_timedwait_sig_hrtime(kcondvar_t * cvp,kmutex_t * mp,hrtime_t tim)489 cv_timedwait_sig_hrtime(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim)
490 {
491 if (tim == -1) {
492 return (cv_wait_sig_swap(cvp, mp));
493 } else {
494 return (cv_timedwait_sig_hires(cvp, mp, tim, 1,
495 CALLOUT_FLAG_ABSOLUTE | CALLOUT_FLAG_ROUNDUP));
496 }
497 }
498
499 /*
500 * Same as cv_timedwait_sig() except that the third argument is a relative
501 * timeout value, as opposed to an absolute one. There is also a fourth
502 * argument that specifies how accurately the timeout must be implemented.
503 */
504 clock_t
cv_reltimedwait_sig(kcondvar_t * cvp,kmutex_t * mp,clock_t delta,time_res_t res)505 cv_reltimedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t delta,
506 time_res_t res)
507 {
508 hrtime_t exp = 0;
509
510 ASSERT(TIME_RES_VALID(res));
511
512 if (delta > 0) {
513 if ((exp = TICK_TO_NSEC(delta)) < 0)
514 exp = CY_INFINITY;
515 }
516
517 return (cv_timedwait_sig_hires(cvp, mp, exp, time_res[res], 0));
518 }
519
520 /*
521 * Like cv_wait_sig_swap but allows the caller to indicate (with a
522 * non-NULL sigret) that they will take care of signalling the cv
523 * after wakeup, if necessary. This is a vile hack that should only
524 * be used when no other option is available; almost all callers
525 * should just use cv_wait_sig_swap (which takes care of the cv_signal
526 * stuff automatically) instead.
527 */
528 int
cv_wait_sig_swap_core(kcondvar_t * cvp,kmutex_t * mp,int * sigret)529 cv_wait_sig_swap_core(kcondvar_t *cvp, kmutex_t *mp, int *sigret)
530 {
531 kthread_t *t = curthread;
532 proc_t *p = ttoproc(t);
533 klwp_t *lwp = ttolwp(t);
534 int cancel_pending;
535 int rval = 1;
536 int signalled = 0;
537
538 if (panicstr)
539 return (rval);
540
541 /*
542 * Threads in system processes don't process signals. This is
543 * true both for standard threads of system processes and for
544 * interrupt threads which have borrowed their pinned thread's LWP.
545 */
546 if (lwp == NULL || (p->p_flag & SSYS)) {
547 cv_wait(cvp, mp);
548 return (rval);
549 }
550 ASSERT(t->t_intr == NULL);
551
552 cancel_pending = schedctl_cancel_pending();
553 lwp->lwp_asleep = 1;
554 lwp->lwp_sysabort = 0;
555 thread_lock(t);
556 cv_block_sig(t, (condvar_impl_t *)cvp);
557 /* I can be swapped now */
558 curthread->t_schedflag &= ~TS_DONT_SWAP;
559 thread_unlock_nopreempt(t);
560 mutex_exit(mp);
561 if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t) || cancel_pending)
562 setrun(t);
563 /* ASSERT(no locks are held) */
564 swtch();
565 signalled = (t->t_schedflag & TS_SIGNALLED);
566 t->t_flag &= ~T_WAKEABLE;
567 /* TS_DONT_SWAP set by disp() */
568 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
569 mutex_enter(mp);
570 if (ISSIG_PENDING(t, lwp, p)) {
571 mutex_exit(mp);
572 if (issig(FORREAL))
573 rval = 0;
574 mutex_enter(mp);
575 }
576 if (lwp->lwp_sysabort || MUSTRETURN(p, t))
577 rval = 0;
578 if (rval != 0 && cancel_pending) {
579 schedctl_cancel_eintr();
580 rval = 0;
581 }
582 lwp->lwp_asleep = 0;
583 lwp->lwp_sysabort = 0;
584 if (rval == 0) {
585 if (sigret != NULL)
586 *sigret = signalled; /* just tell the caller */
587 else if (signalled)
588 cv_signal(cvp); /* avoid consuming the cv_signal() */
589 }
590 return (rval);
591 }
592
593 /*
594 * Same as cv_wait_sig but the thread can be swapped out while waiting.
595 * This should only be used when we know we aren't holding any locks.
596 */
597 int
cv_wait_sig_swap(kcondvar_t * cvp,kmutex_t * mp)598 cv_wait_sig_swap(kcondvar_t *cvp, kmutex_t *mp)
599 {
600 return (cv_wait_sig_swap_core(cvp, mp, NULL));
601 }
602
603 void
cv_signal(kcondvar_t * cvp)604 cv_signal(kcondvar_t *cvp)
605 {
606 condvar_impl_t *cp = (condvar_impl_t *)cvp;
607
608 /* make sure the cv_waiters field looks sane */
609 ASSERT(cp->cv_waiters <= CV_MAX_WAITERS);
610 if (cp->cv_waiters > 0) {
611 sleepq_head_t *sqh = SQHASH(cp);
612 disp_lock_enter(&sqh->sq_lock);
613 ASSERT(CPU_ON_INTR(CPU) == 0);
614 if (cp->cv_waiters & CV_WAITERS_MASK) {
615 kthread_t *t;
616 cp->cv_waiters--;
617 t = sleepq_wakeone_chan(&sqh->sq_queue, cp);
618 /*
619 * If cv_waiters is non-zero (and less than
620 * CV_MAX_WAITERS) there should be a thread
621 * in the queue.
622 */
623 ASSERT(t != NULL);
624 } else if (sleepq_wakeone_chan(&sqh->sq_queue, cp) == NULL) {
625 cp->cv_waiters = 0;
626 }
627 disp_lock_exit(&sqh->sq_lock);
628 }
629 }
630
631 void
cv_broadcast(kcondvar_t * cvp)632 cv_broadcast(kcondvar_t *cvp)
633 {
634 condvar_impl_t *cp = (condvar_impl_t *)cvp;
635
636 /* make sure the cv_waiters field looks sane */
637 ASSERT(cp->cv_waiters <= CV_MAX_WAITERS);
638 if (cp->cv_waiters > 0) {
639 sleepq_head_t *sqh = SQHASH(cp);
640 disp_lock_enter(&sqh->sq_lock);
641 ASSERT(CPU_ON_INTR(CPU) == 0);
642 sleepq_wakeall_chan(&sqh->sq_queue, cp);
643 cp->cv_waiters = 0;
644 disp_lock_exit(&sqh->sq_lock);
645 }
646 }
647
648 /*
649 * Same as cv_wait(), but wakes up (after wakeup_time milliseconds) to check
650 * for requests to stop, like cv_wait_sig() but without dealing with signals.
651 * This is a horrible kludge. It is evil. It is vile. It is swill.
652 * If your code has to call this function then your code is the same.
653 */
654 void
cv_wait_stop(kcondvar_t * cvp,kmutex_t * mp,int wakeup_time)655 cv_wait_stop(kcondvar_t *cvp, kmutex_t *mp, int wakeup_time)
656 {
657 kthread_t *t = curthread;
658 klwp_t *lwp = ttolwp(t);
659 proc_t *p = ttoproc(t);
660 callout_id_t id;
661 clock_t tim;
662
663 if (panicstr)
664 return;
665
666 /*
667 * Threads in system processes don't process signals. This is
668 * true both for standard threads of system processes and for
669 * interrupt threads which have borrowed their pinned thread's LWP.
670 */
671 if (lwp == NULL || (p->p_flag & SSYS)) {
672 cv_wait(cvp, mp);
673 return;
674 }
675 ASSERT(t->t_intr == NULL);
676
677 /*
678 * Wakeup in wakeup_time milliseconds, i.e., human time.
679 */
680 tim = ddi_get_lbolt() + MSEC_TO_TICK(wakeup_time);
681 mutex_enter(&t->t_wait_mutex);
682 id = realtime_timeout_default((void (*)(void *))cv_wakeup, t,
683 tim - ddi_get_lbolt());
684 thread_lock(t); /* lock the thread */
685 cv_block((condvar_impl_t *)cvp);
686 thread_unlock_nopreempt(t);
687 mutex_exit(&t->t_wait_mutex);
688 mutex_exit(mp);
689 /* ASSERT(no locks are held); */
690 swtch();
691 (void) untimeout_default(id, 0);
692
693 /*
694 * Check for reasons to stop, if lwp_nostop is not true.
695 * See issig_forreal() for explanations of the various stops.
696 */
697 mutex_enter(&p->p_lock);
698 while (lwp->lwp_nostop == 0 && !(p->p_flag & SEXITLWPS)) {
699 /*
700 * Hold the lwp here for watchpoint manipulation.
701 */
702 if (t->t_proc_flag & TP_PAUSE) {
703 stop(PR_SUSPENDED, SUSPEND_PAUSE);
704 continue;
705 }
706 /*
707 * System checkpoint.
708 */
709 if (t->t_proc_flag & TP_CHKPT) {
710 stop(PR_CHECKPOINT, 0);
711 continue;
712 }
713 /*
714 * Honor fork1(), watchpoint activity (remapping a page),
715 * and lwp_suspend() requests.
716 */
717 if ((p->p_flag & (SHOLDFORK1|SHOLDWATCH)) ||
718 (t->t_proc_flag & TP_HOLDLWP)) {
719 stop(PR_SUSPENDED, SUSPEND_NORMAL);
720 continue;
721 }
722 /*
723 * Honor /proc requested stop.
724 */
725 if (t->t_proc_flag & TP_PRSTOP) {
726 stop(PR_REQUESTED, 0);
727 }
728 /*
729 * If some lwp in the process has already stopped
730 * showing PR_JOBCONTROL, stop in sympathy with it.
731 */
732 if (p->p_stopsig && t != p->p_agenttp) {
733 stop(PR_JOBCONTROL, p->p_stopsig);
734 continue;
735 }
736 break;
737 }
738 mutex_exit(&p->p_lock);
739 mutex_enter(mp);
740 }
741
742 /*
743 * Like cv_timedwait_sig(), but takes an absolute hires future time
744 * rather than a future time in clock ticks. Will not return showing
745 * that a timeout occurred until the future time is passed.
746 * If 'when' is a NULL pointer, no timeout will occur.
747 * Returns:
748 * Function result in order of precedence:
749 * 0 if a signal was received
750 * -1 if timeout occured
751 * >0 if awakened via cv_signal() or cv_broadcast()
752 * or by a spurious wakeup.
753 * (might return time remaining)
754 * As a special test, if someone abruptly resets the system time
755 * (but not through adjtime(2); drifting of the clock is allowed and
756 * expected [see timespectohz_adj()]), then we force a return of -1
757 * so the caller can return a premature timeout to the calling process
758 * so it can reevaluate the situation in light of the new system time.
759 * (The system clock has been reset if timecheck != timechanged.)
760 *
761 * Generally, cv_timedwait_sig_hrtime() should be used instead of this
762 * routine. It waits based on hrtime rather than wall-clock time and therefore
763 * does not need to deal with the time changing.
764 */
765 int
cv_waituntil_sig(kcondvar_t * cvp,kmutex_t * mp,timestruc_t * when,int timecheck)766 cv_waituntil_sig(kcondvar_t *cvp, kmutex_t *mp, timestruc_t *when,
767 int timecheck)
768 {
769 timestruc_t now;
770 timestruc_t delta;
771 hrtime_t interval;
772 int rval;
773
774 if (when == NULL)
775 return (cv_wait_sig_swap(cvp, mp));
776
777 gethrestime(&now);
778 delta = *when;
779 timespecsub(&delta, &now);
780 if (delta.tv_sec < 0 || (delta.tv_sec == 0 && delta.tv_nsec == 0)) {
781 /*
782 * We have already reached the absolute future time.
783 * Call cv_timedwait_sig() just to check for signals.
784 * We will return immediately with either 0 or -1.
785 */
786 rval = cv_timedwait_sig_hires(cvp, mp, 0, 1, 0);
787 } else {
788 if (timecheck == timechanged) {
789 /*
790 * Make sure that the interval is atleast one tick.
791 * This is to prevent a user from flooding the system
792 * with very small, high resolution timers.
793 */
794 interval = ts2hrt(&delta);
795 if (interval < nsec_per_tick)
796 interval = nsec_per_tick;
797 rval = cv_timedwait_sig_hires(cvp, mp, interval, 1,
798 CALLOUT_FLAG_HRESTIME);
799 } else {
800 /*
801 * Someone reset the system time;
802 * just force an immediate timeout.
803 */
804 rval = -1;
805 }
806 if (rval == -1 && timecheck == timechanged) {
807 /*
808 * Even though cv_timedwait_sig() returned showing a
809 * timeout, the future time may not have passed yet.
810 * If not, change rval to indicate a normal wakeup.
811 */
812 gethrestime(&now);
813 delta = *when;
814 timespecsub(&delta, &now);
815 if (delta.tv_sec > 0 || (delta.tv_sec == 0 &&
816 delta.tv_nsec > 0))
817 rval = 1;
818 }
819 }
820 return (rval);
821 }
822