1 /*- 2 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29 #include "opt_ktrace.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/lock.h> 34 #include <sys/mutex.h> 35 #include <sys/proc.h> 36 #include <sys/kernel.h> 37 #include <sys/ktr.h> 38 #include <sys/condvar.h> 39 #include <sys/signalvar.h> 40 #include <sys/resourcevar.h> 41 #ifdef KTRACE 42 #include <sys/uio.h> 43 #include <sys/ktrace.h> 44 #endif 45 46 /* 47 * Common sanity checks for cv_wait* functions. 48 */ 49 #define CV_ASSERT(cvp, mp, td) do { \ 50 KASSERT((td) != NULL, ("%s: curthread NULL", __func__)); \ 51 KASSERT((td)->td_proc->p_stat == SRUN, ("%s: not SRUN", __func__)); \ 52 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 53 KASSERT((mp) != NULL, ("%s: mp NULL", __func__)); \ 54 mtx_assert((mp), MA_OWNED | MA_NOTRECURSED); \ 55 } while (0) 56 57 #ifdef INVARIANTS 58 #define CV_WAIT_VALIDATE(cvp, mp) do { \ 59 if (TAILQ_EMPTY(&(cvp)->cv_waitq)) { \ 60 /* Only waiter. */ \ 61 (cvp)->cv_mtx = (mp); \ 62 } else { \ 63 /* \ 64 * Other waiter; assert that we're using the \ 65 * same mutex. \ 66 */ \ 67 KASSERT((cvp)->cv_mtx == (mp), \ 68 ("%s: Multiple mutexes", __func__)); \ 69 } \ 70 } while (0) 71 #define CV_SIGNAL_VALIDATE(cvp) do { \ 72 if (!TAILQ_EMPTY(&(cvp)->cv_waitq)) { \ 73 KASSERT(mtx_owned((cvp)->cv_mtx), \ 74 ("%s: Mutex not owned", __func__)); \ 75 } \ 76 } while (0) 77 #else 78 #define CV_WAIT_VALIDATE(cvp, mp) 79 #define CV_SIGNAL_VALIDATE(cvp) 80 #endif 81 82 static void cv_timedwait_end(void *arg); 83 84 /* 85 * Initialize a condition variable. Must be called before use. 86 */ 87 void 88 cv_init(struct cv *cvp, const char *desc) 89 { 90 91 TAILQ_INIT(&cvp->cv_waitq); 92 cvp->cv_mtx = NULL; 93 cvp->cv_description = desc; 94 } 95 96 /* 97 * Destroy a condition variable. The condition variable must be re-initialized 98 * in order to be re-used. 99 */ 100 void 101 cv_destroy(struct cv *cvp) 102 { 103 104 KASSERT(cv_waitq_empty(cvp), ("%s: cv_waitq non-empty", __func__)); 105 } 106 107 /* 108 * Common code for cv_wait* functions. All require sched_lock. 109 */ 110 111 /* 112 * Switch context. 113 */ 114 static __inline void 115 cv_switch(struct thread *td) 116 { 117 118 td->td_proc->p_stat = SSLEEP; 119 td->td_proc->p_stats->p_ru.ru_nvcsw++; 120 mi_switch(); 121 CTR3(KTR_PROC, "cv_switch: resume thread %p (pid %d, %s)", td, 122 td->td_proc->p_pid, td->td_proc->p_comm); 123 } 124 125 /* 126 * Switch context, catching signals. 127 */ 128 static __inline int 129 cv_switch_catch(struct thread *td) 130 { 131 struct proc *p; 132 int sig; 133 134 /* 135 * We put ourselves on the sleep queue and start our timeout before 136 * calling cursig, as we could stop there, and a wakeup or a SIGCONT (or 137 * both) could occur while we were stopped. A SIGCONT would cause us to 138 * be marked as SSLEEP without resuming us, thus we must be ready for 139 * sleep when cursig is called. If the wakeup happens while we're 140 * stopped, td->td_wchan will be 0 upon return from cursig. 141 */ 142 td->td_flags |= TDF_SINTR; 143 mtx_unlock_spin(&sched_lock); 144 p = td->td_proc; 145 PROC_LOCK(p); 146 sig = cursig(p); /* XXXKSE */ 147 mtx_lock_spin(&sched_lock); 148 PROC_UNLOCK(p); 149 if (sig != 0) { 150 if (td->td_wchan != NULL) 151 cv_waitq_remove(td); 152 td->td_proc->p_stat = SRUN; 153 } else if (td->td_wchan != NULL) { 154 cv_switch(td); 155 } 156 td->td_flags &= ~TDF_SINTR; 157 158 return sig; 159 } 160 161 /* 162 * Add a thread to the wait queue of a condition variable. 163 */ 164 static __inline void 165 cv_waitq_add(struct cv *cvp, struct thread *td) 166 { 167 168 /* 169 * Process may be sitting on a slpque if asleep() was called, remove it 170 * before re-adding. 171 */ 172 if (td->td_wchan != NULL) 173 unsleep(td); 174 175 td->td_flags |= TDF_CVWAITQ; 176 td->td_wchan = cvp; 177 td->td_wmesg = cvp->cv_description; 178 td->td_kse->ke_slptime = 0; /* XXXKSE */ 179 td->td_ksegrp->kg_slptime = 0; /* XXXKSE */ 180 td->td_base_pri = td->td_priority; 181 CTR3(KTR_PROC, "cv_waitq_add: thread %p (pid %d, %s)", td, 182 td->td_proc->p_pid, td->td_proc->p_comm); 183 TAILQ_INSERT_TAIL(&cvp->cv_waitq, td, td_slpq); 184 } 185 186 /* 187 * Wait on a condition variable. The current thread is placed on the condition 188 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 189 * condition variable will resume the thread. The mutex is released before 190 * sleeping and will be held on return. It is recommended that the mutex be 191 * held when cv_signal or cv_broadcast are called. 192 */ 193 void 194 cv_wait(struct cv *cvp, struct mtx *mp) 195 { 196 struct thread *td; 197 WITNESS_SAVE_DECL(mp); 198 199 td = curthread; 200 #ifdef KTRACE 201 if (KTRPOINT(td, KTR_CSW)) 202 ktrcsw(1, 0); 203 #endif 204 CV_ASSERT(cvp, mp, td); 205 WITNESS_SLEEP(0, &mp->mtx_object); 206 WITNESS_SAVE(&mp->mtx_object, mp); 207 208 if (cold || panicstr) { 209 /* 210 * After a panic, or during autoconfiguration, just give 211 * interrupts a chance, then just return; don't run any other 212 * thread or panic below, in case this is the idle process and 213 * already asleep. 214 */ 215 return; 216 } 217 218 mtx_lock_spin(&sched_lock); 219 220 CV_WAIT_VALIDATE(cvp, mp); 221 222 DROP_GIANT(); 223 mtx_unlock(mp); 224 225 cv_waitq_add(cvp, td); 226 cv_switch(td); 227 228 mtx_unlock_spin(&sched_lock); 229 #ifdef KTRACE 230 if (KTRPOINT(td, KTR_CSW)) 231 ktrcsw(0, 0); 232 #endif 233 PICKUP_GIANT(); 234 mtx_lock(mp); 235 WITNESS_RESTORE(&mp->mtx_object, mp); 236 } 237 238 /* 239 * Wait on a condition variable, allowing interruption by signals. Return 0 if 240 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 241 * a signal was caught. If ERESTART is returned the system call should be 242 * restarted if possible. 243 */ 244 int 245 cv_wait_sig(struct cv *cvp, struct mtx *mp) 246 { 247 struct thread *td; 248 struct proc *p; 249 int rval; 250 int sig; 251 WITNESS_SAVE_DECL(mp); 252 253 td = curthread; 254 p = td->td_proc; 255 rval = 0; 256 #ifdef KTRACE 257 if (KTRPOINT(td, KTR_CSW)) 258 ktrcsw(1, 0); 259 #endif 260 CV_ASSERT(cvp, mp, td); 261 WITNESS_SLEEP(0, &mp->mtx_object); 262 WITNESS_SAVE(&mp->mtx_object, mp); 263 264 if (cold || panicstr) { 265 /* 266 * After a panic, or during autoconfiguration, just give 267 * interrupts a chance, then just return; don't run any other 268 * procs or panic below, in case this is the idle process and 269 * already asleep. 270 */ 271 return 0; 272 } 273 274 mtx_lock_spin(&sched_lock); 275 276 CV_WAIT_VALIDATE(cvp, mp); 277 278 DROP_GIANT(); 279 mtx_unlock(mp); 280 281 cv_waitq_add(cvp, td); 282 sig = cv_switch_catch(td); 283 284 mtx_unlock_spin(&sched_lock); 285 286 PROC_LOCK(p); 287 if (sig == 0) 288 sig = cursig(p); /* XXXKSE */ 289 if (sig != 0) { 290 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig)) 291 rval = EINTR; 292 else 293 rval = ERESTART; 294 } 295 PROC_UNLOCK(p); 296 297 #ifdef KTRACE 298 if (KTRPOINT(td, KTR_CSW)) 299 ktrcsw(0, 0); 300 #endif 301 PICKUP_GIANT(); 302 mtx_lock(mp); 303 WITNESS_RESTORE(&mp->mtx_object, mp); 304 305 return (rval); 306 } 307 308 /* 309 * Wait on a condition variable for at most timo/hz seconds. Returns 0 if the 310 * process was resumed by cv_signal or cv_broadcast, EWOULDBLOCK if the timeout 311 * expires. 312 */ 313 int 314 cv_timedwait(struct cv *cvp, struct mtx *mp, int timo) 315 { 316 struct thread *td; 317 int rval; 318 WITNESS_SAVE_DECL(mp); 319 320 td = curthread; 321 rval = 0; 322 #ifdef KTRACE 323 if (KTRPOINT(td, KTR_CSW)) 324 ktrcsw(1, 0); 325 #endif 326 CV_ASSERT(cvp, mp, td); 327 WITNESS_SLEEP(0, &mp->mtx_object); 328 WITNESS_SAVE(&mp->mtx_object, mp); 329 330 if (cold || panicstr) { 331 /* 332 * After a panic, or during autoconfiguration, just give 333 * interrupts a chance, then just return; don't run any other 334 * thread or panic below, in case this is the idle process and 335 * already asleep. 336 */ 337 return 0; 338 } 339 340 mtx_lock_spin(&sched_lock); 341 342 CV_WAIT_VALIDATE(cvp, mp); 343 344 DROP_GIANT(); 345 mtx_unlock(mp); 346 347 cv_waitq_add(cvp, td); 348 callout_reset(&td->td_slpcallout, timo, cv_timedwait_end, td); 349 cv_switch(td); 350 351 if (td->td_flags & TDF_TIMEOUT) { 352 td->td_flags &= ~TDF_TIMEOUT; 353 rval = EWOULDBLOCK; 354 } else if (td->td_flags & TDF_TIMOFAIL) 355 td->td_flags &= ~TDF_TIMOFAIL; 356 else if (callout_stop(&td->td_slpcallout) == 0) { 357 /* 358 * Work around race with cv_timedwait_end similar to that 359 * between msleep and endtsleep. 360 */ 361 td->td_flags |= TDF_TIMEOUT; 362 td->td_proc->p_stats->p_ru.ru_nivcsw++; 363 mi_switch(); 364 } 365 366 mtx_unlock_spin(&sched_lock); 367 #ifdef KTRACE 368 if (KTRPOINT(td, KTR_CSW)) 369 ktrcsw(0, 0); 370 #endif 371 PICKUP_GIANT(); 372 mtx_lock(mp); 373 WITNESS_RESTORE(&mp->mtx_object, mp); 374 375 return (rval); 376 } 377 378 /* 379 * Wait on a condition variable for at most timo/hz seconds, allowing 380 * interruption by signals. Returns 0 if the thread was resumed by cv_signal 381 * or cv_broadcast, EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if 382 * a signal was caught. 383 */ 384 int 385 cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo) 386 { 387 struct thread *td; 388 struct proc *p; 389 int rval; 390 int sig; 391 WITNESS_SAVE_DECL(mp); 392 393 td = curthread; 394 p = td->td_proc; 395 rval = 0; 396 #ifdef KTRACE 397 if (KTRPOINT(td, KTR_CSW)) 398 ktrcsw(1, 0); 399 #endif 400 CV_ASSERT(cvp, mp, td); 401 WITNESS_SLEEP(0, &mp->mtx_object); 402 WITNESS_SAVE(&mp->mtx_object, mp); 403 404 if (cold || panicstr) { 405 /* 406 * After a panic, or during autoconfiguration, just give 407 * interrupts a chance, then just return; don't run any other 408 * thread or panic below, in case this is the idle process and 409 * already asleep. 410 */ 411 return 0; 412 } 413 414 mtx_lock_spin(&sched_lock); 415 416 CV_WAIT_VALIDATE(cvp, mp); 417 418 DROP_GIANT(); 419 mtx_unlock(mp); 420 421 cv_waitq_add(cvp, td); 422 callout_reset(&td->td_slpcallout, timo, cv_timedwait_end, td); 423 sig = cv_switch_catch(td); 424 425 if (td->td_flags & TDF_TIMEOUT) { 426 td->td_flags &= ~TDF_TIMEOUT; 427 rval = EWOULDBLOCK; 428 } else if (td->td_flags & TDF_TIMOFAIL) 429 td->td_flags &= ~TDF_TIMOFAIL; 430 else if (callout_stop(&td->td_slpcallout) == 0) { 431 /* 432 * Work around race with cv_timedwait_end similar to that 433 * between msleep and endtsleep. 434 */ 435 td->td_flags |= TDF_TIMEOUT; 436 td->td_proc->p_stats->p_ru.ru_nivcsw++; 437 mi_switch(); 438 } 439 440 mtx_unlock_spin(&sched_lock); 441 442 PROC_LOCK(p); 443 if (sig == 0) 444 sig = cursig(p); 445 if (sig != 0) { 446 if (SIGISMEMBER(p->p_sigacts->ps_sigintr, sig)) 447 rval = EINTR; 448 else 449 rval = ERESTART; 450 } 451 PROC_UNLOCK(p); 452 453 #ifdef KTRACE 454 if (KTRPOINT(td, KTR_CSW)) 455 ktrcsw(0, 0); 456 #endif 457 PICKUP_GIANT(); 458 mtx_lock(mp); 459 WITNESS_RESTORE(&mp->mtx_object, mp); 460 461 return (rval); 462 } 463 464 /* 465 * Common code for signal and broadcast. Assumes waitq is not empty. Must be 466 * called with sched_lock held. 467 */ 468 static __inline void 469 cv_wakeup(struct cv *cvp) 470 { 471 struct thread *td; 472 473 mtx_assert(&sched_lock, MA_OWNED); 474 td = TAILQ_FIRST(&cvp->cv_waitq); 475 KASSERT(td->td_wchan == cvp, ("%s: bogus wchan", __func__)); 476 KASSERT(td->td_flags & TDF_CVWAITQ, ("%s: not on waitq", __func__)); 477 TAILQ_REMOVE(&cvp->cv_waitq, td, td_slpq); 478 td->td_flags &= ~TDF_CVWAITQ; 479 td->td_wchan = 0; 480 if (td->td_proc->p_stat == SSLEEP) { 481 /* OPTIMIZED EXPANSION OF setrunnable(td); */ 482 CTR3(KTR_PROC, "cv_signal: thread %p (pid %d, %s)", 483 td, td->td_proc->p_pid, td->td_proc->p_comm); 484 if (td->td_ksegrp->kg_slptime > 1) /* XXXKSE */ 485 updatepri(td); 486 td->td_kse->ke_slptime = 0; 487 td->td_ksegrp->kg_slptime = 0; 488 td->td_proc->p_stat = SRUN; 489 if (td->td_proc->p_sflag & PS_INMEM) { 490 setrunqueue(td); 491 maybe_resched(td); 492 } else { 493 td->td_proc->p_sflag |= PS_SWAPINREQ; 494 wakeup(&proc0); /* XXXKSE */ 495 } 496 /* END INLINE EXPANSION */ 497 } 498 } 499 500 /* 501 * Signal a condition variable, wakes up one waiting thread. Will also wakeup 502 * the swapper if the process is not in memory, so that it can bring the 503 * sleeping process in. Note that this may also result in additional threads 504 * being made runnable. Should be called with the same mutex as was passed to 505 * cv_wait held. 506 */ 507 void 508 cv_signal(struct cv *cvp) 509 { 510 511 KASSERT(cvp != NULL, ("%s: cvp NULL", __func__)); 512 mtx_lock_spin(&sched_lock); 513 if (!TAILQ_EMPTY(&cvp->cv_waitq)) { 514 CV_SIGNAL_VALIDATE(cvp); 515 cv_wakeup(cvp); 516 } 517 mtx_unlock_spin(&sched_lock); 518 } 519 520 /* 521 * Broadcast a signal to a condition variable. Wakes up all waiting threads. 522 * Should be called with the same mutex as was passed to cv_wait held. 523 */ 524 void 525 cv_broadcast(struct cv *cvp) 526 { 527 528 KASSERT(cvp != NULL, ("%s: cvp NULL", __func__)); 529 mtx_lock_spin(&sched_lock); 530 CV_SIGNAL_VALIDATE(cvp); 531 while (!TAILQ_EMPTY(&cvp->cv_waitq)) 532 cv_wakeup(cvp); 533 mtx_unlock_spin(&sched_lock); 534 } 535 536 /* 537 * Remove a thread from the wait queue of its condition variable. This may be 538 * called externally. 539 */ 540 void 541 cv_waitq_remove(struct thread *td) 542 { 543 struct cv *cvp; 544 545 mtx_lock_spin(&sched_lock); 546 if ((cvp = td->td_wchan) != NULL && td->td_flags & TDF_CVWAITQ) { 547 TAILQ_REMOVE(&cvp->cv_waitq, td, td_slpq); 548 td->td_flags &= ~TDF_CVWAITQ; 549 td->td_wchan = NULL; 550 } 551 mtx_unlock_spin(&sched_lock); 552 } 553 554 /* 555 * Timeout function for cv_timedwait. Put the thread on the runqueue and set 556 * its timeout flag. 557 */ 558 static void 559 cv_timedwait_end(void *arg) 560 { 561 struct thread *td; 562 563 td = arg; 564 CTR3(KTR_PROC, "cv_timedwait_end: thread %p (pid %d, %s)", td, td->td_proc->p_pid, 565 td->td_proc->p_comm); 566 mtx_lock_spin(&sched_lock); 567 if (td->td_flags & TDF_TIMEOUT) { 568 td->td_flags &= ~TDF_TIMEOUT; 569 setrunqueue(td); 570 } else if (td->td_wchan != NULL) { 571 if (td->td_proc->p_stat == SSLEEP) /* XXXKSE */ 572 setrunnable(td); 573 else 574 cv_waitq_remove(td); 575 td->td_flags |= TDF_TIMEOUT; 576 } else 577 td->td_flags |= TDF_TIMOFAIL; 578 mtx_unlock_spin(&sched_lock); 579 } 580