1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include "opt_ktrace.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/limits.h> 37 #include <sys/lock.h> 38 #include <sys/mutex.h> 39 #include <sys/proc.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/condvar.h> 43 #include <sys/sched.h> 44 #include <sys/signalvar.h> 45 #include <sys/sleepqueue.h> 46 #include <sys/resourcevar.h> 47 #ifdef KTRACE 48 #include <sys/uio.h> 49 #include <sys/ktrace.h> 50 #endif 51 52 /* 53 * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound, 54 * cv_signal must manually check the wait queue for threads. 55 */ 56 #define CV_WAITERS_BOUND INT_MAX 57 58 #define CV_WAITERS_INC(cvp) do { \ 59 if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \ 60 (cvp)->cv_waiters++; \ 61 } while (0) 62 63 /* 64 * Common sanity checks for cv_wait* functions. 65 */ 66 #define CV_ASSERT(cvp, lock, td) do { \ 67 KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 68 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 69 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 70 KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 71 } while (0) 72 73 /* 74 * Initialize a condition variable. Must be called before use. 75 */ 76 void 77 cv_init(struct cv *cvp, const char *desc) 78 { 79 80 cvp->cv_description = desc; 81 cvp->cv_waiters = 0; 82 } 83 84 /* 85 * Destroy a condition variable. The condition variable must be re-initialized 86 * in order to be re-used. 87 */ 88 void 89 cv_destroy(struct cv *cvp) 90 { 91 #ifdef INVARIANTS 92 struct sleepqueue *sq; 93 94 sleepq_lock(cvp); 95 sq = sleepq_lookup(cvp); 96 sleepq_release(cvp); 97 KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 98 #endif 99 } 100 101 /* 102 * Wait on a condition variable. The current thread is placed on the condition 103 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 104 * condition variable will resume the thread. The mutex is released before 105 * sleeping and will be held on return. It is recommended that the mutex be 106 * held when cv_signal or cv_broadcast are called. 107 */ 108 void 109 _cv_wait(struct cv *cvp, struct lock_object *lock) 110 { 111 WITNESS_SAVE_DECL(lock_witness); 112 struct lock_class *class; 113 struct thread *td; 114 uintptr_t lock_state; 115 116 td = curthread; 117 lock_state = 0; 118 #ifdef KTRACE 119 if (KTRPOINT(td, KTR_CSW)) 120 ktrcsw(1, 0, cv_wmesg(cvp)); 121 #endif 122 CV_ASSERT(cvp, lock, td); 123 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 124 "Waiting on \"%s\"", cvp->cv_description); 125 class = LOCK_CLASS(lock); 126 127 if (SCHEDULER_STOPPED_TD(td)) 128 return; 129 130 sleepq_lock(cvp); 131 132 CV_WAITERS_INC(cvp); 133 if (lock == &Giant.lock_object) 134 mtx_assert(&Giant, MA_OWNED); 135 DROP_GIANT(); 136 137 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 138 if (lock != &Giant.lock_object) { 139 if (class->lc_flags & LC_SLEEPABLE) 140 sleepq_release(cvp); 141 WITNESS_SAVE(lock, lock_witness); 142 lock_state = class->lc_unlock(lock); 143 if (class->lc_flags & LC_SLEEPABLE) 144 sleepq_lock(cvp); 145 } 146 sleepq_wait(cvp, 0); 147 148 #ifdef KTRACE 149 if (KTRPOINT(td, KTR_CSW)) 150 ktrcsw(0, 0, cv_wmesg(cvp)); 151 #endif 152 PICKUP_GIANT(); 153 if (lock != &Giant.lock_object) { 154 class->lc_lock(lock, lock_state); 155 WITNESS_RESTORE(lock, lock_witness); 156 } 157 } 158 159 /* 160 * Wait on a condition variable. This function differs from cv_wait by 161 * not acquiring the mutex after condition variable was signaled. 162 */ 163 void 164 _cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 165 { 166 struct lock_class *class; 167 struct thread *td; 168 169 td = curthread; 170 #ifdef KTRACE 171 if (KTRPOINT(td, KTR_CSW)) 172 ktrcsw(1, 0, cv_wmesg(cvp)); 173 #endif 174 CV_ASSERT(cvp, lock, td); 175 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 176 "Waiting on \"%s\"", cvp->cv_description); 177 KASSERT(lock != &Giant.lock_object, 178 ("cv_wait_unlock cannot be used with Giant")); 179 class = LOCK_CLASS(lock); 180 181 if (SCHEDULER_STOPPED_TD(td)) { 182 class->lc_unlock(lock); 183 return; 184 } 185 186 sleepq_lock(cvp); 187 188 CV_WAITERS_INC(cvp); 189 DROP_GIANT(); 190 191 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 192 if (class->lc_flags & LC_SLEEPABLE) 193 sleepq_release(cvp); 194 class->lc_unlock(lock); 195 if (class->lc_flags & LC_SLEEPABLE) 196 sleepq_lock(cvp); 197 sleepq_wait(cvp, 0); 198 199 #ifdef KTRACE 200 if (KTRPOINT(td, KTR_CSW)) 201 ktrcsw(0, 0, cv_wmesg(cvp)); 202 #endif 203 PICKUP_GIANT(); 204 } 205 206 /* 207 * Wait on a condition variable, allowing interruption by signals. Return 0 if 208 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 209 * a signal was caught. If ERESTART is returned the system call should be 210 * restarted if possible. 211 */ 212 int 213 _cv_wait_sig(struct cv *cvp, struct lock_object *lock) 214 { 215 WITNESS_SAVE_DECL(lock_witness); 216 struct lock_class *class; 217 struct thread *td; 218 uintptr_t lock_state; 219 int rval; 220 221 td = curthread; 222 lock_state = 0; 223 #ifdef KTRACE 224 if (KTRPOINT(td, KTR_CSW)) 225 ktrcsw(1, 0, cv_wmesg(cvp)); 226 #endif 227 CV_ASSERT(cvp, lock, td); 228 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 229 "Waiting on \"%s\"", cvp->cv_description); 230 class = LOCK_CLASS(lock); 231 232 if (SCHEDULER_STOPPED_TD(td)) 233 return (0); 234 235 sleepq_lock(cvp); 236 237 CV_WAITERS_INC(cvp); 238 if (lock == &Giant.lock_object) 239 mtx_assert(&Giant, MA_OWNED); 240 DROP_GIANT(); 241 242 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 243 SLEEPQ_INTERRUPTIBLE, 0); 244 if (lock != &Giant.lock_object) { 245 if (class->lc_flags & LC_SLEEPABLE) 246 sleepq_release(cvp); 247 WITNESS_SAVE(lock, lock_witness); 248 lock_state = class->lc_unlock(lock); 249 if (class->lc_flags & LC_SLEEPABLE) 250 sleepq_lock(cvp); 251 } 252 rval = sleepq_wait_sig(cvp, 0); 253 254 #ifdef KTRACE 255 if (KTRPOINT(td, KTR_CSW)) 256 ktrcsw(0, 0, cv_wmesg(cvp)); 257 #endif 258 PICKUP_GIANT(); 259 if (lock != &Giant.lock_object) { 260 class->lc_lock(lock, lock_state); 261 WITNESS_RESTORE(lock, lock_witness); 262 } 263 264 return (rval); 265 } 266 267 /* 268 * Wait on a condition variable for (at most) the value specified in sbt 269 * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 270 * EWOULDBLOCK if the timeout expires. 271 */ 272 int 273 _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 274 sbintime_t pr, int flags) 275 { 276 WITNESS_SAVE_DECL(lock_witness); 277 struct lock_class *class; 278 struct thread *td; 279 int lock_state, rval; 280 281 td = curthread; 282 lock_state = 0; 283 #ifdef KTRACE 284 if (KTRPOINT(td, KTR_CSW)) 285 ktrcsw(1, 0, cv_wmesg(cvp)); 286 #endif 287 CV_ASSERT(cvp, lock, td); 288 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 289 "Waiting on \"%s\"", cvp->cv_description); 290 class = LOCK_CLASS(lock); 291 292 if (SCHEDULER_STOPPED_TD(td)) 293 return (0); 294 295 sleepq_lock(cvp); 296 297 CV_WAITERS_INC(cvp); 298 if (lock == &Giant.lock_object) 299 mtx_assert(&Giant, MA_OWNED); 300 DROP_GIANT(); 301 302 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 303 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 304 if (lock != &Giant.lock_object) { 305 if (class->lc_flags & LC_SLEEPABLE) 306 sleepq_release(cvp); 307 WITNESS_SAVE(lock, lock_witness); 308 lock_state = class->lc_unlock(lock); 309 if (class->lc_flags & LC_SLEEPABLE) 310 sleepq_lock(cvp); 311 } 312 rval = sleepq_timedwait(cvp, 0); 313 314 #ifdef KTRACE 315 if (KTRPOINT(td, KTR_CSW)) 316 ktrcsw(0, 0, cv_wmesg(cvp)); 317 #endif 318 PICKUP_GIANT(); 319 if (lock != &Giant.lock_object) { 320 class->lc_lock(lock, lock_state); 321 WITNESS_RESTORE(lock, lock_witness); 322 } 323 324 return (rval); 325 } 326 327 /* 328 * Wait on a condition variable for (at most) the value specified in sbt 329 * argument, allowing interruption by signals. 330 * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 331 * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 332 * was caught. 333 */ 334 int 335 _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 336 sbintime_t sbt, sbintime_t pr, int flags) 337 { 338 WITNESS_SAVE_DECL(lock_witness); 339 struct lock_class *class; 340 struct thread *td; 341 int lock_state, rval; 342 343 td = curthread; 344 lock_state = 0; 345 #ifdef KTRACE 346 if (KTRPOINT(td, KTR_CSW)) 347 ktrcsw(1, 0, cv_wmesg(cvp)); 348 #endif 349 CV_ASSERT(cvp, lock, td); 350 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 351 "Waiting on \"%s\"", cvp->cv_description); 352 class = LOCK_CLASS(lock); 353 354 if (SCHEDULER_STOPPED_TD(td)) 355 return (0); 356 357 sleepq_lock(cvp); 358 359 CV_WAITERS_INC(cvp); 360 if (lock == &Giant.lock_object) 361 mtx_assert(&Giant, MA_OWNED); 362 DROP_GIANT(); 363 364 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 365 SLEEPQ_INTERRUPTIBLE, 0); 366 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 367 if (lock != &Giant.lock_object) { 368 if (class->lc_flags & LC_SLEEPABLE) 369 sleepq_release(cvp); 370 WITNESS_SAVE(lock, lock_witness); 371 lock_state = class->lc_unlock(lock); 372 if (class->lc_flags & LC_SLEEPABLE) 373 sleepq_lock(cvp); 374 } 375 rval = sleepq_timedwait_sig(cvp, 0); 376 377 #ifdef KTRACE 378 if (KTRPOINT(td, KTR_CSW)) 379 ktrcsw(0, 0, cv_wmesg(cvp)); 380 #endif 381 PICKUP_GIANT(); 382 if (lock != &Giant.lock_object) { 383 class->lc_lock(lock, lock_state); 384 WITNESS_RESTORE(lock, lock_witness); 385 } 386 387 return (rval); 388 } 389 390 /* 391 * Signal a condition variable, wakes up one waiting thread. Will also wakeup 392 * the swapper if the process is not in memory, so that it can bring the 393 * sleeping process in. Note that this may also result in additional threads 394 * being made runnable. Should be called with the same mutex as was passed to 395 * cv_wait held. 396 */ 397 void 398 cv_signal(struct cv *cvp) 399 { 400 401 if (cvp->cv_waiters == 0) 402 return; 403 sleepq_lock(cvp); 404 if (cvp->cv_waiters == 0) { 405 sleepq_release(cvp); 406 return; 407 } 408 if (cvp->cv_waiters == CV_WAITERS_BOUND && sleepq_lookup(cvp) == NULL) { 409 cvp->cv_waiters = 0; 410 sleepq_release(cvp); 411 } else { 412 if (cvp->cv_waiters < CV_WAITERS_BOUND) 413 cvp->cv_waiters--; 414 if (sleepq_signal(cvp, SLEEPQ_CONDVAR | SLEEPQ_DROP, 0, 0)) 415 kick_proc0(); 416 } 417 } 418 419 /* 420 * Broadcast a signal to a condition variable. Wakes up all waiting threads. 421 * Should be called with the same mutex as was passed to cv_wait held. 422 */ 423 void 424 cv_broadcastpri(struct cv *cvp, int pri) 425 { 426 int wakeup_swapper; 427 428 if (cvp->cv_waiters == 0) 429 return; 430 /* 431 * XXX sleepq_broadcast pri argument changed from -1 meaning 432 * no pri to 0 meaning no pri. 433 */ 434 wakeup_swapper = 0; 435 if (pri == -1) 436 pri = 0; 437 sleepq_lock(cvp); 438 if (cvp->cv_waiters > 0) { 439 cvp->cv_waiters = 0; 440 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 441 } 442 sleepq_release(cvp); 443 if (wakeup_swapper) 444 kick_proc0(); 445 } 446