1 /*- 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include "opt_witness.h" 30 #include "opt_hwpmc_hooks.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/mutex.h> 40 #include <sys/proc.h> 41 #include <sys/rangelock.h> 42 #include <sys/resourcevar.h> 43 #include <sys/sdt.h> 44 #include <sys/smp.h> 45 #include <sys/sched.h> 46 #include <sys/sleepqueue.h> 47 #include <sys/selinfo.h> 48 #include <sys/sysent.h> 49 #include <sys/turnstile.h> 50 #include <sys/ktr.h> 51 #include <sys/rwlock.h> 52 #include <sys/umtx.h> 53 #include <sys/cpuset.h> 54 #ifdef HWPMC_HOOKS 55 #include <sys/pmckern.h> 56 #endif 57 58 #include <security/audit/audit.h> 59 60 #include <vm/vm.h> 61 #include <vm/vm_extern.h> 62 #include <vm/uma.h> 63 #include <vm/vm_domain.h> 64 #include <sys/eventhandler.h> 65 66 SDT_PROVIDER_DECLARE(proc); 67 SDT_PROBE_DEFINE(proc, , , lwp__exit); 68 69 /* 70 * thread related storage. 71 */ 72 static uma_zone_t thread_zone; 73 74 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 75 static struct mtx zombie_lock; 76 MTX_SYSINIT(zombie_lock, &zombie_lock, "zombie lock", MTX_SPIN); 77 78 static void thread_zombie(struct thread *); 79 static int thread_unsuspend_one(struct thread *td, struct proc *p, 80 bool boundary); 81 82 #define TID_BUFFER_SIZE 1024 83 84 struct mtx tid_lock; 85 static struct unrhdr *tid_unrhdr; 86 static lwpid_t tid_buffer[TID_BUFFER_SIZE]; 87 static int tid_head, tid_tail; 88 static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); 89 90 struct tidhashhead *tidhashtbl; 91 u_long tidhash; 92 struct rwlock tidhash_lock; 93 94 static lwpid_t 95 tid_alloc(void) 96 { 97 lwpid_t tid; 98 99 tid = alloc_unr(tid_unrhdr); 100 if (tid != -1) 101 return (tid); 102 mtx_lock(&tid_lock); 103 if (tid_head == tid_tail) { 104 mtx_unlock(&tid_lock); 105 return (-1); 106 } 107 tid = tid_buffer[tid_head]; 108 tid_head = (tid_head + 1) % TID_BUFFER_SIZE; 109 mtx_unlock(&tid_lock); 110 return (tid); 111 } 112 113 static void 114 tid_free(lwpid_t tid) 115 { 116 lwpid_t tmp_tid = -1; 117 118 mtx_lock(&tid_lock); 119 if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) { 120 tmp_tid = tid_buffer[tid_head]; 121 tid_head = (tid_head + 1) % TID_BUFFER_SIZE; 122 } 123 tid_buffer[tid_tail] = tid; 124 tid_tail = (tid_tail + 1) % TID_BUFFER_SIZE; 125 mtx_unlock(&tid_lock); 126 if (tmp_tid != -1) 127 free_unr(tid_unrhdr, tmp_tid); 128 } 129 130 /* 131 * Prepare a thread for use. 132 */ 133 static int 134 thread_ctor(void *mem, int size, void *arg, int flags) 135 { 136 struct thread *td; 137 138 td = (struct thread *)mem; 139 td->td_state = TDS_INACTIVE; 140 td->td_oncpu = NOCPU; 141 142 td->td_tid = tid_alloc(); 143 144 /* 145 * Note that td_critnest begins life as 1 because the thread is not 146 * running and is thereby implicitly waiting to be on the receiving 147 * end of a context switch. 148 */ 149 td->td_critnest = 1; 150 td->td_lend_user_pri = PRI_MAX; 151 EVENTHANDLER_INVOKE(thread_ctor, td); 152 #ifdef AUDIT 153 audit_thread_alloc(td); 154 #endif 155 umtx_thread_alloc(td); 156 return (0); 157 } 158 159 /* 160 * Reclaim a thread after use. 161 */ 162 static void 163 thread_dtor(void *mem, int size, void *arg) 164 { 165 struct thread *td; 166 167 td = (struct thread *)mem; 168 169 #ifdef INVARIANTS 170 /* Verify that this thread is in a safe state to free. */ 171 switch (td->td_state) { 172 case TDS_INHIBITED: 173 case TDS_RUNNING: 174 case TDS_CAN_RUN: 175 case TDS_RUNQ: 176 /* 177 * We must never unlink a thread that is in one of 178 * these states, because it is currently active. 179 */ 180 panic("bad state for thread unlinking"); 181 /* NOTREACHED */ 182 case TDS_INACTIVE: 183 break; 184 default: 185 panic("bad thread state"); 186 /* NOTREACHED */ 187 } 188 #endif 189 #ifdef AUDIT 190 audit_thread_free(td); 191 #endif 192 /* Free all OSD associated to this thread. */ 193 osd_thread_exit(td); 194 195 EVENTHANDLER_INVOKE(thread_dtor, td); 196 tid_free(td->td_tid); 197 } 198 199 /* 200 * Initialize type-stable parts of a thread (when newly created). 201 */ 202 static int 203 thread_init(void *mem, int size, int flags) 204 { 205 struct thread *td; 206 207 td = (struct thread *)mem; 208 209 td->td_sleepqueue = sleepq_alloc(); 210 td->td_turnstile = turnstile_alloc(); 211 td->td_rlqe = NULL; 212 EVENTHANDLER_INVOKE(thread_init, td); 213 td->td_sched = (struct td_sched *)&td[1]; 214 umtx_thread_init(td); 215 td->td_kstack = 0; 216 td->td_sel = NULL; 217 return (0); 218 } 219 220 /* 221 * Tear down type-stable parts of a thread (just before being discarded). 222 */ 223 static void 224 thread_fini(void *mem, int size) 225 { 226 struct thread *td; 227 228 td = (struct thread *)mem; 229 EVENTHANDLER_INVOKE(thread_fini, td); 230 rlqentry_free(td->td_rlqe); 231 turnstile_free(td->td_turnstile); 232 sleepq_free(td->td_sleepqueue); 233 umtx_thread_fini(td); 234 seltdfini(td); 235 } 236 237 /* 238 * For a newly created process, 239 * link up all the structures and its initial threads etc. 240 * called from: 241 * {arch}/{arch}/machdep.c {arch}_init(), init386() etc. 242 * proc_dtor() (should go away) 243 * proc_init() 244 */ 245 void 246 proc_linkup0(struct proc *p, struct thread *td) 247 { 248 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 249 proc_linkup(p, td); 250 } 251 252 void 253 proc_linkup(struct proc *p, struct thread *td) 254 { 255 256 sigqueue_init(&p->p_sigqueue, p); 257 p->p_ksi = ksiginfo_alloc(1); 258 if (p->p_ksi != NULL) { 259 /* XXX p_ksi may be null if ksiginfo zone is not ready */ 260 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 261 } 262 LIST_INIT(&p->p_mqnotifier); 263 p->p_numthreads = 0; 264 thread_link(td, p); 265 } 266 267 /* 268 * Initialize global thread allocation resources. 269 */ 270 void 271 threadinit(void) 272 { 273 274 mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 275 276 /* 277 * pid_max cannot be greater than PID_MAX. 278 * leave one number for thread0. 279 */ 280 tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); 281 282 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 283 thread_ctor, thread_dtor, thread_init, thread_fini, 284 16 - 1, 0); 285 tidhashtbl = hashinit(maxproc / 2, M_TIDHASH, &tidhash); 286 rw_init(&tidhash_lock, "tidhash"); 287 } 288 289 /* 290 * Place an unused thread on the zombie list. 291 * Use the slpq as that must be unused by now. 292 */ 293 void 294 thread_zombie(struct thread *td) 295 { 296 mtx_lock_spin(&zombie_lock); 297 TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); 298 mtx_unlock_spin(&zombie_lock); 299 } 300 301 /* 302 * Release a thread that has exited after cpu_throw(). 303 */ 304 void 305 thread_stash(struct thread *td) 306 { 307 atomic_subtract_rel_int(&td->td_proc->p_exitthreads, 1); 308 thread_zombie(td); 309 } 310 311 /* 312 * Reap zombie resources. 313 */ 314 void 315 thread_reap(void) 316 { 317 struct thread *td_first, *td_next; 318 319 /* 320 * Don't even bother to lock if none at this instant, 321 * we really don't care about the next instant.. 322 */ 323 if (!TAILQ_EMPTY(&zombie_threads)) { 324 mtx_lock_spin(&zombie_lock); 325 td_first = TAILQ_FIRST(&zombie_threads); 326 if (td_first) 327 TAILQ_INIT(&zombie_threads); 328 mtx_unlock_spin(&zombie_lock); 329 while (td_first) { 330 td_next = TAILQ_NEXT(td_first, td_slpq); 331 thread_cow_free(td_first); 332 thread_free(td_first); 333 td_first = td_next; 334 } 335 } 336 } 337 338 /* 339 * Allocate a thread. 340 */ 341 struct thread * 342 thread_alloc(int pages) 343 { 344 struct thread *td; 345 346 thread_reap(); /* check if any zombies to get */ 347 348 td = (struct thread *)uma_zalloc(thread_zone, M_WAITOK); 349 KASSERT(td->td_kstack == 0, ("thread_alloc got thread with kstack")); 350 if (!vm_thread_new(td, pages)) { 351 uma_zfree(thread_zone, td); 352 return (NULL); 353 } 354 cpu_thread_alloc(td); 355 vm_domain_policy_init(&td->td_vm_dom_policy); 356 return (td); 357 } 358 359 int 360 thread_alloc_stack(struct thread *td, int pages) 361 { 362 363 KASSERT(td->td_kstack == 0, 364 ("thread_alloc_stack called on a thread with kstack")); 365 if (!vm_thread_new(td, pages)) 366 return (0); 367 cpu_thread_alloc(td); 368 return (1); 369 } 370 371 /* 372 * Deallocate a thread. 373 */ 374 void 375 thread_free(struct thread *td) 376 { 377 378 lock_profile_thread_exit(td); 379 if (td->td_cpuset) 380 cpuset_rel(td->td_cpuset); 381 td->td_cpuset = NULL; 382 cpu_thread_free(td); 383 if (td->td_kstack != 0) 384 vm_thread_dispose(td); 385 vm_domain_policy_cleanup(&td->td_vm_dom_policy); 386 uma_zfree(thread_zone, td); 387 } 388 389 void 390 thread_cow_get_proc(struct thread *newtd, struct proc *p) 391 { 392 393 PROC_LOCK_ASSERT(p, MA_OWNED); 394 newtd->td_ucred = crhold(p->p_ucred); 395 newtd->td_limit = lim_hold(p->p_limit); 396 newtd->td_cowgen = p->p_cowgen; 397 } 398 399 void 400 thread_cow_get(struct thread *newtd, struct thread *td) 401 { 402 403 newtd->td_ucred = crhold(td->td_ucred); 404 newtd->td_limit = lim_hold(td->td_limit); 405 newtd->td_cowgen = td->td_cowgen; 406 } 407 408 void 409 thread_cow_free(struct thread *td) 410 { 411 412 if (td->td_ucred) 413 crfree(td->td_ucred); 414 if (td->td_limit) 415 lim_free(td->td_limit); 416 } 417 418 void 419 thread_cow_update(struct thread *td) 420 { 421 struct proc *p; 422 423 p = td->td_proc; 424 PROC_LOCK(p); 425 if (td->td_ucred != p->p_ucred) 426 cred_update_thread(td); 427 if (td->td_limit != p->p_limit) 428 lim_update_thread(td); 429 td->td_cowgen = p->p_cowgen; 430 PROC_UNLOCK(p); 431 } 432 433 /* 434 * Discard the current thread and exit from its context. 435 * Always called with scheduler locked. 436 * 437 * Because we can't free a thread while we're operating under its context, 438 * push the current thread into our CPU's deadthread holder. This means 439 * we needn't worry about someone else grabbing our context before we 440 * do a cpu_throw(). 441 */ 442 void 443 thread_exit(void) 444 { 445 uint64_t runtime, new_switchtime; 446 struct thread *td; 447 struct thread *td2; 448 struct proc *p; 449 int wakeup_swapper; 450 451 td = curthread; 452 p = td->td_proc; 453 454 PROC_SLOCK_ASSERT(p, MA_OWNED); 455 mtx_assert(&Giant, MA_NOTOWNED); 456 457 PROC_LOCK_ASSERT(p, MA_OWNED); 458 KASSERT(p != NULL, ("thread exiting without a process")); 459 CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 460 (long)p->p_pid, td->td_name); 461 KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 462 463 #ifdef AUDIT 464 AUDIT_SYSCALL_EXIT(0, td); 465 #endif 466 /* 467 * drop FPU & debug register state storage, or any other 468 * architecture specific resources that 469 * would not be on a new untouched process. 470 */ 471 cpu_thread_exit(td); /* XXXSMP */ 472 473 /* 474 * The last thread is left attached to the process 475 * So that the whole bundle gets recycled. Skip 476 * all this stuff if we never had threads. 477 * EXIT clears all sign of other threads when 478 * it goes to single threading, so the last thread always 479 * takes the short path. 480 */ 481 if (p->p_flag & P_HADTHREADS) { 482 if (p->p_numthreads > 1) { 483 atomic_add_int(&td->td_proc->p_exitthreads, 1); 484 thread_unlink(td); 485 td2 = FIRST_THREAD_IN_PROC(p); 486 sched_exit_thread(td2, td); 487 488 /* 489 * The test below is NOT true if we are the 490 * sole exiting thread. P_STOPPED_SINGLE is unset 491 * in exit1() after it is the only survivor. 492 */ 493 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 494 if (p->p_numthreads == p->p_suspcount) { 495 thread_lock(p->p_singlethread); 496 wakeup_swapper = thread_unsuspend_one( 497 p->p_singlethread, p, false); 498 thread_unlock(p->p_singlethread); 499 if (wakeup_swapper) 500 kick_proc0(); 501 } 502 } 503 504 PCPU_SET(deadthread, td); 505 } else { 506 /* 507 * The last thread is exiting.. but not through exit() 508 */ 509 panic ("thread_exit: Last thread exiting on its own"); 510 } 511 } 512 #ifdef HWPMC_HOOKS 513 /* 514 * If this thread is part of a process that is being tracked by hwpmc(4), 515 * inform the module of the thread's impending exit. 516 */ 517 if (PMC_PROC_IS_USING_PMCS(td->td_proc)) 518 PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT); 519 #endif 520 PROC_UNLOCK(p); 521 PROC_STATLOCK(p); 522 thread_lock(td); 523 PROC_SUNLOCK(p); 524 525 /* Do the same timestamp bookkeeping that mi_switch() would do. */ 526 new_switchtime = cpu_ticks(); 527 runtime = new_switchtime - PCPU_GET(switchtime); 528 td->td_runtime += runtime; 529 td->td_incruntime += runtime; 530 PCPU_SET(switchtime, new_switchtime); 531 PCPU_SET(switchticks, ticks); 532 PCPU_INC(cnt.v_swtch); 533 534 /* Save our resource usage in our process. */ 535 td->td_ru.ru_nvcsw++; 536 ruxagg(p, td); 537 rucollect(&p->p_ru, &td->td_ru); 538 PROC_STATUNLOCK(p); 539 540 td->td_state = TDS_INACTIVE; 541 #ifdef WITNESS 542 witness_thread_exit(td); 543 #endif 544 CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 545 sched_throw(td); 546 panic("I'm a teapot!"); 547 /* NOTREACHED */ 548 } 549 550 /* 551 * Do any thread specific cleanups that may be needed in wait() 552 * called with Giant, proc and schedlock not held. 553 */ 554 void 555 thread_wait(struct proc *p) 556 { 557 struct thread *td; 558 559 mtx_assert(&Giant, MA_NOTOWNED); 560 KASSERT(p->p_numthreads == 1, ("multiple threads in thread_wait()")); 561 KASSERT(p->p_exitthreads == 0, ("p_exitthreads leaking")); 562 td = FIRST_THREAD_IN_PROC(p); 563 /* Lock the last thread so we spin until it exits cpu_throw(). */ 564 thread_lock(td); 565 thread_unlock(td); 566 lock_profile_thread_exit(td); 567 cpuset_rel(td->td_cpuset); 568 td->td_cpuset = NULL; 569 cpu_thread_clean(td); 570 thread_cow_free(td); 571 thread_reap(); /* check for zombie threads etc. */ 572 } 573 574 /* 575 * Link a thread to a process. 576 * set up anything that needs to be initialized for it to 577 * be used by the process. 578 */ 579 void 580 thread_link(struct thread *td, struct proc *p) 581 { 582 583 /* 584 * XXX This can't be enabled because it's called for proc0 before 585 * its lock has been created. 586 * PROC_LOCK_ASSERT(p, MA_OWNED); 587 */ 588 td->td_state = TDS_INACTIVE; 589 td->td_proc = p; 590 td->td_flags = TDF_INMEM; 591 592 LIST_INIT(&td->td_contested); 593 LIST_INIT(&td->td_lprof[0]); 594 LIST_INIT(&td->td_lprof[1]); 595 sigqueue_init(&td->td_sigqueue, p); 596 callout_init(&td->td_slpcallout, 1); 597 TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist); 598 p->p_numthreads++; 599 } 600 601 /* 602 * Called from: 603 * thread_exit() 604 */ 605 void 606 thread_unlink(struct thread *td) 607 { 608 struct proc *p = td->td_proc; 609 610 PROC_LOCK_ASSERT(p, MA_OWNED); 611 TAILQ_REMOVE(&p->p_threads, td, td_plist); 612 p->p_numthreads--; 613 /* could clear a few other things here */ 614 /* Must NOT clear links to proc! */ 615 } 616 617 static int 618 calc_remaining(struct proc *p, int mode) 619 { 620 int remaining; 621 622 PROC_LOCK_ASSERT(p, MA_OWNED); 623 PROC_SLOCK_ASSERT(p, MA_OWNED); 624 if (mode == SINGLE_EXIT) 625 remaining = p->p_numthreads; 626 else if (mode == SINGLE_BOUNDARY) 627 remaining = p->p_numthreads - p->p_boundary_count; 628 else if (mode == SINGLE_NO_EXIT || mode == SINGLE_ALLPROC) 629 remaining = p->p_numthreads - p->p_suspcount; 630 else 631 panic("calc_remaining: wrong mode %d", mode); 632 return (remaining); 633 } 634 635 static int 636 remain_for_mode(int mode) 637 { 638 639 return (mode == SINGLE_ALLPROC ? 0 : 1); 640 } 641 642 static int 643 weed_inhib(int mode, struct thread *td2, struct proc *p) 644 { 645 int wakeup_swapper; 646 647 PROC_LOCK_ASSERT(p, MA_OWNED); 648 PROC_SLOCK_ASSERT(p, MA_OWNED); 649 THREAD_LOCK_ASSERT(td2, MA_OWNED); 650 651 wakeup_swapper = 0; 652 switch (mode) { 653 case SINGLE_EXIT: 654 if (TD_IS_SUSPENDED(td2)) 655 wakeup_swapper |= thread_unsuspend_one(td2, p, true); 656 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) 657 wakeup_swapper |= sleepq_abort(td2, EINTR); 658 break; 659 case SINGLE_BOUNDARY: 660 if (TD_IS_SUSPENDED(td2) && (td2->td_flags & TDF_BOUNDARY) == 0) 661 wakeup_swapper |= thread_unsuspend_one(td2, p, false); 662 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) 663 wakeup_swapper |= sleepq_abort(td2, ERESTART); 664 break; 665 case SINGLE_NO_EXIT: 666 if (TD_IS_SUSPENDED(td2) && (td2->td_flags & TDF_BOUNDARY) == 0) 667 wakeup_swapper |= thread_unsuspend_one(td2, p, false); 668 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) 669 wakeup_swapper |= sleepq_abort(td2, ERESTART); 670 break; 671 case SINGLE_ALLPROC: 672 /* 673 * ALLPROC suspend tries to avoid spurious EINTR for 674 * threads sleeping interruptable, by suspending the 675 * thread directly, similarly to sig_suspend_threads(). 676 * Since such sleep is not performed at the user 677 * boundary, TDF_BOUNDARY flag is not set, and TDF_ALLPROCSUSP 678 * is used to avoid immediate un-suspend. 679 */ 680 if (TD_IS_SUSPENDED(td2) && (td2->td_flags & (TDF_BOUNDARY | 681 TDF_ALLPROCSUSP)) == 0) 682 wakeup_swapper |= thread_unsuspend_one(td2, p, false); 683 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) != 0) { 684 if ((td2->td_flags & TDF_SBDRY) == 0) { 685 thread_suspend_one(td2); 686 td2->td_flags |= TDF_ALLPROCSUSP; 687 } else { 688 wakeup_swapper |= sleepq_abort(td2, ERESTART); 689 } 690 } 691 break; 692 } 693 return (wakeup_swapper); 694 } 695 696 /* 697 * Enforce single-threading. 698 * 699 * Returns 1 if the caller must abort (another thread is waiting to 700 * exit the process or similar). Process is locked! 701 * Returns 0 when you are successfully the only thread running. 702 * A process has successfully single threaded in the suspend mode when 703 * There are no threads in user mode. Threads in the kernel must be 704 * allowed to continue until they get to the user boundary. They may even 705 * copy out their return values and data before suspending. They may however be 706 * accelerated in reaching the user boundary as we will wake up 707 * any sleeping threads that are interruptable. (PCATCH). 708 */ 709 int 710 thread_single(struct proc *p, int mode) 711 { 712 struct thread *td; 713 struct thread *td2; 714 int remaining, wakeup_swapper; 715 716 td = curthread; 717 KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY || 718 mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT, 719 ("invalid mode %d", mode)); 720 /* 721 * If allowing non-ALLPROC singlethreading for non-curproc 722 * callers, calc_remaining() and remain_for_mode() should be 723 * adjusted to also account for td->td_proc != p. For now 724 * this is not implemented because it is not used. 725 */ 726 KASSERT((mode == SINGLE_ALLPROC && td->td_proc != p) || 727 (mode != SINGLE_ALLPROC && td->td_proc == p), 728 ("mode %d proc %p curproc %p", mode, p, td->td_proc)); 729 mtx_assert(&Giant, MA_NOTOWNED); 730 PROC_LOCK_ASSERT(p, MA_OWNED); 731 732 if ((p->p_flag & P_HADTHREADS) == 0 && mode != SINGLE_ALLPROC) 733 return (0); 734 735 /* Is someone already single threading? */ 736 if (p->p_singlethread != NULL && p->p_singlethread != td) 737 return (1); 738 739 if (mode == SINGLE_EXIT) { 740 p->p_flag |= P_SINGLE_EXIT; 741 p->p_flag &= ~P_SINGLE_BOUNDARY; 742 } else { 743 p->p_flag &= ~P_SINGLE_EXIT; 744 if (mode == SINGLE_BOUNDARY) 745 p->p_flag |= P_SINGLE_BOUNDARY; 746 else 747 p->p_flag &= ~P_SINGLE_BOUNDARY; 748 } 749 if (mode == SINGLE_ALLPROC) 750 p->p_flag |= P_TOTAL_STOP; 751 p->p_flag |= P_STOPPED_SINGLE; 752 PROC_SLOCK(p); 753 p->p_singlethread = td; 754 remaining = calc_remaining(p, mode); 755 while (remaining != remain_for_mode(mode)) { 756 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 757 goto stopme; 758 wakeup_swapper = 0; 759 FOREACH_THREAD_IN_PROC(p, td2) { 760 if (td2 == td) 761 continue; 762 thread_lock(td2); 763 td2->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK; 764 if (TD_IS_INHIBITED(td2)) { 765 wakeup_swapper |= weed_inhib(mode, td2, p); 766 #ifdef SMP 767 } else if (TD_IS_RUNNING(td2) && td != td2) { 768 forward_signal(td2); 769 #endif 770 } 771 thread_unlock(td2); 772 } 773 if (wakeup_swapper) 774 kick_proc0(); 775 remaining = calc_remaining(p, mode); 776 777 /* 778 * Maybe we suspended some threads.. was it enough? 779 */ 780 if (remaining == remain_for_mode(mode)) 781 break; 782 783 stopme: 784 /* 785 * Wake us up when everyone else has suspended. 786 * In the mean time we suspend as well. 787 */ 788 thread_suspend_switch(td, p); 789 remaining = calc_remaining(p, mode); 790 } 791 if (mode == SINGLE_EXIT) { 792 /* 793 * Convert the process to an unthreaded process. The 794 * SINGLE_EXIT is called by exit1() or execve(), in 795 * both cases other threads must be retired. 796 */ 797 KASSERT(p->p_numthreads == 1, ("Unthreading with >1 threads")); 798 p->p_singlethread = NULL; 799 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_HADTHREADS); 800 801 /* 802 * Wait for any remaining threads to exit cpu_throw(). 803 */ 804 while (p->p_exitthreads != 0) { 805 PROC_SUNLOCK(p); 806 PROC_UNLOCK(p); 807 sched_relinquish(td); 808 PROC_LOCK(p); 809 PROC_SLOCK(p); 810 } 811 } else if (mode == SINGLE_BOUNDARY) { 812 /* 813 * Wait until all suspended threads are removed from 814 * the processors. The thread_suspend_check() 815 * increments p_boundary_count while it is still 816 * running, which makes it possible for the execve() 817 * to destroy vmspace while our other threads are 818 * still using the address space. 819 * 820 * We lock the thread, which is only allowed to 821 * succeed after context switch code finished using 822 * the address space. 823 */ 824 FOREACH_THREAD_IN_PROC(p, td2) { 825 if (td2 == td) 826 continue; 827 thread_lock(td2); 828 KASSERT((td2->td_flags & TDF_BOUNDARY) != 0, 829 ("td %p not on boundary", td2)); 830 KASSERT(TD_IS_SUSPENDED(td2), 831 ("td %p is not suspended", td2)); 832 thread_unlock(td2); 833 } 834 } 835 PROC_SUNLOCK(p); 836 return (0); 837 } 838 839 bool 840 thread_suspend_check_needed(void) 841 { 842 struct proc *p; 843 struct thread *td; 844 845 td = curthread; 846 p = td->td_proc; 847 PROC_LOCK_ASSERT(p, MA_OWNED); 848 return (P_SHOULDSTOP(p) || ((p->p_flag & P_TRACED) != 0 && 849 (td->td_dbgflags & TDB_SUSPEND) != 0)); 850 } 851 852 /* 853 * Called in from locations that can safely check to see 854 * whether we have to suspend or at least throttle for a 855 * single-thread event (e.g. fork). 856 * 857 * Such locations include userret(). 858 * If the "return_instead" argument is non zero, the thread must be able to 859 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 860 * 861 * The 'return_instead' argument tells the function if it may do a 862 * thread_exit() or suspend, or whether the caller must abort and back 863 * out instead. 864 * 865 * If the thread that set the single_threading request has set the 866 * P_SINGLE_EXIT bit in the process flags then this call will never return 867 * if 'return_instead' is false, but will exit. 868 * 869 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 870 *---------------+--------------------+--------------------- 871 * 0 | returns 0 | returns 0 or 1 872 * | when ST ends | immediately 873 *---------------+--------------------+--------------------- 874 * 1 | thread exits | returns 1 875 * | | immediately 876 * 0 = thread_exit() or suspension ok, 877 * other = return error instead of stopping the thread. 878 * 879 * While a full suspension is under effect, even a single threading 880 * thread would be suspended if it made this call (but it shouldn't). 881 * This call should only be made from places where 882 * thread_exit() would be safe as that may be the outcome unless 883 * return_instead is set. 884 */ 885 int 886 thread_suspend_check(int return_instead) 887 { 888 struct thread *td; 889 struct proc *p; 890 int wakeup_swapper; 891 892 td = curthread; 893 p = td->td_proc; 894 mtx_assert(&Giant, MA_NOTOWNED); 895 PROC_LOCK_ASSERT(p, MA_OWNED); 896 while (thread_suspend_check_needed()) { 897 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 898 KASSERT(p->p_singlethread != NULL, 899 ("singlethread not set")); 900 /* 901 * The only suspension in action is a 902 * single-threading. Single threader need not stop. 903 * XXX Should be safe to access unlocked 904 * as it can only be set to be true by us. 905 */ 906 if (p->p_singlethread == td) 907 return (0); /* Exempt from stopping. */ 908 } 909 if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 910 return (EINTR); 911 912 /* Should we goto user boundary if we didn't come from there? */ 913 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 914 (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 915 return (ERESTART); 916 917 /* 918 * Ignore suspend requests if they are deferred. 919 */ 920 if ((td->td_flags & TDF_SBDRY) != 0) { 921 KASSERT(return_instead, 922 ("TDF_SBDRY set for unsafe thread_suspend_check")); 923 return (0); 924 } 925 926 /* 927 * If the process is waiting for us to exit, 928 * this thread should just suicide. 929 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 930 */ 931 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 932 PROC_UNLOCK(p); 933 tidhash_remove(td); 934 935 /* 936 * Allow Linux emulation layer to do some work 937 * before thread suicide. 938 */ 939 if (__predict_false(p->p_sysent->sv_thread_detach != NULL)) 940 (p->p_sysent->sv_thread_detach)(td); 941 942 PROC_LOCK(p); 943 tdsigcleanup(td); 944 umtx_thread_exit(td); 945 PROC_SLOCK(p); 946 thread_stopped(p); 947 thread_exit(); 948 } 949 950 PROC_SLOCK(p); 951 thread_stopped(p); 952 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 953 if (p->p_numthreads == p->p_suspcount + 1) { 954 thread_lock(p->p_singlethread); 955 wakeup_swapper = thread_unsuspend_one( 956 p->p_singlethread, p, false); 957 thread_unlock(p->p_singlethread); 958 if (wakeup_swapper) 959 kick_proc0(); 960 } 961 } 962 PROC_UNLOCK(p); 963 thread_lock(td); 964 /* 965 * When a thread suspends, it just 966 * gets taken off all queues. 967 */ 968 thread_suspend_one(td); 969 if (return_instead == 0) { 970 p->p_boundary_count++; 971 td->td_flags |= TDF_BOUNDARY; 972 } 973 PROC_SUNLOCK(p); 974 mi_switch(SW_INVOL | SWT_SUSPEND, NULL); 975 thread_unlock(td); 976 PROC_LOCK(p); 977 } 978 return (0); 979 } 980 981 void 982 thread_suspend_switch(struct thread *td, struct proc *p) 983 { 984 985 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 986 PROC_LOCK_ASSERT(p, MA_OWNED); 987 PROC_SLOCK_ASSERT(p, MA_OWNED); 988 /* 989 * We implement thread_suspend_one in stages here to avoid 990 * dropping the proc lock while the thread lock is owned. 991 */ 992 if (p == td->td_proc) { 993 thread_stopped(p); 994 p->p_suspcount++; 995 } 996 PROC_UNLOCK(p); 997 thread_lock(td); 998 td->td_flags &= ~TDF_NEEDSUSPCHK; 999 TD_SET_SUSPENDED(td); 1000 sched_sleep(td, 0); 1001 PROC_SUNLOCK(p); 1002 DROP_GIANT(); 1003 mi_switch(SW_VOL | SWT_SUSPEND, NULL); 1004 thread_unlock(td); 1005 PICKUP_GIANT(); 1006 PROC_LOCK(p); 1007 PROC_SLOCK(p); 1008 } 1009 1010 void 1011 thread_suspend_one(struct thread *td) 1012 { 1013 struct proc *p; 1014 1015 p = td->td_proc; 1016 PROC_SLOCK_ASSERT(p, MA_OWNED); 1017 THREAD_LOCK_ASSERT(td, MA_OWNED); 1018 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 1019 p->p_suspcount++; 1020 td->td_flags &= ~TDF_NEEDSUSPCHK; 1021 TD_SET_SUSPENDED(td); 1022 sched_sleep(td, 0); 1023 } 1024 1025 static int 1026 thread_unsuspend_one(struct thread *td, struct proc *p, bool boundary) 1027 { 1028 1029 THREAD_LOCK_ASSERT(td, MA_OWNED); 1030 KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); 1031 TD_CLR_SUSPENDED(td); 1032 td->td_flags &= ~TDF_ALLPROCSUSP; 1033 if (td->td_proc == p) { 1034 PROC_SLOCK_ASSERT(p, MA_OWNED); 1035 p->p_suspcount--; 1036 if (boundary && (td->td_flags & TDF_BOUNDARY) != 0) { 1037 td->td_flags &= ~TDF_BOUNDARY; 1038 p->p_boundary_count--; 1039 } 1040 } 1041 return (setrunnable(td)); 1042 } 1043 1044 /* 1045 * Allow all threads blocked by single threading to continue running. 1046 */ 1047 void 1048 thread_unsuspend(struct proc *p) 1049 { 1050 struct thread *td; 1051 int wakeup_swapper; 1052 1053 PROC_LOCK_ASSERT(p, MA_OWNED); 1054 PROC_SLOCK_ASSERT(p, MA_OWNED); 1055 wakeup_swapper = 0; 1056 if (!P_SHOULDSTOP(p)) { 1057 FOREACH_THREAD_IN_PROC(p, td) { 1058 thread_lock(td); 1059 if (TD_IS_SUSPENDED(td)) { 1060 wakeup_swapper |= thread_unsuspend_one(td, p, 1061 true); 1062 } 1063 thread_unlock(td); 1064 } 1065 } else if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 1066 p->p_numthreads == p->p_suspcount) { 1067 /* 1068 * Stopping everything also did the job for the single 1069 * threading request. Now we've downgraded to single-threaded, 1070 * let it continue. 1071 */ 1072 if (p->p_singlethread->td_proc == p) { 1073 thread_lock(p->p_singlethread); 1074 wakeup_swapper = thread_unsuspend_one( 1075 p->p_singlethread, p, false); 1076 thread_unlock(p->p_singlethread); 1077 } 1078 } 1079 if (wakeup_swapper) 1080 kick_proc0(); 1081 } 1082 1083 /* 1084 * End the single threading mode.. 1085 */ 1086 void 1087 thread_single_end(struct proc *p, int mode) 1088 { 1089 struct thread *td; 1090 int wakeup_swapper; 1091 1092 KASSERT(mode == SINGLE_EXIT || mode == SINGLE_BOUNDARY || 1093 mode == SINGLE_ALLPROC || mode == SINGLE_NO_EXIT, 1094 ("invalid mode %d", mode)); 1095 PROC_LOCK_ASSERT(p, MA_OWNED); 1096 KASSERT((mode == SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) != 0) || 1097 (mode != SINGLE_ALLPROC && (p->p_flag & P_TOTAL_STOP) == 0), 1098 ("mode %d does not match P_TOTAL_STOP", mode)); 1099 KASSERT(mode == SINGLE_ALLPROC || p->p_singlethread == curthread, 1100 ("thread_single_end from other thread %p %p", 1101 curthread, p->p_singlethread)); 1102 KASSERT(mode != SINGLE_BOUNDARY || 1103 (p->p_flag & P_SINGLE_BOUNDARY) != 0, 1104 ("mis-matched SINGLE_BOUNDARY flags %x", p->p_flag)); 1105 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY | 1106 P_TOTAL_STOP); 1107 PROC_SLOCK(p); 1108 p->p_singlethread = NULL; 1109 wakeup_swapper = 0; 1110 /* 1111 * If there are other threads they may now run, 1112 * unless of course there is a blanket 'stop order' 1113 * on the process. The single threader must be allowed 1114 * to continue however as this is a bad place to stop. 1115 */ 1116 if (p->p_numthreads != remain_for_mode(mode) && !P_SHOULDSTOP(p)) { 1117 FOREACH_THREAD_IN_PROC(p, td) { 1118 thread_lock(td); 1119 if (TD_IS_SUSPENDED(td)) { 1120 wakeup_swapper |= thread_unsuspend_one(td, p, 1121 mode == SINGLE_BOUNDARY); 1122 } 1123 thread_unlock(td); 1124 } 1125 } 1126 KASSERT(mode != SINGLE_BOUNDARY || p->p_boundary_count == 0, 1127 ("inconsistent boundary count %d", p->p_boundary_count)); 1128 PROC_SUNLOCK(p); 1129 if (wakeup_swapper) 1130 kick_proc0(); 1131 } 1132 1133 struct thread * 1134 thread_find(struct proc *p, lwpid_t tid) 1135 { 1136 struct thread *td; 1137 1138 PROC_LOCK_ASSERT(p, MA_OWNED); 1139 FOREACH_THREAD_IN_PROC(p, td) { 1140 if (td->td_tid == tid) 1141 break; 1142 } 1143 return (td); 1144 } 1145 1146 /* Locate a thread by number; return with proc lock held. */ 1147 struct thread * 1148 tdfind(lwpid_t tid, pid_t pid) 1149 { 1150 #define RUN_THRESH 16 1151 struct thread *td; 1152 int run = 0; 1153 1154 rw_rlock(&tidhash_lock); 1155 LIST_FOREACH(td, TIDHASH(tid), td_hash) { 1156 if (td->td_tid == tid) { 1157 if (pid != -1 && td->td_proc->p_pid != pid) { 1158 td = NULL; 1159 break; 1160 } 1161 PROC_LOCK(td->td_proc); 1162 if (td->td_proc->p_state == PRS_NEW) { 1163 PROC_UNLOCK(td->td_proc); 1164 td = NULL; 1165 break; 1166 } 1167 if (run > RUN_THRESH) { 1168 if (rw_try_upgrade(&tidhash_lock)) { 1169 LIST_REMOVE(td, td_hash); 1170 LIST_INSERT_HEAD(TIDHASH(td->td_tid), 1171 td, td_hash); 1172 rw_wunlock(&tidhash_lock); 1173 return (td); 1174 } 1175 } 1176 break; 1177 } 1178 run++; 1179 } 1180 rw_runlock(&tidhash_lock); 1181 return (td); 1182 } 1183 1184 void 1185 tidhash_add(struct thread *td) 1186 { 1187 rw_wlock(&tidhash_lock); 1188 LIST_INSERT_HEAD(TIDHASH(td->td_tid), td, td_hash); 1189 rw_wunlock(&tidhash_lock); 1190 } 1191 1192 void 1193 tidhash_remove(struct thread *td) 1194 { 1195 rw_wlock(&tidhash_lock); 1196 LIST_REMOVE(td, td_hash); 1197 rw_wunlock(&tidhash_lock); 1198 } 1199