1 /*- 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/resourcevar.h> 39 #include <sys/smp.h> 40 #include <sys/sysctl.h> 41 #include <sys/sched.h> 42 #include <sys/sleepqueue.h> 43 #include <sys/turnstile.h> 44 #include <sys/ktr.h> 45 #include <sys/umtx.h> 46 47 #include <security/audit/audit.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_extern.h> 51 #include <vm/uma.h> 52 53 /* 54 * KSEGRP related storage. 55 */ 56 static uma_zone_t ksegrp_zone; 57 static uma_zone_t thread_zone; 58 59 /* DEBUG ONLY */ 60 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 61 static int thread_debug = 0; 62 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 63 &thread_debug, 0, "thread debug"); 64 65 int max_threads_per_proc = 1500; 66 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 67 &max_threads_per_proc, 0, "Limit on threads per proc"); 68 69 int max_groups_per_proc = 1500; 70 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 71 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 72 73 int max_threads_hits; 74 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 75 &max_threads_hits, 0, ""); 76 77 int virtual_cpu; 78 79 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 80 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 81 struct mtx kse_zombie_lock; 82 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 83 84 static int 85 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 86 { 87 int error, new_val; 88 int def_val; 89 90 def_val = mp_ncpus; 91 if (virtual_cpu == 0) 92 new_val = def_val; 93 else 94 new_val = virtual_cpu; 95 error = sysctl_handle_int(oidp, &new_val, 0, req); 96 if (error != 0 || req->newptr == NULL) 97 return (error); 98 if (new_val < 0) 99 return (EINVAL); 100 virtual_cpu = new_val; 101 return (0); 102 } 103 104 /* DEBUG ONLY */ 105 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 106 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 107 "debug virtual cpus"); 108 109 struct mtx tid_lock; 110 static struct unrhdr *tid_unrhdr; 111 112 /* 113 * Prepare a thread for use. 114 */ 115 static int 116 thread_ctor(void *mem, int size, void *arg, int flags) 117 { 118 struct thread *td; 119 120 td = (struct thread *)mem; 121 td->td_state = TDS_INACTIVE; 122 td->td_oncpu = NOCPU; 123 124 td->td_tid = alloc_unr(tid_unrhdr); 125 126 /* 127 * Note that td_critnest begins life as 1 because the thread is not 128 * running and is thereby implicitly waiting to be on the receiving 129 * end of a context switch. A context switch must occur inside a 130 * critical section, and in fact, includes hand-off of the sched_lock. 131 * After a context switch to a newly created thread, it will release 132 * sched_lock for the first time, and its td_critnest will hit 0 for 133 * the first time. This happens on the far end of a context switch, 134 * and when it context switches away from itself, it will in fact go 135 * back into a critical section, and hand off the sched lock to the 136 * next thread. 137 */ 138 td->td_critnest = 1; 139 140 #ifdef AUDIT 141 audit_thread_alloc(td); 142 #endif 143 umtx_thread_alloc(td); 144 return (0); 145 } 146 147 /* 148 * Reclaim a thread after use. 149 */ 150 static void 151 thread_dtor(void *mem, int size, void *arg) 152 { 153 struct thread *td; 154 155 td = (struct thread *)mem; 156 157 #ifdef INVARIANTS 158 /* Verify that this thread is in a safe state to free. */ 159 switch (td->td_state) { 160 case TDS_INHIBITED: 161 case TDS_RUNNING: 162 case TDS_CAN_RUN: 163 case TDS_RUNQ: 164 /* 165 * We must never unlink a thread that is in one of 166 * these states, because it is currently active. 167 */ 168 panic("bad state for thread unlinking"); 169 /* NOTREACHED */ 170 case TDS_INACTIVE: 171 break; 172 default: 173 panic("bad thread state"); 174 /* NOTREACHED */ 175 } 176 #endif 177 #ifdef AUDIT 178 audit_thread_free(td); 179 #endif 180 free_unr(tid_unrhdr, td->td_tid); 181 sched_newthread(td); 182 } 183 184 /* 185 * Initialize type-stable parts of a thread (when newly created). 186 */ 187 static int 188 thread_init(void *mem, int size, int flags) 189 { 190 struct thread *td; 191 192 td = (struct thread *)mem; 193 194 vm_thread_new(td, 0); 195 cpu_thread_setup(td); 196 td->td_sleepqueue = sleepq_alloc(); 197 td->td_turnstile = turnstile_alloc(); 198 td->td_sched = (struct td_sched *)&td[1]; 199 sched_newthread(td); 200 umtx_thread_init(td); 201 return (0); 202 } 203 204 /* 205 * Tear down type-stable parts of a thread (just before being discarded). 206 */ 207 static void 208 thread_fini(void *mem, int size) 209 { 210 struct thread *td; 211 212 td = (struct thread *)mem; 213 turnstile_free(td->td_turnstile); 214 sleepq_free(td->td_sleepqueue); 215 umtx_thread_fini(td); 216 vm_thread_dispose(td); 217 } 218 219 /* 220 * Initialize type-stable parts of a ksegrp (when newly created). 221 */ 222 static int 223 ksegrp_ctor(void *mem, int size, void *arg, int flags) 224 { 225 struct ksegrp *kg; 226 227 kg = (struct ksegrp *)mem; 228 bzero(mem, size); 229 kg->kg_sched = (struct kg_sched *)&kg[1]; 230 return (0); 231 } 232 233 void 234 ksegrp_link(struct ksegrp *kg, struct proc *p) 235 { 236 237 TAILQ_INIT(&kg->kg_threads); 238 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 239 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 240 kg->kg_proc = p; 241 /* 242 * the following counters are in the -zero- section 243 * and may not need clearing 244 */ 245 kg->kg_numthreads = 0; 246 kg->kg_numupcalls = 0; 247 /* link it in now that it's consistent */ 248 p->p_numksegrps++; 249 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 250 } 251 252 /* 253 * Called from: 254 * thread-exit() 255 */ 256 void 257 ksegrp_unlink(struct ksegrp *kg) 258 { 259 struct proc *p; 260 261 mtx_assert(&sched_lock, MA_OWNED); 262 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 263 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 264 265 p = kg->kg_proc; 266 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 267 p->p_numksegrps--; 268 /* 269 * Aggregate stats from the KSE 270 */ 271 if (p->p_procscopegrp == kg) 272 p->p_procscopegrp = NULL; 273 } 274 275 /* 276 * For a newly created process, 277 * link up all the structures and its initial threads etc. 278 * called from: 279 * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 280 * proc_dtor() (should go away) 281 * proc_init() 282 */ 283 void 284 proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td) 285 { 286 287 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 288 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 289 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 290 sigqueue_init(&p->p_sigqueue, p); 291 p->p_ksi = ksiginfo_alloc(1); 292 if (p->p_ksi != NULL) { 293 /* XXX p_ksi may be null if ksiginfo zone is not ready */ 294 p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; 295 } 296 LIST_INIT(&p->p_mqnotifier); 297 p->p_numksegrps = 0; 298 p->p_numthreads = 0; 299 300 ksegrp_link(kg, p); 301 thread_link(td, kg); 302 } 303 304 /* 305 * Initialize global thread allocation resources. 306 */ 307 void 308 threadinit(void) 309 { 310 311 mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); 312 tid_unrhdr = new_unrhdr(PID_MAX + 1, INT_MAX, &tid_lock); 313 314 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 315 thread_ctor, thread_dtor, thread_init, thread_fini, 316 UMA_ALIGN_CACHE, 0); 317 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 318 ksegrp_ctor, NULL, NULL, NULL, 319 UMA_ALIGN_CACHE, 0); 320 kseinit(); /* set up kse specific stuff e.g. upcall zone*/ 321 } 322 323 /* 324 * Stash an embarasingly extra thread into the zombie thread queue. 325 */ 326 void 327 thread_stash(struct thread *td) 328 { 329 mtx_lock_spin(&kse_zombie_lock); 330 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 331 mtx_unlock_spin(&kse_zombie_lock); 332 } 333 334 /* 335 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 336 */ 337 void 338 ksegrp_stash(struct ksegrp *kg) 339 { 340 mtx_lock_spin(&kse_zombie_lock); 341 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 342 mtx_unlock_spin(&kse_zombie_lock); 343 } 344 345 /* 346 * Reap zombie kse resource. 347 */ 348 void 349 thread_reap(void) 350 { 351 struct thread *td_first, *td_next; 352 struct ksegrp *kg_first, * kg_next; 353 354 /* 355 * Don't even bother to lock if none at this instant, 356 * we really don't care about the next instant.. 357 */ 358 if ((!TAILQ_EMPTY(&zombie_threads)) 359 || (!TAILQ_EMPTY(&zombie_ksegrps))) { 360 mtx_lock_spin(&kse_zombie_lock); 361 td_first = TAILQ_FIRST(&zombie_threads); 362 kg_first = TAILQ_FIRST(&zombie_ksegrps); 363 if (td_first) 364 TAILQ_INIT(&zombie_threads); 365 if (kg_first) 366 TAILQ_INIT(&zombie_ksegrps); 367 mtx_unlock_spin(&kse_zombie_lock); 368 while (td_first) { 369 td_next = TAILQ_NEXT(td_first, td_runq); 370 if (td_first->td_ucred) 371 crfree(td_first->td_ucred); 372 thread_free(td_first); 373 td_first = td_next; 374 } 375 while (kg_first) { 376 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 377 ksegrp_free(kg_first); 378 kg_first = kg_next; 379 } 380 /* 381 * there will always be a thread on the list if one of these 382 * is there. 383 */ 384 kse_GC(); 385 } 386 } 387 388 /* 389 * Allocate a ksegrp. 390 */ 391 struct ksegrp * 392 ksegrp_alloc(void) 393 { 394 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 395 } 396 397 /* 398 * Allocate a thread. 399 */ 400 struct thread * 401 thread_alloc(void) 402 { 403 thread_reap(); /* check if any zombies to get */ 404 return (uma_zalloc(thread_zone, M_WAITOK)); 405 } 406 407 /* 408 * Deallocate a ksegrp. 409 */ 410 void 411 ksegrp_free(struct ksegrp *td) 412 { 413 uma_zfree(ksegrp_zone, td); 414 } 415 416 /* 417 * Deallocate a thread. 418 */ 419 void 420 thread_free(struct thread *td) 421 { 422 423 cpu_thread_clean(td); 424 uma_zfree(thread_zone, td); 425 } 426 427 /* 428 * Discard the current thread and exit from its context. 429 * Always called with scheduler locked. 430 * 431 * Because we can't free a thread while we're operating under its context, 432 * push the current thread into our CPU's deadthread holder. This means 433 * we needn't worry about someone else grabbing our context before we 434 * do a cpu_throw(). This may not be needed now as we are under schedlock. 435 * Maybe we can just do a thread_stash() as thr_exit1 does. 436 */ 437 /* XXX 438 * libthr expects its thread exit to return for the last 439 * thread, meaning that the program is back to non-threaded 440 * mode I guess. Because we do this (cpu_throw) unconditionally 441 * here, they have their own version of it. (thr_exit1()) 442 * that doesn't do it all if this was the last thread. 443 * It is also called from thread_suspend_check(). 444 * Of course in the end, they end up coming here through exit1 445 * anyhow.. After fixing 'thr' to play by the rules we should be able 446 * to merge these two functions together. 447 * 448 * called from: 449 * exit1() 450 * kse_exit() 451 * thr_exit() 452 * thread_user_enter() 453 * thread_userret() 454 * thread_suspend_check() 455 */ 456 void 457 thread_exit(void) 458 { 459 uint64_t new_switchtime; 460 struct thread *td; 461 struct proc *p; 462 struct ksegrp *kg; 463 464 td = curthread; 465 kg = td->td_ksegrp; 466 p = td->td_proc; 467 468 mtx_assert(&sched_lock, MA_OWNED); 469 mtx_assert(&Giant, MA_NOTOWNED); 470 PROC_LOCK_ASSERT(p, MA_OWNED); 471 KASSERT(p != NULL, ("thread exiting without a process")); 472 KASSERT(kg != NULL, ("thread exiting without a kse group")); 473 CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 474 (long)p->p_pid, p->p_comm); 475 KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); 476 477 #ifdef AUDIT 478 AUDIT_SYSCALL_EXIT(0, td); 479 #endif 480 481 if (td->td_standin != NULL) { 482 /* 483 * Note that we don't need to free the cred here as it 484 * is done in thread_reap(). 485 */ 486 thread_stash(td->td_standin); 487 td->td_standin = NULL; 488 } 489 490 umtx_thread_exit(td); 491 492 /* 493 * drop FPU & debug register state storage, or any other 494 * architecture specific resources that 495 * would not be on a new untouched process. 496 */ 497 cpu_thread_exit(td); /* XXXSMP */ 498 499 /* 500 * The thread is exiting. scheduler can release its stuff 501 * and collect stats etc. 502 * XXX this is not very right, since PROC_UNLOCK may still 503 * need scheduler stuff. 504 */ 505 sched_thread_exit(td); 506 507 /* Do the same timestamp bookkeeping that mi_switch() would do. */ 508 new_switchtime = cpu_ticks(); 509 p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime)); 510 p->p_rux.rux_uticks += td->td_uticks; 511 p->p_rux.rux_sticks += td->td_sticks; 512 p->p_rux.rux_iticks += td->td_iticks; 513 PCPU_SET(switchtime, new_switchtime); 514 PCPU_SET(switchticks, ticks); 515 cnt.v_swtch++; 516 517 /* Add our usage into the usage of all our children. */ 518 if (p->p_numthreads == 1) 519 ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux); 520 521 /* 522 * The last thread is left attached to the process 523 * So that the whole bundle gets recycled. Skip 524 * all this stuff if we never had threads. 525 * EXIT clears all sign of other threads when 526 * it goes to single threading, so the last thread always 527 * takes the short path. 528 */ 529 if (p->p_flag & P_HADTHREADS) { 530 if (p->p_numthreads > 1) { 531 thread_unlink(td); 532 533 /* XXX first arg not used in 4BSD or ULE */ 534 sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 535 536 /* 537 * The test below is NOT true if we are the 538 * sole exiting thread. P_STOPPED_SNGL is unset 539 * in exit1() after it is the only survivor. 540 */ 541 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 542 if (p->p_numthreads == p->p_suspcount) { 543 thread_unsuspend_one(p->p_singlethread); 544 } 545 } 546 547 /* 548 * Because each upcall structure has an owner thread, 549 * owner thread exits only when process is in exiting 550 * state, so upcall to userland is no longer needed, 551 * deleting upcall structure is safe here. 552 * So when all threads in a group is exited, all upcalls 553 * in the group should be automatically freed. 554 * XXXKSE This is a KSE thing and should be exported 555 * there somehow. 556 */ 557 upcall_remove(td); 558 559 /* 560 * If the thread we unlinked above was the last one, 561 * then this ksegrp should go away too. 562 */ 563 if (kg->kg_numthreads == 0) { 564 /* 565 * let the scheduler know about this in case 566 * it needs to recover stats or resources. 567 * Theoretically we could let 568 * sched_exit_ksegrp() do the equivalent of 569 * setting the concurrency to 0 570 * but don't do it yet to avoid changing 571 * the existing scheduler code until we 572 * are ready. 573 * We supply a random other ksegrp 574 * as the recipient of any built up 575 * cpu usage etc. (If the scheduler wants it). 576 * XXXKSE 577 * This is probably not fair so think of 578 * a better answer. 579 */ 580 sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); 581 sched_set_concurrency(kg, 0); /* XXX TEMP */ 582 ksegrp_unlink(kg); 583 ksegrp_stash(kg); 584 } 585 PROC_UNLOCK(p); 586 td->td_ksegrp = NULL; 587 PCPU_SET(deadthread, td); 588 } else { 589 /* 590 * The last thread is exiting.. but not through exit() 591 * what should we do? 592 * Theoretically this can't happen 593 * exit1() - clears threading flags before coming here 594 * kse_exit() - treats last thread specially 595 * thr_exit() - treats last thread specially 596 * thread_user_enter() - only if more exist 597 * thread_userret() - only if more exist 598 * thread_suspend_check() - only if more exist 599 */ 600 panic ("thread_exit: Last thread exiting on its own"); 601 } 602 } else { 603 /* 604 * non threaded process comes here. 605 * This includes an EX threaded process that is coming 606 * here via exit1(). (exit1 dethreads the proc first). 607 */ 608 PROC_UNLOCK(p); 609 } 610 td->td_state = TDS_INACTIVE; 611 CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 612 cpu_throw(td, choosethread()); 613 panic("I'm a teapot!"); 614 /* NOTREACHED */ 615 } 616 617 /* 618 * Do any thread specific cleanups that may be needed in wait() 619 * called with Giant, proc and schedlock not held. 620 */ 621 void 622 thread_wait(struct proc *p) 623 { 624 struct thread *td; 625 626 mtx_assert(&Giant, MA_NOTOWNED); 627 KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 628 KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); 629 FOREACH_THREAD_IN_PROC(p, td) { 630 if (td->td_standin != NULL) { 631 if (td->td_standin->td_ucred != NULL) { 632 crfree(td->td_standin->td_ucred); 633 td->td_standin->td_ucred = NULL; 634 } 635 thread_free(td->td_standin); 636 td->td_standin = NULL; 637 } 638 cpu_thread_clean(td); 639 crfree(td->td_ucred); 640 } 641 thread_reap(); /* check for zombie threads etc. */ 642 } 643 644 /* 645 * Link a thread to a process. 646 * set up anything that needs to be initialized for it to 647 * be used by the process. 648 * 649 * Note that we do not link to the proc's ucred here. 650 * The thread is linked as if running but no KSE assigned. 651 * Called from: 652 * proc_linkup() 653 * thread_schedule_upcall() 654 * thr_create() 655 */ 656 void 657 thread_link(struct thread *td, struct ksegrp *kg) 658 { 659 struct proc *p; 660 661 p = kg->kg_proc; 662 td->td_state = TDS_INACTIVE; 663 td->td_proc = p; 664 td->td_ksegrp = kg; 665 td->td_flags = 0; 666 td->td_kflags = 0; 667 668 LIST_INIT(&td->td_contested); 669 sigqueue_init(&td->td_sigqueue, p); 670 callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 671 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 672 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 673 p->p_numthreads++; 674 kg->kg_numthreads++; 675 } 676 677 /* 678 * Convert a process with one thread to an unthreaded process. 679 * Called from: 680 * thread_single(exit) (called from execve and exit) 681 * kse_exit() XXX may need cleaning up wrt KSE stuff 682 */ 683 void 684 thread_unthread(struct thread *td) 685 { 686 struct proc *p = td->td_proc; 687 688 KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 689 upcall_remove(td); 690 p->p_flag &= ~(P_SA|P_HADTHREADS); 691 td->td_mailbox = NULL; 692 td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND); 693 if (td->td_standin != NULL) { 694 thread_stash(td->td_standin); 695 td->td_standin = NULL; 696 } 697 sched_set_concurrency(td->td_ksegrp, 1); 698 } 699 700 /* 701 * Called from: 702 * thread_exit() 703 */ 704 void 705 thread_unlink(struct thread *td) 706 { 707 struct proc *p = td->td_proc; 708 struct ksegrp *kg = td->td_ksegrp; 709 710 mtx_assert(&sched_lock, MA_OWNED); 711 TAILQ_REMOVE(&p->p_threads, td, td_plist); 712 p->p_numthreads--; 713 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 714 kg->kg_numthreads--; 715 /* could clear a few other things here */ 716 /* Must NOT clear links to proc and ksegrp! */ 717 } 718 719 /* 720 * Enforce single-threading. 721 * 722 * Returns 1 if the caller must abort (another thread is waiting to 723 * exit the process or similar). Process is locked! 724 * Returns 0 when you are successfully the only thread running. 725 * A process has successfully single threaded in the suspend mode when 726 * There are no threads in user mode. Threads in the kernel must be 727 * allowed to continue until they get to the user boundary. They may even 728 * copy out their return values and data before suspending. They may however be 729 * accelerated in reaching the user boundary as we will wake up 730 * any sleeping threads that are interruptable. (PCATCH). 731 */ 732 int 733 thread_single(int mode) 734 { 735 struct thread *td; 736 struct thread *td2; 737 struct proc *p; 738 int remaining; 739 740 td = curthread; 741 p = td->td_proc; 742 mtx_assert(&Giant, MA_NOTOWNED); 743 PROC_LOCK_ASSERT(p, MA_OWNED); 744 KASSERT((td != NULL), ("curthread is NULL")); 745 746 if ((p->p_flag & P_HADTHREADS) == 0) 747 return (0); 748 749 /* Is someone already single threading? */ 750 if (p->p_singlethread != NULL && p->p_singlethread != td) 751 return (1); 752 753 if (mode == SINGLE_EXIT) { 754 p->p_flag |= P_SINGLE_EXIT; 755 p->p_flag &= ~P_SINGLE_BOUNDARY; 756 } else { 757 p->p_flag &= ~P_SINGLE_EXIT; 758 if (mode == SINGLE_BOUNDARY) 759 p->p_flag |= P_SINGLE_BOUNDARY; 760 else 761 p->p_flag &= ~P_SINGLE_BOUNDARY; 762 } 763 p->p_flag |= P_STOPPED_SINGLE; 764 mtx_lock_spin(&sched_lock); 765 p->p_singlethread = td; 766 if (mode == SINGLE_EXIT) 767 remaining = p->p_numthreads; 768 else if (mode == SINGLE_BOUNDARY) 769 remaining = p->p_numthreads - p->p_boundary_count; 770 else 771 remaining = p->p_numthreads - p->p_suspcount; 772 while (remaining != 1) { 773 if (P_SHOULDSTOP(p) != P_STOPPED_SINGLE) 774 goto stopme; 775 FOREACH_THREAD_IN_PROC(p, td2) { 776 if (td2 == td) 777 continue; 778 td2->td_flags |= TDF_ASTPENDING; 779 if (TD_IS_INHIBITED(td2)) { 780 switch (mode) { 781 case SINGLE_EXIT: 782 if (td->td_flags & TDF_DBSUSPEND) 783 td->td_flags &= ~TDF_DBSUSPEND; 784 if (TD_IS_SUSPENDED(td2)) 785 thread_unsuspend_one(td2); 786 if (TD_ON_SLEEPQ(td2) && 787 (td2->td_flags & TDF_SINTR)) 788 sleepq_abort(td2, EINTR); 789 break; 790 case SINGLE_BOUNDARY: 791 if (TD_IS_SUSPENDED(td2) && 792 !(td2->td_flags & TDF_BOUNDARY)) 793 thread_unsuspend_one(td2); 794 if (TD_ON_SLEEPQ(td2) && 795 (td2->td_flags & TDF_SINTR)) 796 sleepq_abort(td2, ERESTART); 797 break; 798 default: 799 if (TD_IS_SUSPENDED(td2)) 800 continue; 801 /* 802 * maybe other inhibitted states too? 803 */ 804 if ((td2->td_flags & TDF_SINTR) && 805 (td2->td_inhibitors & 806 (TDI_SLEEPING | TDI_SWAPPED))) 807 thread_suspend_one(td2); 808 break; 809 } 810 } 811 #ifdef SMP 812 else if (TD_IS_RUNNING(td2) && td != td2) { 813 forward_signal(td2); 814 } 815 #endif 816 } 817 if (mode == SINGLE_EXIT) 818 remaining = p->p_numthreads; 819 else if (mode == SINGLE_BOUNDARY) 820 remaining = p->p_numthreads - p->p_boundary_count; 821 else 822 remaining = p->p_numthreads - p->p_suspcount; 823 824 /* 825 * Maybe we suspended some threads.. was it enough? 826 */ 827 if (remaining == 1) 828 break; 829 830 stopme: 831 /* 832 * Wake us up when everyone else has suspended. 833 * In the mean time we suspend as well. 834 */ 835 thread_stopped(p); 836 thread_suspend_one(td); 837 PROC_UNLOCK(p); 838 mi_switch(SW_VOL, NULL); 839 mtx_unlock_spin(&sched_lock); 840 PROC_LOCK(p); 841 mtx_lock_spin(&sched_lock); 842 if (mode == SINGLE_EXIT) 843 remaining = p->p_numthreads; 844 else if (mode == SINGLE_BOUNDARY) 845 remaining = p->p_numthreads - p->p_boundary_count; 846 else 847 remaining = p->p_numthreads - p->p_suspcount; 848 } 849 if (mode == SINGLE_EXIT) { 850 /* 851 * We have gotten rid of all the other threads and we 852 * are about to either exit or exec. In either case, 853 * we try our utmost to revert to being a non-threaded 854 * process. 855 */ 856 p->p_singlethread = NULL; 857 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 858 thread_unthread(td); 859 } 860 mtx_unlock_spin(&sched_lock); 861 return (0); 862 } 863 864 /* 865 * Called in from locations that can safely check to see 866 * whether we have to suspend or at least throttle for a 867 * single-thread event (e.g. fork). 868 * 869 * Such locations include userret(). 870 * If the "return_instead" argument is non zero, the thread must be able to 871 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 872 * 873 * The 'return_instead' argument tells the function if it may do a 874 * thread_exit() or suspend, or whether the caller must abort and back 875 * out instead. 876 * 877 * If the thread that set the single_threading request has set the 878 * P_SINGLE_EXIT bit in the process flags then this call will never return 879 * if 'return_instead' is false, but will exit. 880 * 881 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 882 *---------------+--------------------+--------------------- 883 * 0 | returns 0 | returns 0 or 1 884 * | when ST ends | immediatly 885 *---------------+--------------------+--------------------- 886 * 1 | thread exits | returns 1 887 * | | immediatly 888 * 0 = thread_exit() or suspension ok, 889 * other = return error instead of stopping the thread. 890 * 891 * While a full suspension is under effect, even a single threading 892 * thread would be suspended if it made this call (but it shouldn't). 893 * This call should only be made from places where 894 * thread_exit() would be safe as that may be the outcome unless 895 * return_instead is set. 896 */ 897 int 898 thread_suspend_check(int return_instead) 899 { 900 struct thread *td; 901 struct proc *p; 902 903 td = curthread; 904 p = td->td_proc; 905 mtx_assert(&Giant, MA_NOTOWNED); 906 PROC_LOCK_ASSERT(p, MA_OWNED); 907 while (P_SHOULDSTOP(p) || 908 ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { 909 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 910 KASSERT(p->p_singlethread != NULL, 911 ("singlethread not set")); 912 /* 913 * The only suspension in action is a 914 * single-threading. Single threader need not stop. 915 * XXX Should be safe to access unlocked 916 * as it can only be set to be true by us. 917 */ 918 if (p->p_singlethread == td) 919 return (0); /* Exempt from stopping. */ 920 } 921 if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 922 return (EINTR); 923 924 /* Should we goto user boundary if we didn't come from there? */ 925 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 926 (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 927 return (ERESTART); 928 929 /* If thread will exit, flush its pending signals */ 930 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 931 sigqueue_flush(&td->td_sigqueue); 932 933 mtx_lock_spin(&sched_lock); 934 thread_stopped(p); 935 /* 936 * If the process is waiting for us to exit, 937 * this thread should just suicide. 938 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 939 */ 940 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 941 thread_exit(); 942 943 /* 944 * When a thread suspends, it just 945 * moves to the processes's suspend queue 946 * and stays there. 947 */ 948 thread_suspend_one(td); 949 if (return_instead == 0) { 950 p->p_boundary_count++; 951 td->td_flags |= TDF_BOUNDARY; 952 } 953 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 954 if (p->p_numthreads == p->p_suspcount) 955 thread_unsuspend_one(p->p_singlethread); 956 } 957 PROC_UNLOCK(p); 958 mi_switch(SW_INVOL, NULL); 959 if (return_instead == 0) { 960 p->p_boundary_count--; 961 td->td_flags &= ~TDF_BOUNDARY; 962 } 963 mtx_unlock_spin(&sched_lock); 964 PROC_LOCK(p); 965 } 966 return (0); 967 } 968 969 void 970 thread_suspend_one(struct thread *td) 971 { 972 struct proc *p = td->td_proc; 973 974 mtx_assert(&sched_lock, MA_OWNED); 975 PROC_LOCK_ASSERT(p, MA_OWNED); 976 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 977 p->p_suspcount++; 978 TD_SET_SUSPENDED(td); 979 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 980 } 981 982 void 983 thread_unsuspend_one(struct thread *td) 984 { 985 struct proc *p = td->td_proc; 986 987 mtx_assert(&sched_lock, MA_OWNED); 988 PROC_LOCK_ASSERT(p, MA_OWNED); 989 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 990 TD_CLR_SUSPENDED(td); 991 p->p_suspcount--; 992 setrunnable(td); 993 } 994 995 /* 996 * Allow all threads blocked by single threading to continue running. 997 */ 998 void 999 thread_unsuspend(struct proc *p) 1000 { 1001 struct thread *td; 1002 1003 mtx_assert(&sched_lock, MA_OWNED); 1004 PROC_LOCK_ASSERT(p, MA_OWNED); 1005 if (!P_SHOULDSTOP(p)) { 1006 while ((td = TAILQ_FIRST(&p->p_suspended))) { 1007 thread_unsuspend_one(td); 1008 } 1009 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 1010 (p->p_numthreads == p->p_suspcount)) { 1011 /* 1012 * Stopping everything also did the job for the single 1013 * threading request. Now we've downgraded to single-threaded, 1014 * let it continue. 1015 */ 1016 thread_unsuspend_one(p->p_singlethread); 1017 } 1018 } 1019 1020 /* 1021 * End the single threading mode.. 1022 */ 1023 void 1024 thread_single_end(void) 1025 { 1026 struct thread *td; 1027 struct proc *p; 1028 1029 td = curthread; 1030 p = td->td_proc; 1031 PROC_LOCK_ASSERT(p, MA_OWNED); 1032 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 1033 mtx_lock_spin(&sched_lock); 1034 p->p_singlethread = NULL; 1035 p->p_procscopegrp = NULL; 1036 /* 1037 * If there are other threads they mey now run, 1038 * unless of course there is a blanket 'stop order' 1039 * on the process. The single threader must be allowed 1040 * to continue however as this is a bad place to stop. 1041 */ 1042 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 1043 while ((td = TAILQ_FIRST(&p->p_suspended))) { 1044 thread_unsuspend_one(td); 1045 } 1046 } 1047 mtx_unlock_spin(&sched_lock); 1048 } 1049 1050 struct thread * 1051 thread_find(struct proc *p, lwpid_t tid) 1052 { 1053 struct thread *td; 1054 1055 PROC_LOCK_ASSERT(p, MA_OWNED); 1056 mtx_lock_spin(&sched_lock); 1057 FOREACH_THREAD_IN_PROC(p, td) { 1058 if (td->td_tid == tid) 1059 break; 1060 } 1061 mtx_unlock_spin(&sched_lock); 1062 return (td); 1063 } 1064