1 /*- 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/sched.h> 41 #include <sys/sleepqueue.h> 42 #include <sys/turnstile.h> 43 #include <sys/ktr.h> 44 #include <sys/umtx.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_extern.h> 48 #include <vm/uma.h> 49 50 /* 51 * KSEGRP related storage. 52 */ 53 static uma_zone_t ksegrp_zone; 54 static uma_zone_t thread_zone; 55 56 /* DEBUG ONLY */ 57 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 58 static int thread_debug = 0; 59 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 60 &thread_debug, 0, "thread debug"); 61 62 int max_threads_per_proc = 1500; 63 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 64 &max_threads_per_proc, 0, "Limit on threads per proc"); 65 66 int max_groups_per_proc = 1500; 67 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 68 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 69 70 int max_threads_hits; 71 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 72 &max_threads_hits, 0, ""); 73 74 int virtual_cpu; 75 76 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 77 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 78 struct mtx kse_zombie_lock; 79 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 80 81 static int 82 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 83 { 84 int error, new_val; 85 int def_val; 86 87 def_val = mp_ncpus; 88 if (virtual_cpu == 0) 89 new_val = def_val; 90 else 91 new_val = virtual_cpu; 92 error = sysctl_handle_int(oidp, &new_val, 0, req); 93 if (error != 0 || req->newptr == NULL) 94 return (error); 95 if (new_val < 0) 96 return (EINVAL); 97 virtual_cpu = new_val; 98 return (0); 99 } 100 101 /* DEBUG ONLY */ 102 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 103 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 104 "debug virtual cpus"); 105 106 /* 107 * Thread ID allocator. The allocator keeps track of assigned IDs by 108 * using a bitmap. The bitmap is created in parts. The parts are linked 109 * together. 110 */ 111 typedef u_long tid_bitmap_word; 112 113 #define TID_IDS_PER_PART 1024 114 #define TID_IDS_PER_IDX (sizeof(tid_bitmap_word) << 3) 115 #define TID_BITMAP_SIZE (TID_IDS_PER_PART / TID_IDS_PER_IDX) 116 #define TID_MIN (PID_MAX + 1) 117 118 struct tid_bitmap_part { 119 STAILQ_ENTRY(tid_bitmap_part) bmp_next; 120 tid_bitmap_word bmp_bitmap[TID_BITMAP_SIZE]; 121 lwpid_t bmp_base; 122 int bmp_free; 123 }; 124 125 static STAILQ_HEAD(, tid_bitmap_part) tid_bitmap = 126 STAILQ_HEAD_INITIALIZER(tid_bitmap); 127 static uma_zone_t tid_zone; 128 129 struct mtx tid_lock; 130 MTX_SYSINIT(tid_lock, &tid_lock, "TID lock", MTX_DEF); 131 132 /* 133 * Prepare a thread for use. 134 */ 135 static int 136 thread_ctor(void *mem, int size, void *arg, int flags) 137 { 138 struct thread *td; 139 140 td = (struct thread *)mem; 141 td->td_state = TDS_INACTIVE; 142 td->td_oncpu = NOCPU; 143 144 /* 145 * Note that td_critnest begins life as 1 because the thread is not 146 * running and is thereby implicitly waiting to be on the receiving 147 * end of a context switch. A context switch must occur inside a 148 * critical section, and in fact, includes hand-off of the sched_lock. 149 * After a context switch to a newly created thread, it will release 150 * sched_lock for the first time, and its td_critnest will hit 0 for 151 * the first time. This happens on the far end of a context switch, 152 * and when it context switches away from itself, it will in fact go 153 * back into a critical section, and hand off the sched lock to the 154 * next thread. 155 */ 156 td->td_critnest = 1; 157 return (0); 158 } 159 160 /* 161 * Reclaim a thread after use. 162 */ 163 static void 164 thread_dtor(void *mem, int size, void *arg) 165 { 166 struct thread *td; 167 168 td = (struct thread *)mem; 169 170 #ifdef INVARIANTS 171 /* Verify that this thread is in a safe state to free. */ 172 switch (td->td_state) { 173 case TDS_INHIBITED: 174 case TDS_RUNNING: 175 case TDS_CAN_RUN: 176 case TDS_RUNQ: 177 /* 178 * We must never unlink a thread that is in one of 179 * these states, because it is currently active. 180 */ 181 panic("bad state for thread unlinking"); 182 /* NOTREACHED */ 183 case TDS_INACTIVE: 184 break; 185 default: 186 panic("bad thread state"); 187 /* NOTREACHED */ 188 } 189 #endif 190 sched_newthread(td); 191 } 192 193 /* 194 * Initialize type-stable parts of a thread (when newly created). 195 */ 196 static int 197 thread_init(void *mem, int size, int flags) 198 { 199 struct thread *td; 200 struct tid_bitmap_part *bmp, *new; 201 int bit, idx; 202 203 td = (struct thread *)mem; 204 205 mtx_lock(&tid_lock); 206 STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { 207 if (bmp->bmp_free) 208 break; 209 } 210 /* Create a new bitmap if we run out of free bits. */ 211 if (bmp == NULL) { 212 mtx_unlock(&tid_lock); 213 new = uma_zalloc(tid_zone, M_WAITOK); 214 mtx_lock(&tid_lock); 215 bmp = STAILQ_LAST(&tid_bitmap, tid_bitmap_part, bmp_next); 216 if (bmp == NULL || bmp->bmp_free < TID_IDS_PER_PART/2) { 217 /* 1=free, 0=assigned. This way we can use ffsl(). */ 218 memset(new->bmp_bitmap, ~0U, sizeof(new->bmp_bitmap)); 219 new->bmp_base = (bmp == NULL) ? TID_MIN : 220 bmp->bmp_base + TID_IDS_PER_PART; 221 new->bmp_free = TID_IDS_PER_PART; 222 STAILQ_INSERT_TAIL(&tid_bitmap, new, bmp_next); 223 bmp = new; 224 new = NULL; 225 } 226 } else 227 new = NULL; 228 /* We have a bitmap with available IDs. */ 229 idx = 0; 230 while (idx < TID_BITMAP_SIZE && bmp->bmp_bitmap[idx] == 0UL) 231 idx++; 232 bit = ffsl(bmp->bmp_bitmap[idx]) - 1; 233 td->td_tid = bmp->bmp_base + idx * TID_IDS_PER_IDX + bit; 234 bmp->bmp_bitmap[idx] &= ~(1UL << bit); 235 bmp->bmp_free--; 236 mtx_unlock(&tid_lock); 237 if (new != NULL) 238 uma_zfree(tid_zone, new); 239 240 vm_thread_new(td, 0); 241 cpu_thread_setup(td); 242 td->td_sleepqueue = sleepq_alloc(); 243 td->td_turnstile = turnstile_alloc(); 244 td->td_umtxq = umtxq_alloc(); 245 td->td_sched = (struct td_sched *)&td[1]; 246 sched_newthread(td); 247 return (0); 248 } 249 250 /* 251 * Tear down type-stable parts of a thread (just before being discarded). 252 */ 253 static void 254 thread_fini(void *mem, int size) 255 { 256 struct thread *td; 257 struct tid_bitmap_part *bmp; 258 lwpid_t tid; 259 int bit, idx; 260 261 td = (struct thread *)mem; 262 turnstile_free(td->td_turnstile); 263 sleepq_free(td->td_sleepqueue); 264 umtxq_free(td->td_umtxq); 265 vm_thread_dispose(td); 266 267 STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { 268 if (td->td_tid >= bmp->bmp_base && 269 td->td_tid < bmp->bmp_base + TID_IDS_PER_PART) 270 break; 271 } 272 KASSERT(bmp != NULL, ("No TID bitmap?")); 273 mtx_lock(&tid_lock); 274 tid = td->td_tid - bmp->bmp_base; 275 idx = tid / TID_IDS_PER_IDX; 276 bit = 1UL << (tid % TID_IDS_PER_IDX); 277 bmp->bmp_bitmap[idx] |= bit; 278 bmp->bmp_free++; 279 mtx_unlock(&tid_lock); 280 } 281 282 /* 283 * Initialize type-stable parts of a ksegrp (when newly created). 284 */ 285 static int 286 ksegrp_ctor(void *mem, int size, void *arg, int flags) 287 { 288 struct ksegrp *kg; 289 290 kg = (struct ksegrp *)mem; 291 bzero(mem, size); 292 kg->kg_sched = (struct kg_sched *)&kg[1]; 293 return (0); 294 } 295 296 void 297 ksegrp_link(struct ksegrp *kg, struct proc *p) 298 { 299 300 TAILQ_INIT(&kg->kg_threads); 301 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 302 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 303 kg->kg_proc = p; 304 /* 305 * the following counters are in the -zero- section 306 * and may not need clearing 307 */ 308 kg->kg_numthreads = 0; 309 kg->kg_numupcalls = 0; 310 /* link it in now that it's consistent */ 311 p->p_numksegrps++; 312 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 313 } 314 315 /* 316 * Called from: 317 * thread-exit() 318 */ 319 void 320 ksegrp_unlink(struct ksegrp *kg) 321 { 322 struct proc *p; 323 324 mtx_assert(&sched_lock, MA_OWNED); 325 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 326 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 327 328 p = kg->kg_proc; 329 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 330 p->p_numksegrps--; 331 /* 332 * Aggregate stats from the KSE 333 */ 334 } 335 336 /* 337 * For a newly created process, 338 * link up all the structures and its initial threads etc. 339 * called from: 340 * {arch}/{arch}/machdep.c ia64_init(), init386() etc. 341 * proc_dtor() (should go away) 342 * proc_init() 343 */ 344 void 345 proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td) 346 { 347 348 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 349 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 350 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 351 p->p_numksegrps = 0; 352 p->p_numthreads = 0; 353 354 ksegrp_link(kg, p); 355 thread_link(td, kg); 356 } 357 358 /* 359 * Initialize global thread allocation resources. 360 */ 361 void 362 threadinit(void) 363 { 364 365 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 366 thread_ctor, thread_dtor, thread_init, thread_fini, 367 UMA_ALIGN_CACHE, 0); 368 tid_zone = uma_zcreate("TID", sizeof(struct tid_bitmap_part), 369 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 370 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 371 ksegrp_ctor, NULL, NULL, NULL, 372 UMA_ALIGN_CACHE, 0); 373 kseinit(); /* set up kse specific stuff e.g. upcall zone*/ 374 } 375 376 /* 377 * Stash an embarasingly extra thread into the zombie thread queue. 378 */ 379 void 380 thread_stash(struct thread *td) 381 { 382 mtx_lock_spin(&kse_zombie_lock); 383 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 384 mtx_unlock_spin(&kse_zombie_lock); 385 } 386 387 /* 388 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 389 */ 390 void 391 ksegrp_stash(struct ksegrp *kg) 392 { 393 mtx_lock_spin(&kse_zombie_lock); 394 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 395 mtx_unlock_spin(&kse_zombie_lock); 396 } 397 398 /* 399 * Reap zombie kse resource. 400 */ 401 void 402 thread_reap(void) 403 { 404 struct thread *td_first, *td_next; 405 struct ksegrp *kg_first, * kg_next; 406 407 /* 408 * Don't even bother to lock if none at this instant, 409 * we really don't care about the next instant.. 410 */ 411 if ((!TAILQ_EMPTY(&zombie_threads)) 412 || (!TAILQ_EMPTY(&zombie_ksegrps))) { 413 mtx_lock_spin(&kse_zombie_lock); 414 td_first = TAILQ_FIRST(&zombie_threads); 415 kg_first = TAILQ_FIRST(&zombie_ksegrps); 416 if (td_first) 417 TAILQ_INIT(&zombie_threads); 418 if (kg_first) 419 TAILQ_INIT(&zombie_ksegrps); 420 mtx_unlock_spin(&kse_zombie_lock); 421 while (td_first) { 422 td_next = TAILQ_NEXT(td_first, td_runq); 423 if (td_first->td_ucred) 424 crfree(td_first->td_ucred); 425 thread_free(td_first); 426 td_first = td_next; 427 } 428 while (kg_first) { 429 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 430 ksegrp_free(kg_first); 431 kg_first = kg_next; 432 } 433 /* 434 * there will always be a thread on the list if one of these 435 * is there. 436 */ 437 kse_GC(); 438 } 439 } 440 441 /* 442 * Allocate a ksegrp. 443 */ 444 struct ksegrp * 445 ksegrp_alloc(void) 446 { 447 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 448 } 449 450 /* 451 * Allocate a thread. 452 */ 453 struct thread * 454 thread_alloc(void) 455 { 456 thread_reap(); /* check if any zombies to get */ 457 return (uma_zalloc(thread_zone, M_WAITOK)); 458 } 459 460 /* 461 * Deallocate a ksegrp. 462 */ 463 void 464 ksegrp_free(struct ksegrp *td) 465 { 466 uma_zfree(ksegrp_zone, td); 467 } 468 469 /* 470 * Deallocate a thread. 471 */ 472 void 473 thread_free(struct thread *td) 474 { 475 476 cpu_thread_clean(td); 477 uma_zfree(thread_zone, td); 478 } 479 480 /* 481 * Discard the current thread and exit from its context. 482 * Always called with scheduler locked. 483 * 484 * Because we can't free a thread while we're operating under its context, 485 * push the current thread into our CPU's deadthread holder. This means 486 * we needn't worry about someone else grabbing our context before we 487 * do a cpu_throw(). This may not be needed now as we are under schedlock. 488 * Maybe we can just do a thread_stash() as thr_exit1 does. 489 */ 490 /* XXX 491 * libthr expects its thread exit to return for the last 492 * thread, meaning that the program is back to non-threaded 493 * mode I guess. Because we do this (cpu_throw) unconditionally 494 * here, they have their own version of it. (thr_exit1()) 495 * that doesn't do it all if this was the last thread. 496 * It is also called from thread_suspend_check(). 497 * Of course in the end, they end up coming here through exit1 498 * anyhow.. After fixing 'thr' to play by the rules we should be able 499 * to merge these two functions together. 500 * 501 * called from: 502 * exit1() 503 * kse_exit() 504 * thr_exit() 505 * thread_user_enter() 506 * thread_userret() 507 * thread_suspend_check() 508 */ 509 void 510 thread_exit(void) 511 { 512 struct thread *td; 513 struct proc *p; 514 struct ksegrp *kg; 515 516 td = curthread; 517 kg = td->td_ksegrp; 518 p = td->td_proc; 519 520 mtx_assert(&sched_lock, MA_OWNED); 521 mtx_assert(&Giant, MA_NOTOWNED); 522 PROC_LOCK_ASSERT(p, MA_OWNED); 523 KASSERT(p != NULL, ("thread exiting without a process")); 524 KASSERT(kg != NULL, ("thread exiting without a kse group")); 525 CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, 526 (long)p->p_pid, p->p_comm); 527 528 if (td->td_standin != NULL) { 529 /* 530 * Note that we don't need to free the cred here as it 531 * is done in thread_reap(). 532 */ 533 thread_stash(td->td_standin); 534 td->td_standin = NULL; 535 } 536 537 /* 538 * drop FPU & debug register state storage, or any other 539 * architecture specific resources that 540 * would not be on a new untouched process. 541 */ 542 cpu_thread_exit(td); /* XXXSMP */ 543 544 /* 545 * The thread is exiting. scheduler can release its stuff 546 * and collect stats etc. 547 */ 548 sched_thread_exit(td); 549 550 /* 551 * The last thread is left attached to the process 552 * So that the whole bundle gets recycled. Skip 553 * all this stuff if we never had threads. 554 * EXIT clears all sign of other threads when 555 * it goes to single threading, so the last thread always 556 * takes the short path. 557 */ 558 if (p->p_flag & P_HADTHREADS) { 559 if (p->p_numthreads > 1) { 560 thread_unlink(td); 561 562 /* XXX first arg not used in 4BSD or ULE */ 563 sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 564 565 /* 566 * as we are exiting there is room for another 567 * to be created. 568 */ 569 if (p->p_maxthrwaits) 570 wakeup(&p->p_numthreads); 571 572 /* 573 * The test below is NOT true if we are the 574 * sole exiting thread. P_STOPPED_SNGL is unset 575 * in exit1() after it is the only survivor. 576 */ 577 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 578 if (p->p_numthreads == p->p_suspcount) { 579 thread_unsuspend_one(p->p_singlethread); 580 } 581 } 582 583 /* 584 * Because each upcall structure has an owner thread, 585 * owner thread exits only when process is in exiting 586 * state, so upcall to userland is no longer needed, 587 * deleting upcall structure is safe here. 588 * So when all threads in a group is exited, all upcalls 589 * in the group should be automatically freed. 590 * XXXKSE This is a KSE thing and should be exported 591 * there somehow. 592 */ 593 upcall_remove(td); 594 595 /* 596 * If the thread we unlinked above was the last one, 597 * then this ksegrp should go away too. 598 */ 599 if (kg->kg_numthreads == 0) { 600 /* 601 * let the scheduler know about this in case 602 * it needs to recover stats or resources. 603 * Theoretically we could let 604 * sched_exit_ksegrp() do the equivalent of 605 * setting the concurrency to 0 606 * but don't do it yet to avoid changing 607 * the existing scheduler code until we 608 * are ready. 609 * We supply a random other ksegrp 610 * as the recipient of any built up 611 * cpu usage etc. (If the scheduler wants it). 612 * XXXKSE 613 * This is probably not fair so think of 614 * a better answer. 615 */ 616 sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); 617 sched_set_concurrency(kg, 0); /* XXX TEMP */ 618 ksegrp_unlink(kg); 619 ksegrp_stash(kg); 620 } 621 PROC_UNLOCK(p); 622 td->td_ksegrp = NULL; 623 PCPU_SET(deadthread, td); 624 } else { 625 /* 626 * The last thread is exiting.. but not through exit() 627 * what should we do? 628 * Theoretically this can't happen 629 * exit1() - clears threading flags before coming here 630 * kse_exit() - treats last thread specially 631 * thr_exit() - treats last thread specially 632 * thread_user_enter() - only if more exist 633 * thread_userret() - only if more exist 634 * thread_suspend_check() - only if more exist 635 */ 636 panic ("thread_exit: Last thread exiting on its own"); 637 } 638 } else { 639 /* 640 * non threaded process comes here. 641 * This includes an EX threaded process that is coming 642 * here via exit1(). (exit1 dethreads the proc first). 643 */ 644 PROC_UNLOCK(p); 645 } 646 td->td_state = TDS_INACTIVE; 647 CTR1(KTR_PROC, "thread_exit: cpu_throw() thread %p", td); 648 cpu_throw(td, choosethread()); 649 panic("I'm a teapot!"); 650 /* NOTREACHED */ 651 } 652 653 /* 654 * Do any thread specific cleanups that may be needed in wait() 655 * called with Giant, proc and schedlock not held. 656 */ 657 void 658 thread_wait(struct proc *p) 659 { 660 struct thread *td; 661 662 mtx_assert(&Giant, MA_NOTOWNED); 663 KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 664 KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); 665 FOREACH_THREAD_IN_PROC(p, td) { 666 if (td->td_standin != NULL) { 667 crfree(td->td_ucred); 668 td->td_ucred = NULL; 669 thread_free(td->td_standin); 670 td->td_standin = NULL; 671 } 672 cpu_thread_clean(td); 673 crfree(td->td_ucred); 674 } 675 thread_reap(); /* check for zombie threads etc. */ 676 } 677 678 /* 679 * Link a thread to a process. 680 * set up anything that needs to be initialized for it to 681 * be used by the process. 682 * 683 * Note that we do not link to the proc's ucred here. 684 * The thread is linked as if running but no KSE assigned. 685 * Called from: 686 * proc_linkup() 687 * thread_schedule_upcall() 688 * thr_create() 689 */ 690 void 691 thread_link(struct thread *td, struct ksegrp *kg) 692 { 693 struct proc *p; 694 695 p = kg->kg_proc; 696 td->td_state = TDS_INACTIVE; 697 td->td_proc = p; 698 td->td_ksegrp = kg; 699 td->td_flags = 0; 700 td->td_kflags = 0; 701 702 LIST_INIT(&td->td_contested); 703 callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 704 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 705 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 706 p->p_numthreads++; 707 kg->kg_numthreads++; 708 } 709 710 /* 711 * Convert a process with one thread to an unthreaded process. 712 * Called from: 713 * thread_single(exit) (called from execve and exit) 714 * kse_exit() XXX may need cleaning up wrt KSE stuff 715 */ 716 void 717 thread_unthread(struct thread *td) 718 { 719 struct proc *p = td->td_proc; 720 721 KASSERT((p->p_numthreads == 1), ("Unthreading with >1 threads")); 722 upcall_remove(td); 723 p->p_flag &= ~(P_SA|P_HADTHREADS); 724 td->td_mailbox = NULL; 725 td->td_pflags &= ~(TDP_SA | TDP_CAN_UNBIND); 726 if (td->td_standin != NULL) { 727 thread_stash(td->td_standin); 728 td->td_standin = NULL; 729 } 730 sched_set_concurrency(td->td_ksegrp, 1); 731 } 732 733 /* 734 * Called from: 735 * thread_exit() 736 */ 737 void 738 thread_unlink(struct thread *td) 739 { 740 struct proc *p = td->td_proc; 741 struct ksegrp *kg = td->td_ksegrp; 742 743 mtx_assert(&sched_lock, MA_OWNED); 744 TAILQ_REMOVE(&p->p_threads, td, td_plist); 745 p->p_numthreads--; 746 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 747 kg->kg_numthreads--; 748 /* could clear a few other things here */ 749 /* Must NOT clear links to proc and ksegrp! */ 750 } 751 752 /* 753 * Enforce single-threading. 754 * 755 * Returns 1 if the caller must abort (another thread is waiting to 756 * exit the process or similar). Process is locked! 757 * Returns 0 when you are successfully the only thread running. 758 * A process has successfully single threaded in the suspend mode when 759 * There are no threads in user mode. Threads in the kernel must be 760 * allowed to continue until they get to the user boundary. They may even 761 * copy out their return values and data before suspending. They may however be 762 * accellerated in reaching the user boundary as we will wake up 763 * any sleeping threads that are interruptable. (PCATCH). 764 */ 765 int 766 thread_single(int mode) 767 { 768 struct thread *td; 769 struct thread *td2; 770 struct proc *p; 771 int remaining; 772 773 td = curthread; 774 p = td->td_proc; 775 mtx_assert(&Giant, MA_NOTOWNED); 776 PROC_LOCK_ASSERT(p, MA_OWNED); 777 KASSERT((td != NULL), ("curthread is NULL")); 778 779 if ((p->p_flag & P_HADTHREADS) == 0) 780 return (0); 781 782 /* Is someone already single threading? */ 783 if (p->p_singlethread != NULL && p->p_singlethread != td) 784 return (1); 785 786 if (mode == SINGLE_EXIT) { 787 p->p_flag |= P_SINGLE_EXIT; 788 p->p_flag &= ~P_SINGLE_BOUNDARY; 789 } else { 790 p->p_flag &= ~P_SINGLE_EXIT; 791 if (mode == SINGLE_BOUNDARY) 792 p->p_flag |= P_SINGLE_BOUNDARY; 793 else 794 p->p_flag &= ~P_SINGLE_BOUNDARY; 795 } 796 p->p_flag |= P_STOPPED_SINGLE; 797 mtx_lock_spin(&sched_lock); 798 p->p_singlethread = td; 799 if (mode == SINGLE_EXIT) 800 remaining = p->p_numthreads; 801 else if (mode == SINGLE_BOUNDARY) 802 remaining = p->p_numthreads - p->p_boundary_count; 803 else 804 remaining = p->p_numthreads - p->p_suspcount; 805 while (remaining != 1) { 806 FOREACH_THREAD_IN_PROC(p, td2) { 807 if (td2 == td) 808 continue; 809 td2->td_flags |= TDF_ASTPENDING; 810 if (TD_IS_INHIBITED(td2)) { 811 switch (mode) { 812 case SINGLE_EXIT: 813 if (td->td_flags & TDF_DBSUSPEND) 814 td->td_flags &= ~TDF_DBSUSPEND; 815 if (TD_IS_SUSPENDED(td2)) 816 thread_unsuspend_one(td2); 817 if (TD_ON_SLEEPQ(td2) && 818 (td2->td_flags & TDF_SINTR)) 819 sleepq_abort(td2); 820 break; 821 case SINGLE_BOUNDARY: 822 if (TD_IS_SUSPENDED(td2) && 823 !(td2->td_flags & TDF_BOUNDARY)) 824 thread_unsuspend_one(td2); 825 if (TD_ON_SLEEPQ(td2) && 826 (td2->td_flags & TDF_SINTR)) 827 sleepq_abort(td2); 828 break; 829 default: 830 if (TD_IS_SUSPENDED(td2)) 831 continue; 832 /* 833 * maybe other inhibitted states too? 834 */ 835 if ((td2->td_flags & TDF_SINTR) && 836 (td2->td_inhibitors & 837 (TDI_SLEEPING | TDI_SWAPPED))) 838 thread_suspend_one(td2); 839 break; 840 } 841 } 842 } 843 if (mode == SINGLE_EXIT) 844 remaining = p->p_numthreads; 845 else if (mode == SINGLE_BOUNDARY) 846 remaining = p->p_numthreads - p->p_boundary_count; 847 else 848 remaining = p->p_numthreads - p->p_suspcount; 849 850 /* 851 * Maybe we suspended some threads.. was it enough? 852 */ 853 if (remaining == 1) 854 break; 855 856 /* 857 * Wake us up when everyone else has suspended. 858 * In the mean time we suspend as well. 859 */ 860 thread_suspend_one(td); 861 PROC_UNLOCK(p); 862 mi_switch(SW_VOL, NULL); 863 mtx_unlock_spin(&sched_lock); 864 PROC_LOCK(p); 865 mtx_lock_spin(&sched_lock); 866 if (mode == SINGLE_EXIT) 867 remaining = p->p_numthreads; 868 else if (mode == SINGLE_BOUNDARY) 869 remaining = p->p_numthreads - p->p_boundary_count; 870 else 871 remaining = p->p_numthreads - p->p_suspcount; 872 } 873 if (mode == SINGLE_EXIT) { 874 /* 875 * We have gotten rid of all the other threads and we 876 * are about to either exit or exec. In either case, 877 * we try our utmost to revert to being a non-threaded 878 * process. 879 */ 880 p->p_singlethread = NULL; 881 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 882 thread_unthread(td); 883 } 884 mtx_unlock_spin(&sched_lock); 885 return (0); 886 } 887 888 /* 889 * Called in from locations that can safely check to see 890 * whether we have to suspend or at least throttle for a 891 * single-thread event (e.g. fork). 892 * 893 * Such locations include userret(). 894 * If the "return_instead" argument is non zero, the thread must be able to 895 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 896 * 897 * The 'return_instead' argument tells the function if it may do a 898 * thread_exit() or suspend, or whether the caller must abort and back 899 * out instead. 900 * 901 * If the thread that set the single_threading request has set the 902 * P_SINGLE_EXIT bit in the process flags then this call will never return 903 * if 'return_instead' is false, but will exit. 904 * 905 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 906 *---------------+--------------------+--------------------- 907 * 0 | returns 0 | returns 0 or 1 908 * | when ST ends | immediatly 909 *---------------+--------------------+--------------------- 910 * 1 | thread exits | returns 1 911 * | | immediatly 912 * 0 = thread_exit() or suspension ok, 913 * other = return error instead of stopping the thread. 914 * 915 * While a full suspension is under effect, even a single threading 916 * thread would be suspended if it made this call (but it shouldn't). 917 * This call should only be made from places where 918 * thread_exit() would be safe as that may be the outcome unless 919 * return_instead is set. 920 */ 921 int 922 thread_suspend_check(int return_instead) 923 { 924 struct thread *td; 925 struct proc *p; 926 927 td = curthread; 928 p = td->td_proc; 929 mtx_assert(&Giant, MA_NOTOWNED); 930 PROC_LOCK_ASSERT(p, MA_OWNED); 931 while (P_SHOULDSTOP(p) || 932 ((p->p_flag & P_TRACED) && (td->td_flags & TDF_DBSUSPEND))) { 933 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 934 KASSERT(p->p_singlethread != NULL, 935 ("singlethread not set")); 936 /* 937 * The only suspension in action is a 938 * single-threading. Single threader need not stop. 939 * XXX Should be safe to access unlocked 940 * as it can only be set to be true by us. 941 */ 942 if (p->p_singlethread == td) 943 return (0); /* Exempt from stopping. */ 944 } 945 if ((p->p_flag & P_SINGLE_EXIT) && return_instead) 946 return (1); 947 948 /* Should we goto user boundary if we didn't come from there? */ 949 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE && 950 (p->p_flag & P_SINGLE_BOUNDARY) && return_instead) 951 return (1); 952 953 mtx_lock_spin(&sched_lock); 954 thread_stopped(p); 955 /* 956 * If the process is waiting for us to exit, 957 * this thread should just suicide. 958 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 959 */ 960 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) 961 thread_exit(); 962 963 /* 964 * When a thread suspends, it just 965 * moves to the processes's suspend queue 966 * and stays there. 967 */ 968 thread_suspend_one(td); 969 if (return_instead == 0) { 970 p->p_boundary_count++; 971 td->td_flags |= TDF_BOUNDARY; 972 } 973 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 974 if (p->p_numthreads == p->p_suspcount) 975 thread_unsuspend_one(p->p_singlethread); 976 } 977 PROC_UNLOCK(p); 978 mi_switch(SW_INVOL, NULL); 979 if (return_instead == 0) { 980 p->p_boundary_count--; 981 td->td_flags &= ~TDF_BOUNDARY; 982 } 983 mtx_unlock_spin(&sched_lock); 984 PROC_LOCK(p); 985 } 986 return (0); 987 } 988 989 void 990 thread_suspend_one(struct thread *td) 991 { 992 struct proc *p = td->td_proc; 993 994 mtx_assert(&sched_lock, MA_OWNED); 995 PROC_LOCK_ASSERT(p, MA_OWNED); 996 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 997 p->p_suspcount++; 998 TD_SET_SUSPENDED(td); 999 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1000 /* 1001 * Hack: If we are suspending but are on the sleep queue 1002 * then we are in msleep or the cv equivalent. We 1003 * want to look like we have two Inhibitors. 1004 * May already be set.. doesn't matter. 1005 */ 1006 if (TD_ON_SLEEPQ(td)) 1007 TD_SET_SLEEPING(td); 1008 } 1009 1010 void 1011 thread_unsuspend_one(struct thread *td) 1012 { 1013 struct proc *p = td->td_proc; 1014 1015 mtx_assert(&sched_lock, MA_OWNED); 1016 PROC_LOCK_ASSERT(p, MA_OWNED); 1017 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 1018 TD_CLR_SUSPENDED(td); 1019 p->p_suspcount--; 1020 setrunnable(td); 1021 } 1022 1023 /* 1024 * Allow all threads blocked by single threading to continue running. 1025 */ 1026 void 1027 thread_unsuspend(struct proc *p) 1028 { 1029 struct thread *td; 1030 1031 mtx_assert(&sched_lock, MA_OWNED); 1032 PROC_LOCK_ASSERT(p, MA_OWNED); 1033 if (!P_SHOULDSTOP(p)) { 1034 while ((td = TAILQ_FIRST(&p->p_suspended))) { 1035 thread_unsuspend_one(td); 1036 } 1037 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 1038 (p->p_numthreads == p->p_suspcount)) { 1039 /* 1040 * Stopping everything also did the job for the single 1041 * threading request. Now we've downgraded to single-threaded, 1042 * let it continue. 1043 */ 1044 thread_unsuspend_one(p->p_singlethread); 1045 } 1046 } 1047 1048 /* 1049 * End the single threading mode.. 1050 */ 1051 void 1052 thread_single_end(void) 1053 { 1054 struct thread *td; 1055 struct proc *p; 1056 1057 td = curthread; 1058 p = td->td_proc; 1059 PROC_LOCK_ASSERT(p, MA_OWNED); 1060 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT | P_SINGLE_BOUNDARY); 1061 mtx_lock_spin(&sched_lock); 1062 p->p_singlethread = NULL; 1063 /* 1064 * If there are other threads they mey now run, 1065 * unless of course there is a blanket 'stop order' 1066 * on the process. The single threader must be allowed 1067 * to continue however as this is a bad place to stop. 1068 */ 1069 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 1070 while ((td = TAILQ_FIRST(&p->p_suspended))) { 1071 thread_unsuspend_one(td); 1072 } 1073 } 1074 mtx_unlock_spin(&sched_lock); 1075 } 1076 1077 /* 1078 * Called before going into an interruptible sleep to see if we have been 1079 * interrupted or requested to exit. 1080 */ 1081 int 1082 thread_sleep_check(struct thread *td) 1083 { 1084 struct proc *p; 1085 1086 p = td->td_proc; 1087 mtx_assert(&sched_lock, MA_OWNED); 1088 if (p->p_flag & P_HADTHREADS) { 1089 if (p->p_singlethread != td) { 1090 if (p->p_flag & P_SINGLE_EXIT) 1091 return (EINTR); 1092 if (p->p_flag & P_SINGLE_BOUNDARY) 1093 return (ERESTART); 1094 } 1095 if (td->td_flags & TDF_INTERRUPT) 1096 return (td->td_intrval); 1097 } 1098 return (0); 1099 } 1100