1 /* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/lock.h> 35 #include <sys/malloc.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/sysproto.h> 41 #include <sys/filedesc.h> 42 #include <sys/sched.h> 43 #include <sys/signalvar.h> 44 #include <sys/sx.h> 45 #include <sys/tty.h> 46 #include <sys/user.h> 47 #include <sys/jail.h> 48 #include <sys/kse.h> 49 #include <sys/ktr.h> 50 #include <sys/ucontext.h> 51 52 #include <vm/vm.h> 53 #include <vm/vm_object.h> 54 #include <vm/pmap.h> 55 #include <vm/uma.h> 56 #include <vm/vm_map.h> 57 58 #include <machine/frame.h> 59 60 /* 61 * KSEGRP related storage. 62 */ 63 static uma_zone_t ksegrp_zone; 64 static uma_zone_t kse_zone; 65 static uma_zone_t thread_zone; 66 static uma_zone_t upcall_zone; 67 68 /* DEBUG ONLY */ 69 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 70 static int thread_debug = 0; 71 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 72 &thread_debug, 0, "thread debug"); 73 74 static int max_threads_per_proc = 30; 75 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 76 &max_threads_per_proc, 0, "Limit on threads per proc"); 77 78 static int max_groups_per_proc = 5; 79 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 80 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 81 82 static int max_threads_hits; 83 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 84 &max_threads_hits, 0, ""); 85 86 static int virtual_cpu; 87 88 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 89 90 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 91 TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 92 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 93 TAILQ_HEAD(, kse_upcall) zombie_upcalls = 94 TAILQ_HEAD_INITIALIZER(zombie_upcalls); 95 struct mtx kse_zombie_lock; 96 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 97 98 static void kse_purge(struct proc *p, struct thread *td); 99 static void kse_purge_group(struct thread *td); 100 static int thread_update_usr_ticks(struct thread *td, int user); 101 static void thread_alloc_spare(struct thread *td, struct thread *spare); 102 103 static int 104 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 105 { 106 int error, new_val; 107 int def_val; 108 109 #ifdef SMP 110 def_val = mp_ncpus; 111 #else 112 def_val = 1; 113 #endif 114 if (virtual_cpu == 0) 115 new_val = def_val; 116 else 117 new_val = virtual_cpu; 118 error = sysctl_handle_int(oidp, &new_val, 0, req); 119 if (error != 0 || req->newptr == NULL) 120 return (error); 121 if (new_val < 0) 122 return (EINVAL); 123 virtual_cpu = new_val; 124 return (0); 125 } 126 127 /* DEBUG ONLY */ 128 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 129 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 130 "debug virtual cpus"); 131 132 /* 133 * Prepare a thread for use. 134 */ 135 static void 136 thread_ctor(void *mem, int size, void *arg) 137 { 138 struct thread *td; 139 140 td = (struct thread *)mem; 141 td->td_state = TDS_INACTIVE; 142 td->td_oncpu = NOCPU; 143 } 144 145 /* 146 * Reclaim a thread after use. 147 */ 148 static void 149 thread_dtor(void *mem, int size, void *arg) 150 { 151 struct thread *td; 152 153 td = (struct thread *)mem; 154 155 #ifdef INVARIANTS 156 /* Verify that this thread is in a safe state to free. */ 157 switch (td->td_state) { 158 case TDS_INHIBITED: 159 case TDS_RUNNING: 160 case TDS_CAN_RUN: 161 case TDS_RUNQ: 162 /* 163 * We must never unlink a thread that is in one of 164 * these states, because it is currently active. 165 */ 166 panic("bad state for thread unlinking"); 167 /* NOTREACHED */ 168 case TDS_INACTIVE: 169 break; 170 default: 171 panic("bad thread state"); 172 /* NOTREACHED */ 173 } 174 #endif 175 } 176 177 /* 178 * Initialize type-stable parts of a thread (when newly created). 179 */ 180 static void 181 thread_init(void *mem, int size) 182 { 183 struct thread *td; 184 185 td = (struct thread *)mem; 186 mtx_lock(&Giant); 187 pmap_new_thread(td, 0); 188 mtx_unlock(&Giant); 189 cpu_thread_setup(td); 190 td->td_sched = (struct td_sched *)&td[1]; 191 } 192 193 /* 194 * Tear down type-stable parts of a thread (just before being discarded). 195 */ 196 static void 197 thread_fini(void *mem, int size) 198 { 199 struct thread *td; 200 201 td = (struct thread *)mem; 202 pmap_dispose_thread(td); 203 } 204 205 /* 206 * Initialize type-stable parts of a kse (when newly created). 207 */ 208 static void 209 kse_init(void *mem, int size) 210 { 211 struct kse *ke; 212 213 ke = (struct kse *)mem; 214 ke->ke_sched = (struct ke_sched *)&ke[1]; 215 } 216 217 /* 218 * Initialize type-stable parts of a ksegrp (when newly created). 219 */ 220 static void 221 ksegrp_init(void *mem, int size) 222 { 223 struct ksegrp *kg; 224 225 kg = (struct ksegrp *)mem; 226 kg->kg_sched = (struct kg_sched *)&kg[1]; 227 } 228 229 /* 230 * KSE is linked into kse group. 231 */ 232 void 233 kse_link(struct kse *ke, struct ksegrp *kg) 234 { 235 struct proc *p = kg->kg_proc; 236 237 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 238 kg->kg_kses++; 239 ke->ke_state = KES_UNQUEUED; 240 ke->ke_proc = p; 241 ke->ke_ksegrp = kg; 242 ke->ke_thread = NULL; 243 ke->ke_oncpu = NOCPU; 244 ke->ke_flags = 0; 245 } 246 247 void 248 kse_unlink(struct kse *ke) 249 { 250 struct ksegrp *kg; 251 252 mtx_assert(&sched_lock, MA_OWNED); 253 kg = ke->ke_ksegrp; 254 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 255 if (ke->ke_state == KES_IDLE) { 256 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 257 kg->kg_idle_kses--; 258 } 259 if (--kg->kg_kses == 0) 260 ksegrp_unlink(kg); 261 /* 262 * Aggregate stats from the KSE 263 */ 264 kse_stash(ke); 265 } 266 267 void 268 ksegrp_link(struct ksegrp *kg, struct proc *p) 269 { 270 271 TAILQ_INIT(&kg->kg_threads); 272 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 273 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 274 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 275 TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 276 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 277 kg->kg_proc = p; 278 /* 279 * the following counters are in the -zero- section 280 * and may not need clearing 281 */ 282 kg->kg_numthreads = 0; 283 kg->kg_runnable = 0; 284 kg->kg_kses = 0; 285 kg->kg_runq_kses = 0; /* XXXKSE change name */ 286 kg->kg_idle_kses = 0; 287 kg->kg_numupcalls = 0; 288 /* link it in now that it's consistent */ 289 p->p_numksegrps++; 290 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 291 } 292 293 void 294 ksegrp_unlink(struct ksegrp *kg) 295 { 296 struct proc *p; 297 298 mtx_assert(&sched_lock, MA_OWNED); 299 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 300 KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 301 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 302 303 p = kg->kg_proc; 304 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 305 p->p_numksegrps--; 306 /* 307 * Aggregate stats from the KSE 308 */ 309 ksegrp_stash(kg); 310 } 311 312 struct kse_upcall * 313 upcall_alloc(void) 314 { 315 struct kse_upcall *ku; 316 317 ku = uma_zalloc(upcall_zone, M_WAITOK); 318 bzero(ku, sizeof(*ku)); 319 return (ku); 320 } 321 322 void 323 upcall_free(struct kse_upcall *ku) 324 { 325 326 uma_zfree(upcall_zone, ku); 327 } 328 329 void 330 upcall_link(struct kse_upcall *ku, struct ksegrp *kg) 331 { 332 333 mtx_assert(&sched_lock, MA_OWNED); 334 TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 335 ku->ku_ksegrp = kg; 336 kg->kg_numupcalls++; 337 } 338 339 void 340 upcall_unlink(struct kse_upcall *ku) 341 { 342 struct ksegrp *kg = ku->ku_ksegrp; 343 344 mtx_assert(&sched_lock, MA_OWNED); 345 KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 346 TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 347 kg->kg_numupcalls--; 348 upcall_stash(ku); 349 } 350 351 void 352 upcall_remove(struct thread *td) 353 { 354 355 if (td->td_upcall) { 356 td->td_upcall->ku_owner = NULL; 357 upcall_unlink(td->td_upcall); 358 td->td_upcall = 0; 359 } 360 } 361 362 /* 363 * For a newly created process, 364 * link up all the structures and its initial threads etc. 365 */ 366 void 367 proc_linkup(struct proc *p, struct ksegrp *kg, 368 struct kse *ke, struct thread *td) 369 { 370 371 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 372 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 373 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 374 p->p_numksegrps = 0; 375 p->p_numthreads = 0; 376 377 ksegrp_link(kg, p); 378 kse_link(ke, kg); 379 thread_link(td, kg); 380 } 381 382 /* 383 struct kse_thr_interrupt_args { 384 struct kse_thr_mailbox * tmbx; 385 }; 386 */ 387 int 388 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 389 { 390 struct proc *p; 391 struct thread *td2; 392 393 p = td->td_proc; 394 if (!(p->p_flag & P_THREADED) || (uap->tmbx == NULL)) 395 return (EINVAL); 396 mtx_lock_spin(&sched_lock); 397 FOREACH_THREAD_IN_PROC(p, td2) { 398 if (td2->td_mailbox == uap->tmbx) { 399 td2->td_flags |= TDF_INTERRUPT; 400 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 401 if (td2->td_flags & TDF_CVWAITQ) 402 cv_abort(td2); 403 else 404 abortsleep(td2); 405 } 406 mtx_unlock_spin(&sched_lock); 407 return (0); 408 } 409 } 410 mtx_unlock_spin(&sched_lock); 411 return (ESRCH); 412 } 413 414 /* 415 struct kse_exit_args { 416 register_t dummy; 417 }; 418 */ 419 int 420 kse_exit(struct thread *td, struct kse_exit_args *uap) 421 { 422 struct proc *p; 423 struct ksegrp *kg; 424 struct kse *ke; 425 426 p = td->td_proc; 427 /* 428 * Only UTS can call the syscall and current group 429 * should be a threaded group. 430 */ 431 if ((td->td_mailbox != NULL) || (td->td_ksegrp->kg_numupcalls == 0)) 432 return (EINVAL); 433 KASSERT((td->td_upcall != NULL), ("%s: not own an upcall", __func__)); 434 435 kg = td->td_ksegrp; 436 /* Serialize removing upcall */ 437 PROC_LOCK(p); 438 mtx_lock_spin(&sched_lock); 439 if ((kg->kg_numupcalls == 1) && (kg->kg_numthreads > 1)) { 440 mtx_unlock_spin(&sched_lock); 441 PROC_UNLOCK(p); 442 return (EDEADLK); 443 } 444 ke = td->td_kse; 445 upcall_remove(td); 446 if (p->p_numthreads == 1) { 447 kse_purge(p, td); 448 p->p_flag &= ~P_THREADED; 449 mtx_unlock_spin(&sched_lock); 450 PROC_UNLOCK(p); 451 } else { 452 if (kg->kg_numthreads == 1) { /* Shutdown a group */ 453 kse_purge_group(td); 454 ke->ke_flags |= KEF_EXIT; 455 } 456 thread_stopped(p); 457 thread_exit(); 458 /* NOTREACHED */ 459 } 460 return (0); 461 } 462 463 /* 464 * Either becomes an upcall or waits for an awakening event and 465 * then becomes an upcall. Only error cases return. 466 */ 467 /* 468 struct kse_release_args { 469 struct timespec *timeout; 470 }; 471 */ 472 int 473 kse_release(struct thread *td, struct kse_release_args *uap) 474 { 475 struct proc *p; 476 struct ksegrp *kg; 477 struct timespec ts, ts2, ts3, timeout; 478 struct timeval tv; 479 int error; 480 481 p = td->td_proc; 482 kg = td->td_ksegrp; 483 /* 484 * Only UTS can call the syscall and current group 485 * should be a threaded group. 486 */ 487 if ((td->td_mailbox != NULL) || (td->td_ksegrp->kg_numupcalls == 0)) 488 return (EINVAL); 489 KASSERT((td->td_upcall != NULL), ("%s: not own an upcall", __func__)); 490 if (uap->timeout != NULL) { 491 if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 492 return (error); 493 getnanouptime(&ts); 494 timespecadd(&ts, &timeout); 495 TIMESPEC_TO_TIMEVAL(&tv, &timeout); 496 } 497 mtx_lock_spin(&sched_lock); 498 /* Change OURSELF to become an upcall. */ 499 td->td_flags = TDF_UPCALLING; 500 #if 0 /* XXX This shouldn't be necessary */ 501 if (p->p_sflag & PS_NEEDSIGCHK) 502 td->td_flags |= TDF_ASTPENDING; 503 #endif 504 mtx_unlock_spin(&sched_lock); 505 PROC_LOCK(p); 506 while ((td->td_upcall->ku_flags & KUF_DOUPCALL) == 0 && 507 (kg->kg_completed == NULL)) { 508 kg->kg_upsleeps++; 509 error = msleep(&kg->kg_completed, &p->p_mtx, PPAUSE|PCATCH, 510 "kse_rel", (uap->timeout ? tvtohz(&tv) : 0)); 511 kg->kg_upsleeps--; 512 PROC_UNLOCK(p); 513 if (uap->timeout == NULL || error != EWOULDBLOCK) 514 return (0); 515 getnanouptime(&ts2); 516 if (timespeccmp(&ts2, &ts, >=)) 517 return (0); 518 ts3 = ts; 519 timespecsub(&ts3, &ts2); 520 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 521 PROC_LOCK(p); 522 } 523 PROC_UNLOCK(p); 524 return (0); 525 } 526 527 /* struct kse_wakeup_args { 528 struct kse_mailbox *mbx; 529 }; */ 530 int 531 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 532 { 533 struct proc *p; 534 struct ksegrp *kg; 535 struct kse_upcall *ku; 536 struct thread *td2; 537 538 p = td->td_proc; 539 td2 = NULL; 540 ku = NULL; 541 /* KSE-enabled processes only, please. */ 542 if (!(p->p_flag & P_THREADED)) 543 return (EINVAL); 544 PROC_LOCK(p); 545 mtx_lock_spin(&sched_lock); 546 if (uap->mbx) { 547 FOREACH_KSEGRP_IN_PROC(p, kg) { 548 FOREACH_UPCALL_IN_GROUP(kg, ku) { 549 if (ku->ku_mailbox == uap->mbx) 550 break; 551 } 552 if (ku) 553 break; 554 } 555 } else { 556 kg = td->td_ksegrp; 557 if (kg->kg_upsleeps) { 558 wakeup_one(&kg->kg_completed); 559 mtx_unlock_spin(&sched_lock); 560 PROC_UNLOCK(p); 561 return (0); 562 } 563 ku = TAILQ_FIRST(&kg->kg_upcalls); 564 } 565 if (ku) { 566 if ((td2 = ku->ku_owner) == NULL) { 567 panic("%s: no owner", __func__); 568 } else if (TD_ON_SLEEPQ(td2) && 569 (td2->td_wchan == &kg->kg_completed)) { 570 abortsleep(td2); 571 } else { 572 ku->ku_flags |= KUF_DOUPCALL; 573 } 574 mtx_unlock_spin(&sched_lock); 575 PROC_UNLOCK(p); 576 return (0); 577 } 578 mtx_unlock_spin(&sched_lock); 579 PROC_UNLOCK(p); 580 return (ESRCH); 581 } 582 583 /* 584 * No new KSEG: first call: use current KSE, don't schedule an upcall 585 * All other situations, do allocate max new KSEs and schedule an upcall. 586 */ 587 /* struct kse_create_args { 588 struct kse_mailbox *mbx; 589 int newgroup; 590 }; */ 591 int 592 kse_create(struct thread *td, struct kse_create_args *uap) 593 { 594 struct kse *newke; 595 struct ksegrp *newkg; 596 struct ksegrp *kg; 597 struct proc *p; 598 struct kse_mailbox mbx; 599 struct kse_upcall *newku; 600 int err, ncpus; 601 602 p = td->td_proc; 603 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 604 return (err); 605 606 /* Too bad, why hasn't kernel always a cpu counter !? */ 607 #ifdef SMP 608 ncpus = mp_ncpus; 609 #else 610 ncpus = 1; 611 #endif 612 if (thread_debug && virtual_cpu != 0) 613 ncpus = virtual_cpu; 614 615 /* Easier to just set it than to test and set */ 616 PROC_LOCK(p); 617 p->p_flag |= P_THREADED; 618 PROC_UNLOCK(p); 619 kg = td->td_ksegrp; 620 if (uap->newgroup) { 621 /* Have race condition but it is cheap */ 622 if (p->p_numksegrps >= max_groups_per_proc) 623 return (EPROCLIM); 624 /* 625 * If we want a new KSEGRP it doesn't matter whether 626 * we have already fired up KSE mode before or not. 627 * We put the process in KSE mode and create a new KSEGRP. 628 */ 629 newkg = ksegrp_alloc(); 630 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 631 kg_startzero, kg_endzero)); 632 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 633 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 634 mtx_lock_spin(&sched_lock); 635 if (p->p_numksegrps >= max_groups_per_proc) { 636 mtx_unlock_spin(&sched_lock); 637 ksegrp_free(newkg); 638 return (EPROCLIM); 639 } 640 ksegrp_link(newkg, p); 641 mtx_unlock_spin(&sched_lock); 642 } else { 643 newkg = kg; 644 } 645 646 /* 647 * Creating upcalls more than number of physical cpu does 648 * not help performance. 649 */ 650 if (newkg->kg_numupcalls >= ncpus) 651 return (EPROCLIM); 652 653 if (newkg->kg_numupcalls == 0) { 654 /* 655 * Initialize KSE group, optimized for MP. 656 * Create KSEs as many as physical cpus, this increases 657 * concurrent even if userland is not MP safe and can only run 658 * on single CPU (for early version of libpthread, it is true). 659 * In ideal world, every physical cpu should execute a thread. 660 * If there is enough KSEs, threads in kernel can be 661 * executed parallel on different cpus with full speed, 662 * Concurrent in kernel shouldn't be restricted by number of 663 * upcalls userland provides. 664 * Adding more upcall structures only increases concurrent 665 * in userland. 666 * Highest performance configuration is: 667 * N kses = N upcalls = N phyiscal cpus 668 */ 669 while (newkg->kg_kses < ncpus) { 670 newke = kse_alloc(); 671 bzero(&newke->ke_startzero, RANGEOF(struct kse, 672 ke_startzero, ke_endzero)); 673 #if 0 674 mtx_lock_spin(&sched_lock); 675 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 676 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 677 mtx_unlock_spin(&sched_lock); 678 #endif 679 mtx_lock_spin(&sched_lock); 680 kse_link(newke, newkg); 681 /* Add engine */ 682 kse_reassign(newke); 683 mtx_unlock_spin(&sched_lock); 684 } 685 } 686 newku = upcall_alloc(); 687 newku->ku_mailbox = uap->mbx; 688 newku->ku_func = mbx.km_func; 689 bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 690 691 /* For the first call this may not have been set */ 692 if (td->td_standin == NULL) 693 thread_alloc_spare(td, NULL); 694 695 mtx_lock_spin(&sched_lock); 696 if (newkg->kg_numupcalls >= ncpus) { 697 mtx_unlock_spin(&sched_lock); 698 upcall_free(newku); 699 return (EPROCLIM); 700 } 701 upcall_link(newku, newkg); 702 if (mbx.km_quantum) 703 newkg->kg_upquantum = max(1, mbx.km_quantum/tick); 704 705 /* 706 * Each upcall structure has an owner thread, find which 707 * one owns it. 708 */ 709 if (uap->newgroup) { 710 /* 711 * Because new ksegrp hasn't thread, 712 * create an initial upcall thread to own it. 713 */ 714 thread_schedule_upcall(td, newku); 715 } else { 716 /* 717 * If current thread hasn't an upcall structure, 718 * just assign the upcall to it. 719 */ 720 if (td->td_upcall == NULL) { 721 newku->ku_owner = td; 722 td->td_upcall = newku; 723 } else { 724 /* 725 * Create a new upcall thread to own it. 726 */ 727 thread_schedule_upcall(td, newku); 728 } 729 } 730 mtx_unlock_spin(&sched_lock); 731 return (0); 732 } 733 734 /* 735 * Fill a ucontext_t with a thread's context information. 736 * 737 * This is an analogue to getcontext(3). 738 */ 739 void 740 thread_getcontext(struct thread *td, ucontext_t *uc) 741 { 742 743 /* 744 * XXX this is declared in a MD include file, i386/include/ucontext.h but 745 * is used in MI code. 746 */ 747 #ifdef __i386__ 748 get_mcontext(td, &uc->uc_mcontext); 749 #endif 750 uc->uc_sigmask = td->td_sigmask; 751 } 752 753 /* 754 * Set a thread's context from a ucontext_t. 755 * 756 * This is an analogue to setcontext(3). 757 */ 758 int 759 thread_setcontext(struct thread *td, ucontext_t *uc) 760 { 761 int ret; 762 763 /* 764 * XXX this is declared in a MD include file, i386/include/ucontext.h but 765 * is used in MI code. 766 */ 767 #ifdef __i386__ 768 ret = set_mcontext(td, &uc->uc_mcontext); 769 #else 770 ret = ENOSYS; 771 #endif 772 if (ret == 0) { 773 SIG_CANTMASK(uc->uc_sigmask); 774 PROC_LOCK(td->td_proc); 775 td->td_sigmask = uc->uc_sigmask; 776 PROC_UNLOCK(td->td_proc); 777 } 778 return (ret); 779 } 780 781 /* 782 * Initialize global thread allocation resources. 783 */ 784 void 785 threadinit(void) 786 { 787 788 #ifndef __ia64__ 789 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 790 thread_ctor, thread_dtor, thread_init, thread_fini, 791 UMA_ALIGN_CACHE, 0); 792 #else 793 /* 794 * XXX the ia64 kstack allocator is really lame and is at the mercy 795 * of contigmallloc(). This hackery is to pre-construct a whole 796 * pile of thread structures with associated kernel stacks early 797 * in the system startup while contigmalloc() still works. Once we 798 * have them, keep them. Sigh. 799 */ 800 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 801 thread_ctor, thread_dtor, thread_init, thread_fini, 802 UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); 803 uma_prealloc(thread_zone, 512); /* XXX arbitary */ 804 #endif 805 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 806 NULL, NULL, ksegrp_init, NULL, 807 UMA_ALIGN_CACHE, 0); 808 kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 809 NULL, NULL, kse_init, NULL, 810 UMA_ALIGN_CACHE, 0); 811 upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 812 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 813 } 814 815 /* 816 * Stash an embarasingly extra thread into the zombie thread queue. 817 */ 818 void 819 thread_stash(struct thread *td) 820 { 821 mtx_lock_spin(&kse_zombie_lock); 822 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 823 mtx_unlock_spin(&kse_zombie_lock); 824 } 825 826 /* 827 * Stash an embarasingly extra kse into the zombie kse queue. 828 */ 829 void 830 kse_stash(struct kse *ke) 831 { 832 mtx_lock_spin(&kse_zombie_lock); 833 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 834 mtx_unlock_spin(&kse_zombie_lock); 835 } 836 837 /* 838 * Stash an embarasingly extra upcall into the zombie upcall queue. 839 */ 840 841 void 842 upcall_stash(struct kse_upcall *ku) 843 { 844 mtx_lock_spin(&kse_zombie_lock); 845 TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 846 mtx_unlock_spin(&kse_zombie_lock); 847 } 848 849 /* 850 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 851 */ 852 void 853 ksegrp_stash(struct ksegrp *kg) 854 { 855 mtx_lock_spin(&kse_zombie_lock); 856 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 857 mtx_unlock_spin(&kse_zombie_lock); 858 } 859 860 /* 861 * Reap zombie kse resource. 862 */ 863 void 864 thread_reap(void) 865 { 866 struct thread *td_first, *td_next; 867 struct kse *ke_first, *ke_next; 868 struct ksegrp *kg_first, * kg_next; 869 struct kse_upcall *ku_first, *ku_next; 870 871 /* 872 * Don't even bother to lock if none at this instant, 873 * we really don't care about the next instant.. 874 */ 875 if ((!TAILQ_EMPTY(&zombie_threads)) 876 || (!TAILQ_EMPTY(&zombie_kses)) 877 || (!TAILQ_EMPTY(&zombie_ksegrps)) 878 || (!TAILQ_EMPTY(&zombie_upcalls))) { 879 mtx_lock_spin(&kse_zombie_lock); 880 td_first = TAILQ_FIRST(&zombie_threads); 881 ke_first = TAILQ_FIRST(&zombie_kses); 882 kg_first = TAILQ_FIRST(&zombie_ksegrps); 883 ku_first = TAILQ_FIRST(&zombie_upcalls); 884 if (td_first) 885 TAILQ_INIT(&zombie_threads); 886 if (ke_first) 887 TAILQ_INIT(&zombie_kses); 888 if (kg_first) 889 TAILQ_INIT(&zombie_ksegrps); 890 if (ku_first) 891 TAILQ_INIT(&zombie_upcalls); 892 mtx_unlock_spin(&kse_zombie_lock); 893 while (td_first) { 894 td_next = TAILQ_NEXT(td_first, td_runq); 895 if (td_first->td_ucred) 896 crfree(td_first->td_ucred); 897 thread_free(td_first); 898 td_first = td_next; 899 } 900 while (ke_first) { 901 ke_next = TAILQ_NEXT(ke_first, ke_procq); 902 kse_free(ke_first); 903 ke_first = ke_next; 904 } 905 while (kg_first) { 906 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 907 ksegrp_free(kg_first); 908 kg_first = kg_next; 909 } 910 while (ku_first) { 911 ku_next = TAILQ_NEXT(ku_first, ku_link); 912 upcall_free(ku_first); 913 ku_first = ku_next; 914 } 915 } 916 } 917 918 /* 919 * Allocate a ksegrp. 920 */ 921 struct ksegrp * 922 ksegrp_alloc(void) 923 { 924 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 925 } 926 927 /* 928 * Allocate a kse. 929 */ 930 struct kse * 931 kse_alloc(void) 932 { 933 return (uma_zalloc(kse_zone, M_WAITOK)); 934 } 935 936 /* 937 * Allocate a thread. 938 */ 939 struct thread * 940 thread_alloc(void) 941 { 942 thread_reap(); /* check if any zombies to get */ 943 return (uma_zalloc(thread_zone, M_WAITOK)); 944 } 945 946 /* 947 * Deallocate a ksegrp. 948 */ 949 void 950 ksegrp_free(struct ksegrp *td) 951 { 952 uma_zfree(ksegrp_zone, td); 953 } 954 955 /* 956 * Deallocate a kse. 957 */ 958 void 959 kse_free(struct kse *td) 960 { 961 uma_zfree(kse_zone, td); 962 } 963 964 /* 965 * Deallocate a thread. 966 */ 967 void 968 thread_free(struct thread *td) 969 { 970 971 cpu_thread_clean(td); 972 uma_zfree(thread_zone, td); 973 } 974 975 /* 976 * Store the thread context in the UTS's mailbox. 977 * then add the mailbox at the head of a list we are building in user space. 978 * The list is anchored in the ksegrp structure. 979 */ 980 int 981 thread_export_context(struct thread *td) 982 { 983 struct proc *p; 984 struct ksegrp *kg; 985 uintptr_t mbx; 986 void *addr; 987 int error,temp; 988 ucontext_t uc; 989 990 p = td->td_proc; 991 kg = td->td_ksegrp; 992 993 /* Export the user/machine context. */ 994 addr = (void *)(&td->td_mailbox->tm_context); 995 error = copyin(addr, &uc, sizeof(ucontext_t)); 996 if (error) 997 goto bad; 998 999 thread_getcontext(td, &uc); 1000 error = copyout(&uc, addr, sizeof(ucontext_t)); 1001 if (error) 1002 goto bad; 1003 1004 /* Exports clock ticks in kernel mode */ 1005 addr = (caddr_t)(&td->td_mailbox->tm_sticks); 1006 temp = fuword(addr) + td->td_usticks; 1007 if (suword(addr, temp)) 1008 goto bad; 1009 1010 /* Get address in latest mbox of list pointer */ 1011 addr = (void *)(&td->td_mailbox->tm_next); 1012 /* 1013 * Put the saved address of the previous first 1014 * entry into this one 1015 */ 1016 for (;;) { 1017 mbx = (uintptr_t)kg->kg_completed; 1018 if (suword(addr, mbx)) { 1019 error = EFAULT; 1020 goto bad; 1021 } 1022 PROC_LOCK(p); 1023 if (mbx == (uintptr_t)kg->kg_completed) { 1024 kg->kg_completed = td->td_mailbox; 1025 /* 1026 * The thread context may be taken away by 1027 * other upcall threads when we unlock 1028 * process lock. it's no longer valid to 1029 * use it again in any other places. 1030 */ 1031 td->td_mailbox = NULL; 1032 PROC_UNLOCK(p); 1033 break; 1034 } 1035 PROC_UNLOCK(p); 1036 } 1037 td->td_usticks = 0; 1038 return (0); 1039 1040 bad: 1041 PROC_LOCK(p); 1042 psignal(p, SIGSEGV); 1043 PROC_UNLOCK(p); 1044 /* The mailbox is bad, don't use it */ 1045 td->td_mailbox = NULL; 1046 td->td_usticks = 0; 1047 return (error); 1048 } 1049 1050 /* 1051 * Take the list of completed mailboxes for this KSEGRP and put them on this 1052 * upcall's mailbox as it's the next one going up. 1053 */ 1054 static int 1055 thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1056 { 1057 struct proc *p = kg->kg_proc; 1058 void *addr; 1059 uintptr_t mbx; 1060 1061 addr = (void *)(&ku->ku_mailbox->km_completed); 1062 for (;;) { 1063 mbx = (uintptr_t)kg->kg_completed; 1064 if (suword(addr, mbx)) { 1065 PROC_LOCK(p); 1066 psignal(p, SIGSEGV); 1067 PROC_UNLOCK(p); 1068 return (EFAULT); 1069 } 1070 PROC_LOCK(p); 1071 if (mbx == (uintptr_t)kg->kg_completed) { 1072 kg->kg_completed = NULL; 1073 PROC_UNLOCK(p); 1074 break; 1075 } 1076 PROC_UNLOCK(p); 1077 } 1078 return (0); 1079 } 1080 1081 /* 1082 * This function should be called at statclock interrupt time 1083 */ 1084 int 1085 thread_statclock(int user) 1086 { 1087 struct thread *td = curthread; 1088 1089 if (td->td_ksegrp->kg_numupcalls == 0) 1090 return (-1); 1091 if (user) { 1092 /* Current always do via ast() */ 1093 mtx_lock_spin(&sched_lock); 1094 td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1095 mtx_unlock_spin(&sched_lock); 1096 td->td_uuticks++; 1097 } else { 1098 if (td->td_mailbox != NULL) 1099 td->td_usticks++; 1100 else { 1101 /* XXXKSE 1102 * We will call thread_user_enter() for every 1103 * kernel entry in future, so if the thread mailbox 1104 * is NULL, it must be a UTS kernel, don't account 1105 * clock ticks for it. 1106 */ 1107 } 1108 } 1109 return (0); 1110 } 1111 1112 /* 1113 * Export state clock ticks for userland 1114 */ 1115 static int 1116 thread_update_usr_ticks(struct thread *td, int user) 1117 { 1118 struct proc *p = td->td_proc; 1119 struct kse_thr_mailbox *tmbx; 1120 struct kse_upcall *ku; 1121 struct ksegrp *kg; 1122 caddr_t addr; 1123 uint uticks; 1124 1125 if ((ku = td->td_upcall) == NULL) 1126 return (-1); 1127 1128 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1129 if ((tmbx == NULL) || (tmbx == (void *)-1)) 1130 return (-1); 1131 if (user) { 1132 uticks = td->td_uuticks; 1133 td->td_uuticks = 0; 1134 addr = (caddr_t)&tmbx->tm_uticks; 1135 } else { 1136 uticks = td->td_usticks; 1137 td->td_usticks = 0; 1138 addr = (caddr_t)&tmbx->tm_sticks; 1139 } 1140 if (uticks) { 1141 if (suword(addr, uticks+fuword(addr))) { 1142 PROC_LOCK(p); 1143 psignal(p, SIGSEGV); 1144 PROC_UNLOCK(p); 1145 return (-2); 1146 } 1147 } 1148 kg = td->td_ksegrp; 1149 if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { 1150 mtx_lock_spin(&sched_lock); 1151 td->td_upcall->ku_flags |= KUF_DOUPCALL; 1152 mtx_unlock_spin(&sched_lock); 1153 } 1154 return (0); 1155 } 1156 1157 /* 1158 * Discard the current thread and exit from its context. 1159 * 1160 * Because we can't free a thread while we're operating under its context, 1161 * push the current thread into our CPU's deadthread holder. This means 1162 * we needn't worry about someone else grabbing our context before we 1163 * do a cpu_throw(). 1164 */ 1165 void 1166 thread_exit(void) 1167 { 1168 struct thread *td; 1169 struct kse *ke; 1170 struct proc *p; 1171 struct ksegrp *kg; 1172 1173 td = curthread; 1174 kg = td->td_ksegrp; 1175 p = td->td_proc; 1176 ke = td->td_kse; 1177 1178 mtx_assert(&sched_lock, MA_OWNED); 1179 KASSERT(p != NULL, ("thread exiting without a process")); 1180 KASSERT(ke != NULL, ("thread exiting without a kse")); 1181 KASSERT(kg != NULL, ("thread exiting without a kse group")); 1182 PROC_LOCK_ASSERT(p, MA_OWNED); 1183 CTR1(KTR_PROC, "thread_exit: thread %p", td); 1184 KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 1185 1186 if (td->td_standin != NULL) { 1187 thread_stash(td->td_standin); 1188 td->td_standin = NULL; 1189 } 1190 1191 cpu_thread_exit(td); /* XXXSMP */ 1192 1193 /* 1194 * The last thread is left attached to the process 1195 * So that the whole bundle gets recycled. Skip 1196 * all this stuff. 1197 */ 1198 if (p->p_numthreads > 1) { 1199 /* 1200 * Unlink this thread from its proc and the kseg. 1201 * In keeping with the other structs we probably should 1202 * have a thread_unlink() that does some of this but it 1203 * would only be called from here (I think) so it would 1204 * be a waste. (might be useful for proc_fini() as well.) 1205 */ 1206 TAILQ_REMOVE(&p->p_threads, td, td_plist); 1207 p->p_numthreads--; 1208 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1209 kg->kg_numthreads--; 1210 if (p->p_maxthrwaits) 1211 wakeup(&p->p_numthreads); 1212 /* 1213 * The test below is NOT true if we are the 1214 * sole exiting thread. P_STOPPED_SNGL is unset 1215 * in exit1() after it is the only survivor. 1216 */ 1217 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1218 if (p->p_numthreads == p->p_suspcount) { 1219 thread_unsuspend_one(p->p_singlethread); 1220 } 1221 } 1222 1223 /* 1224 * Because each upcall structure has an owner thread, 1225 * owner thread exits only when process is in exiting 1226 * state, so upcall to userland is no longer needed, 1227 * deleting upcall structure is safe here. 1228 * So when all threads in a group is exited, all upcalls 1229 * in the group should be automatically freed. 1230 */ 1231 if (td->td_upcall) 1232 upcall_remove(td); 1233 1234 ke->ke_state = KES_UNQUEUED; 1235 ke->ke_thread = NULL; 1236 /* 1237 * Decide what to do with the KSE attached to this thread. 1238 */ 1239 if (ke->ke_flags & KEF_EXIT) 1240 kse_unlink(ke); 1241 else 1242 kse_reassign(ke); 1243 PROC_UNLOCK(p); 1244 td->td_kse = NULL; 1245 td->td_state = TDS_INACTIVE; 1246 #if 0 1247 td->td_proc = NULL; 1248 #endif 1249 td->td_ksegrp = NULL; 1250 td->td_last_kse = NULL; 1251 PCPU_SET(deadthread, td); 1252 } else { 1253 PROC_UNLOCK(p); 1254 } 1255 /* XXX Shouldn't cpu_throw() here. */ 1256 mtx_assert(&sched_lock, MA_OWNED); 1257 #if defined(__i386__) || defined(__sparc64__) 1258 cpu_throw(td, choosethread()); 1259 #else 1260 cpu_throw(); 1261 #endif 1262 panic("I'm a teapot!"); 1263 /* NOTREACHED */ 1264 } 1265 1266 /* 1267 * Do any thread specific cleanups that may be needed in wait() 1268 * called with Giant held, proc and schedlock not held. 1269 */ 1270 void 1271 thread_wait(struct proc *p) 1272 { 1273 struct thread *td; 1274 1275 KASSERT((p->p_numthreads == 1), ("Muliple threads in wait1()")); 1276 KASSERT((p->p_numksegrps == 1), ("Muliple ksegrps in wait1()")); 1277 FOREACH_THREAD_IN_PROC(p, td) { 1278 if (td->td_standin != NULL) { 1279 thread_free(td->td_standin); 1280 td->td_standin = NULL; 1281 } 1282 cpu_thread_clean(td); 1283 } 1284 thread_reap(); /* check for zombie threads etc. */ 1285 } 1286 1287 /* 1288 * Link a thread to a process. 1289 * set up anything that needs to be initialized for it to 1290 * be used by the process. 1291 * 1292 * Note that we do not link to the proc's ucred here. 1293 * The thread is linked as if running but no KSE assigned. 1294 */ 1295 void 1296 thread_link(struct thread *td, struct ksegrp *kg) 1297 { 1298 struct proc *p; 1299 1300 p = kg->kg_proc; 1301 td->td_state = TDS_INACTIVE; 1302 td->td_proc = p; 1303 td->td_ksegrp = kg; 1304 td->td_last_kse = NULL; 1305 td->td_flags = 0; 1306 td->td_kse = NULL; 1307 1308 LIST_INIT(&td->td_contested); 1309 callout_init(&td->td_slpcallout, 1); 1310 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 1311 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 1312 p->p_numthreads++; 1313 kg->kg_numthreads++; 1314 } 1315 1316 /* 1317 * Purge a ksegrp resource. When a ksegrp is preparing to 1318 * exit, it calls this function. 1319 */ 1320 void 1321 kse_purge_group(struct thread *td) 1322 { 1323 struct ksegrp *kg; 1324 struct kse *ke; 1325 1326 kg = td->td_ksegrp; 1327 KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1328 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1329 KASSERT(ke->ke_state == KES_IDLE, 1330 ("%s: wrong idle KSE state", __func__)); 1331 kse_unlink(ke); 1332 } 1333 KASSERT((kg->kg_kses == 1), 1334 ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1335 KASSERT((kg->kg_numupcalls == 0), 1336 ("%s: ksegrp still has %d upcall datas", 1337 __func__, kg->kg_numupcalls)); 1338 } 1339 1340 /* 1341 * Purge a process's KSE resource. When a process is preparing to 1342 * exit, it calls kse_purge to release any extra KSE resources in 1343 * the process. 1344 */ 1345 void 1346 kse_purge(struct proc *p, struct thread *td) 1347 { 1348 struct ksegrp *kg; 1349 struct kse *ke; 1350 1351 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1352 mtx_lock_spin(&sched_lock); 1353 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1354 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1355 p->p_numksegrps--; 1356 /* 1357 * There is no ownership for KSE, after all threads 1358 * in the group exited, it is possible that some KSEs 1359 * were left in idle queue, gc them now. 1360 */ 1361 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1362 KASSERT(ke->ke_state == KES_IDLE, 1363 ("%s: wrong idle KSE state", __func__)); 1364 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1365 kg->kg_idle_kses--; 1366 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1367 kg->kg_kses--; 1368 kse_stash(ke); 1369 } 1370 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1371 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1372 ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1373 KASSERT((kg->kg_numupcalls == 0), 1374 ("%s: ksegrp still has %d upcall datas", 1375 __func__, kg->kg_numupcalls)); 1376 1377 if (kg != td->td_ksegrp) 1378 ksegrp_stash(kg); 1379 } 1380 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1381 p->p_numksegrps++; 1382 mtx_unlock_spin(&sched_lock); 1383 } 1384 1385 /* 1386 * This function is intended to be used to initialize a spare thread 1387 * for upcall. Initialize thread's large data area outside sched_lock 1388 * for thread_schedule_upcall(). 1389 */ 1390 void 1391 thread_alloc_spare(struct thread *td, struct thread *spare) 1392 { 1393 if (td->td_standin) 1394 return; 1395 if (spare == NULL) 1396 spare = thread_alloc(); 1397 td->td_standin = spare; 1398 bzero(&spare->td_startzero, 1399 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1400 spare->td_proc = td->td_proc; 1401 spare->td_ucred = crhold(td->td_ucred); 1402 } 1403 1404 /* 1405 * Create a thread and schedule it for upcall on the KSE given. 1406 * Use our thread's standin so that we don't have to allocate one. 1407 */ 1408 struct thread * 1409 thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 1410 { 1411 struct thread *td2; 1412 1413 mtx_assert(&sched_lock, MA_OWNED); 1414 1415 /* 1416 * Schedule an upcall thread on specified kse_upcall, 1417 * the kse_upcall must be free. 1418 * td must have a spare thread. 1419 */ 1420 KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1421 if ((td2 = td->td_standin) != NULL) { 1422 td->td_standin = NULL; 1423 } else { 1424 panic("no reserve thread when scheduling an upcall"); 1425 return (NULL); 1426 } 1427 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1428 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1429 bcopy(&td->td_startcopy, &td2->td_startcopy, 1430 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1431 thread_link(td2, ku->ku_ksegrp); 1432 /* inherit blocked thread's context */ 1433 bcopy(td->td_frame, td2->td_frame, sizeof(struct trapframe)); 1434 cpu_set_upcall(td2, td->td_pcb); 1435 /* Let the new thread become owner of the upcall */ 1436 ku->ku_owner = td2; 1437 td2->td_upcall = ku; 1438 td2->td_flags = TDF_UPCALLING; 1439 #if 0 /* XXX This shouldn't be necessary */ 1440 if (td->td_proc->p_sflag & PS_NEEDSIGCHK) 1441 td2->td_flags |= TDF_ASTPENDING; 1442 #endif 1443 td2->td_kse = NULL; 1444 td2->td_state = TDS_CAN_RUN; 1445 td2->td_inhibitors = 0; 1446 setrunqueue(td2); 1447 return (td2); /* bogus.. should be a void function */ 1448 } 1449 1450 void 1451 thread_signal_add(struct thread *td, int sig) 1452 { 1453 struct kse_upcall *ku; 1454 struct proc *p; 1455 sigset_t ss; 1456 int error; 1457 1458 PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); 1459 td = curthread; 1460 ku = td->td_upcall; 1461 p = td->td_proc; 1462 1463 PROC_UNLOCK(p); 1464 error = copyin(&ku->ku_mailbox->km_sigscaught, &ss, sizeof(sigset_t)); 1465 if (error) 1466 goto error; 1467 1468 SIGADDSET(ss, sig); 1469 1470 error = copyout(&ss, &ku->ku_mailbox->km_sigscaught, sizeof(sigset_t)); 1471 if (error) 1472 goto error; 1473 1474 PROC_LOCK(p); 1475 return; 1476 error: 1477 PROC_LOCK(p); 1478 sigexit(td, SIGILL); 1479 } 1480 1481 1482 /* 1483 * Schedule an upcall to notify a KSE process recieved signals. 1484 * 1485 */ 1486 void 1487 thread_signal_upcall(struct thread *td) 1488 { 1489 mtx_lock_spin(&sched_lock); 1490 td->td_flags |= TDF_UPCALLING; 1491 mtx_unlock_spin(&sched_lock); 1492 1493 return; 1494 } 1495 1496 void 1497 thread_switchout(struct thread *td) 1498 { 1499 struct kse_upcall *ku; 1500 1501 mtx_assert(&sched_lock, MA_OWNED); 1502 1503 /* 1504 * If the outgoing thread is in threaded group and has never 1505 * scheduled an upcall, decide whether this is a short 1506 * or long term event and thus whether or not to schedule 1507 * an upcall. 1508 * If it is a short term event, just suspend it in 1509 * a way that takes its KSE with it. 1510 * Select the events for which we want to schedule upcalls. 1511 * For now it's just sleep. 1512 * XXXKSE eventually almost any inhibition could do. 1513 */ 1514 if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { 1515 /* 1516 * Release ownership of upcall, and schedule an upcall 1517 * thread, this new upcall thread becomes the owner of 1518 * the upcall structure. 1519 */ 1520 ku = td->td_upcall; 1521 ku->ku_owner = NULL; 1522 td->td_upcall = NULL; 1523 td->td_flags &= ~TDF_CAN_UNBIND; 1524 thread_schedule_upcall(td, ku); 1525 } 1526 } 1527 1528 /* 1529 * Setup done on the thread when it enters the kernel. 1530 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1531 */ 1532 void 1533 thread_user_enter(struct proc *p, struct thread *td) 1534 { 1535 struct ksegrp *kg; 1536 struct kse_upcall *ku; 1537 1538 kg = td->td_ksegrp; 1539 /* 1540 * First check that we shouldn't just abort. 1541 * But check if we are the single thread first! 1542 * XXX p_singlethread not locked, but should be safe. 1543 */ 1544 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1545 PROC_LOCK(p); 1546 mtx_lock_spin(&sched_lock); 1547 thread_stopped(p); 1548 thread_exit(); 1549 /* NOTREACHED */ 1550 } 1551 1552 /* 1553 * If we are doing a syscall in a KSE environment, 1554 * note where our mailbox is. There is always the 1555 * possibility that we could do this lazily (in kse_reassign()), 1556 * but for now do it every time. 1557 */ 1558 kg = td->td_ksegrp; 1559 if (kg->kg_numupcalls) { 1560 ku = td->td_upcall; 1561 KASSERT(ku, ("%s: no upcall owned", __func__)); 1562 KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1563 td->td_mailbox = 1564 (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1565 if ((td->td_mailbox == NULL) || 1566 (td->td_mailbox == (void *)-1)) { 1567 /* Don't schedule upcall when blocked */ 1568 td->td_mailbox = NULL; 1569 mtx_lock_spin(&sched_lock); 1570 td->td_flags &= ~TDF_CAN_UNBIND; 1571 mtx_unlock_spin(&sched_lock); 1572 } else { 1573 if (td->td_standin == NULL) 1574 thread_alloc_spare(td, NULL); 1575 mtx_lock_spin(&sched_lock); 1576 td->td_flags |= TDF_CAN_UNBIND; 1577 mtx_unlock_spin(&sched_lock); 1578 } 1579 } 1580 } 1581 1582 /* 1583 * The extra work we go through if we are a threaded process when we 1584 * return to userland. 1585 * 1586 * If we are a KSE process and returning to user mode, check for 1587 * extra work to do before we return (e.g. for more syscalls 1588 * to complete first). If we were in a critical section, we should 1589 * just return to let it finish. Same if we were in the UTS (in 1590 * which case the mailbox's context's busy indicator will be set). 1591 * The only traps we suport will have set the mailbox. 1592 * We will clear it here. 1593 */ 1594 int 1595 thread_userret(struct thread *td, struct trapframe *frame) 1596 { 1597 int error = 0, upcalls; 1598 struct kse_upcall *ku; 1599 struct ksegrp *kg, *kg2; 1600 struct proc *p; 1601 struct timespec ts; 1602 1603 p = td->td_proc; 1604 kg = td->td_ksegrp; 1605 1606 1607 /* Nothing to do with non-threaded group/process */ 1608 if (td->td_ksegrp->kg_numupcalls == 0) 1609 return (0); 1610 1611 /* 1612 * Stat clock interrupt hit in userland, it 1613 * is returning from interrupt, charge thread's 1614 * userland time for UTS. 1615 */ 1616 if (td->td_flags & TDF_USTATCLOCK) { 1617 thread_update_usr_ticks(td, 1); 1618 mtx_lock_spin(&sched_lock); 1619 td->td_flags &= ~TDF_USTATCLOCK; 1620 mtx_unlock_spin(&sched_lock); 1621 if (kg->kg_completed || 1622 (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1623 thread_user_enter(p, td); 1624 } 1625 1626 /* 1627 * Optimisation: 1628 * This thread has not started any upcall. 1629 * If there is no work to report other than ourself, 1630 * then it can return direct to userland. 1631 */ 1632 if (TD_CAN_UNBIND(td)) { 1633 mtx_lock_spin(&sched_lock); 1634 td->td_flags &= ~TDF_CAN_UNBIND; 1635 ku = td->td_upcall; 1636 if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1637 (kg->kg_completed == NULL) && 1638 (ku->ku_flags & KUF_DOUPCALL) == 0 && 1639 (kg->kg_upquantum && ticks >= kg->kg_nextupcall)) { 1640 mtx_unlock_spin(&sched_lock); 1641 thread_update_usr_ticks(td, 0); 1642 nanotime(&ts); 1643 error = copyout(&ts, 1644 (caddr_t)&ku->ku_mailbox->km_timeofday, 1645 sizeof(ts)); 1646 td->td_mailbox = 0; 1647 if (error) 1648 goto out; 1649 return (0); 1650 } 1651 mtx_unlock_spin(&sched_lock); 1652 error = thread_export_context(td); 1653 if (error) { 1654 /* 1655 * Failing to do the KSE operation just defaults 1656 * back to synchonous operation, so just return from 1657 * the syscall. 1658 */ 1659 return (0); 1660 } 1661 /* 1662 * There is something to report, and we own an upcall 1663 * strucuture, we can go to userland. 1664 * Turn ourself into an upcall thread. 1665 */ 1666 mtx_lock_spin(&sched_lock); 1667 td->td_flags |= TDF_UPCALLING; 1668 mtx_unlock_spin(&sched_lock); 1669 } else if (td->td_mailbox) { 1670 error = thread_export_context(td); 1671 /* possibly upcall with error? */ 1672 PROC_LOCK(p); 1673 /* 1674 * There are upcall threads waiting for 1675 * work to do, wake one of them up. 1676 * XXXKSE Maybe wake all of them up. 1677 */ 1678 if (!error && kg->kg_upsleeps) 1679 wakeup_one(&kg->kg_completed); 1680 mtx_lock_spin(&sched_lock); 1681 thread_stopped(p); 1682 thread_exit(); 1683 /* NOTREACHED */ 1684 } 1685 1686 KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1687 1688 if (p->p_numthreads > max_threads_per_proc) { 1689 max_threads_hits++; 1690 PROC_LOCK(p); 1691 while (p->p_numthreads > max_threads_per_proc) { 1692 if (P_SHOULDSTOP(p)) 1693 break; 1694 upcalls = 0; 1695 mtx_lock_spin(&sched_lock); 1696 FOREACH_KSEGRP_IN_PROC(p, kg2) { 1697 if (kg2->kg_numupcalls == 0) 1698 upcalls++; 1699 else 1700 upcalls += kg2->kg_numupcalls; 1701 } 1702 mtx_unlock_spin(&sched_lock); 1703 if (upcalls >= max_threads_per_proc) 1704 break; 1705 p->p_maxthrwaits++; 1706 msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1707 "maxthreads", NULL); 1708 p->p_maxthrwaits--; 1709 } 1710 PROC_UNLOCK(p); 1711 } 1712 1713 if (td->td_flags & TDF_UPCALLING) { 1714 kg->kg_nextupcall = ticks+kg->kg_upquantum; 1715 ku = td->td_upcall; 1716 /* 1717 * There is no more work to do and we are going to ride 1718 * this thread up to userland as an upcall. 1719 * Do the last parts of the setup needed for the upcall. 1720 */ 1721 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1722 td, td->td_proc->p_pid, td->td_proc->p_comm); 1723 1724 /* 1725 * Set user context to the UTS. 1726 * Will use Giant in cpu_thread_clean() because it uses 1727 * kmem_free(kernel_map, ...) 1728 */ 1729 cpu_set_upcall_kse(td, ku); 1730 mtx_lock_spin(&sched_lock); 1731 td->td_flags &= ~TDF_UPCALLING; 1732 if (ku->ku_flags & KUF_DOUPCALL) 1733 ku->ku_flags &= ~KUF_DOUPCALL; 1734 mtx_unlock_spin(&sched_lock); 1735 1736 /* 1737 * Unhook the list of completed threads. 1738 * anything that completes after this gets to 1739 * come in next time. 1740 * Put the list of completed thread mailboxes on 1741 * this KSE's mailbox. 1742 */ 1743 error = thread_link_mboxes(kg, ku); 1744 if (error) 1745 goto out; 1746 1747 /* 1748 * Set state and clear the thread mailbox pointer. 1749 * From now on we are just a bound outgoing process. 1750 * **Problem** userret is often called several times. 1751 * it would be nice if this all happenned only on the first 1752 * time through. (the scan for extra work etc.) 1753 */ 1754 error = suword((caddr_t)&ku->ku_mailbox->km_curthread, 0); 1755 if (error) 1756 goto out; 1757 1758 /* Export current system time */ 1759 nanotime(&ts); 1760 error = copyout(&ts, (caddr_t)&ku->ku_mailbox->km_timeofday, 1761 sizeof(ts)); 1762 } 1763 1764 out: 1765 if (error) { 1766 /* 1767 * Things are going to be so screwed we should just kill 1768 * the process. 1769 * how do we do that? 1770 */ 1771 PROC_LOCK(td->td_proc); 1772 psignal(td->td_proc, SIGSEGV); 1773 PROC_UNLOCK(td->td_proc); 1774 } else { 1775 /* 1776 * Optimisation: 1777 * Ensure that we have a spare thread available, 1778 * for when we re-enter the kernel. 1779 */ 1780 if (td->td_standin == NULL) 1781 thread_alloc_spare(td, NULL); 1782 } 1783 1784 /* 1785 * Clear thread mailbox first, then clear system tick count. 1786 * The order is important because thread_statclock() use 1787 * mailbox pointer to see if it is an userland thread or 1788 * an UTS kernel thread. 1789 */ 1790 td->td_mailbox = NULL; 1791 td->td_usticks = 0; 1792 return (error); /* go sync */ 1793 } 1794 1795 /* 1796 * Enforce single-threading. 1797 * 1798 * Returns 1 if the caller must abort (another thread is waiting to 1799 * exit the process or similar). Process is locked! 1800 * Returns 0 when you are successfully the only thread running. 1801 * A process has successfully single threaded in the suspend mode when 1802 * There are no threads in user mode. Threads in the kernel must be 1803 * allowed to continue until they get to the user boundary. They may even 1804 * copy out their return values and data before suspending. They may however be 1805 * accellerated in reaching the user boundary as we will wake up 1806 * any sleeping threads that are interruptable. (PCATCH). 1807 */ 1808 int 1809 thread_single(int force_exit) 1810 { 1811 struct thread *td; 1812 struct thread *td2; 1813 struct proc *p; 1814 1815 td = curthread; 1816 p = td->td_proc; 1817 mtx_assert(&Giant, MA_OWNED); 1818 PROC_LOCK_ASSERT(p, MA_OWNED); 1819 KASSERT((td != NULL), ("curthread is NULL")); 1820 1821 if ((p->p_flag & P_THREADED) == 0 && p->p_numthreads == 1) 1822 return (0); 1823 1824 /* Is someone already single threading? */ 1825 if (p->p_singlethread) 1826 return (1); 1827 1828 if (force_exit == SINGLE_EXIT) { 1829 p->p_flag |= P_SINGLE_EXIT; 1830 } else 1831 p->p_flag &= ~P_SINGLE_EXIT; 1832 p->p_flag |= P_STOPPED_SINGLE; 1833 p->p_singlethread = td; 1834 /* XXXKSE Which lock protects the below values? */ 1835 while ((p->p_numthreads - p->p_suspcount) != 1) { 1836 mtx_lock_spin(&sched_lock); 1837 FOREACH_THREAD_IN_PROC(p, td2) { 1838 if (td2 == td) 1839 continue; 1840 td->td_flags |= TDF_ASTPENDING; 1841 if (TD_IS_INHIBITED(td2)) { 1842 if (force_exit == SINGLE_EXIT) { 1843 if (TD_IS_SUSPENDED(td2)) { 1844 thread_unsuspend_one(td2); 1845 } 1846 if (TD_ON_SLEEPQ(td2) && 1847 (td2->td_flags & TDF_SINTR)) { 1848 if (td2->td_flags & TDF_CVWAITQ) 1849 cv_abort(td2); 1850 else 1851 abortsleep(td2); 1852 } 1853 } else { 1854 if (TD_IS_SUSPENDED(td2)) 1855 continue; 1856 /* 1857 * maybe other inhibitted states too? 1858 * XXXKSE Is it totally safe to 1859 * suspend a non-interruptable thread? 1860 */ 1861 if (td2->td_inhibitors & 1862 (TDI_SLEEPING | TDI_SWAPPED)) 1863 thread_suspend_one(td2); 1864 } 1865 } 1866 } 1867 /* 1868 * Maybe we suspended some threads.. was it enough? 1869 */ 1870 if ((p->p_numthreads - p->p_suspcount) == 1) { 1871 mtx_unlock_spin(&sched_lock); 1872 break; 1873 } 1874 1875 /* 1876 * Wake us up when everyone else has suspended. 1877 * In the mean time we suspend as well. 1878 */ 1879 thread_suspend_one(td); 1880 /* XXX If you recursed this is broken. */ 1881 mtx_unlock(&Giant); 1882 PROC_UNLOCK(p); 1883 p->p_stats->p_ru.ru_nvcsw++; 1884 mi_switch(); 1885 mtx_unlock_spin(&sched_lock); 1886 mtx_lock(&Giant); 1887 PROC_LOCK(p); 1888 } 1889 if (force_exit == SINGLE_EXIT) { 1890 if (td->td_upcall) { 1891 mtx_lock_spin(&sched_lock); 1892 upcall_remove(td); 1893 mtx_unlock_spin(&sched_lock); 1894 } 1895 kse_purge(p, td); 1896 } 1897 return (0); 1898 } 1899 1900 /* 1901 * Called in from locations that can safely check to see 1902 * whether we have to suspend or at least throttle for a 1903 * single-thread event (e.g. fork). 1904 * 1905 * Such locations include userret(). 1906 * If the "return_instead" argument is non zero, the thread must be able to 1907 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 1908 * 1909 * The 'return_instead' argument tells the function if it may do a 1910 * thread_exit() or suspend, or whether the caller must abort and back 1911 * out instead. 1912 * 1913 * If the thread that set the single_threading request has set the 1914 * P_SINGLE_EXIT bit in the process flags then this call will never return 1915 * if 'return_instead' is false, but will exit. 1916 * 1917 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 1918 *---------------+--------------------+--------------------- 1919 * 0 | returns 0 | returns 0 or 1 1920 * | when ST ends | immediatly 1921 *---------------+--------------------+--------------------- 1922 * 1 | thread exits | returns 1 1923 * | | immediatly 1924 * 0 = thread_exit() or suspension ok, 1925 * other = return error instead of stopping the thread. 1926 * 1927 * While a full suspension is under effect, even a single threading 1928 * thread would be suspended if it made this call (but it shouldn't). 1929 * This call should only be made from places where 1930 * thread_exit() would be safe as that may be the outcome unless 1931 * return_instead is set. 1932 */ 1933 int 1934 thread_suspend_check(int return_instead) 1935 { 1936 struct thread *td; 1937 struct proc *p; 1938 struct ksegrp *kg; 1939 1940 td = curthread; 1941 p = td->td_proc; 1942 kg = td->td_ksegrp; 1943 PROC_LOCK_ASSERT(p, MA_OWNED); 1944 while (P_SHOULDSTOP(p)) { 1945 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1946 KASSERT(p->p_singlethread != NULL, 1947 ("singlethread not set")); 1948 /* 1949 * The only suspension in action is a 1950 * single-threading. Single threader need not stop. 1951 * XXX Should be safe to access unlocked 1952 * as it can only be set to be true by us. 1953 */ 1954 if (p->p_singlethread == td) 1955 return (0); /* Exempt from stopping. */ 1956 } 1957 if (return_instead) 1958 return (1); 1959 1960 mtx_lock_spin(&sched_lock); 1961 thread_stopped(p); 1962 /* 1963 * If the process is waiting for us to exit, 1964 * this thread should just suicide. 1965 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 1966 */ 1967 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1968 while (mtx_owned(&Giant)) 1969 mtx_unlock(&Giant); 1970 if (p->p_flag & P_THREADED) 1971 thread_exit(); 1972 else 1973 thr_exit1(); 1974 } 1975 1976 mtx_assert(&Giant, MA_NOTOWNED); 1977 /* 1978 * When a thread suspends, it just 1979 * moves to the processes's suspend queue 1980 * and stays there. 1981 */ 1982 thread_suspend_one(td); 1983 PROC_UNLOCK(p); 1984 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1985 if (p->p_numthreads == p->p_suspcount) { 1986 thread_unsuspend_one(p->p_singlethread); 1987 } 1988 } 1989 p->p_stats->p_ru.ru_nivcsw++; 1990 mi_switch(); 1991 mtx_unlock_spin(&sched_lock); 1992 PROC_LOCK(p); 1993 } 1994 return (0); 1995 } 1996 1997 void 1998 thread_suspend_one(struct thread *td) 1999 { 2000 struct proc *p = td->td_proc; 2001 2002 mtx_assert(&sched_lock, MA_OWNED); 2003 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 2004 p->p_suspcount++; 2005 TD_SET_SUSPENDED(td); 2006 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 2007 /* 2008 * Hack: If we are suspending but are on the sleep queue 2009 * then we are in msleep or the cv equivalent. We 2010 * want to look like we have two Inhibitors. 2011 * May already be set.. doesn't matter. 2012 */ 2013 if (TD_ON_SLEEPQ(td)) 2014 TD_SET_SLEEPING(td); 2015 } 2016 2017 void 2018 thread_unsuspend_one(struct thread *td) 2019 { 2020 struct proc *p = td->td_proc; 2021 2022 mtx_assert(&sched_lock, MA_OWNED); 2023 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 2024 TD_CLR_SUSPENDED(td); 2025 p->p_suspcount--; 2026 setrunnable(td); 2027 } 2028 2029 /* 2030 * Allow all threads blocked by single threading to continue running. 2031 */ 2032 void 2033 thread_unsuspend(struct proc *p) 2034 { 2035 struct thread *td; 2036 2037 mtx_assert(&sched_lock, MA_OWNED); 2038 PROC_LOCK_ASSERT(p, MA_OWNED); 2039 if (!P_SHOULDSTOP(p)) { 2040 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2041 thread_unsuspend_one(td); 2042 } 2043 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 2044 (p->p_numthreads == p->p_suspcount)) { 2045 /* 2046 * Stopping everything also did the job for the single 2047 * threading request. Now we've downgraded to single-threaded, 2048 * let it continue. 2049 */ 2050 thread_unsuspend_one(p->p_singlethread); 2051 } 2052 } 2053 2054 void 2055 thread_single_end(void) 2056 { 2057 struct thread *td; 2058 struct proc *p; 2059 2060 td = curthread; 2061 p = td->td_proc; 2062 PROC_LOCK_ASSERT(p, MA_OWNED); 2063 p->p_flag &= ~P_STOPPED_SINGLE; 2064 p->p_singlethread = NULL; 2065 /* 2066 * If there are other threads they mey now run, 2067 * unless of course there is a blanket 'stop order' 2068 * on the process. The single threader must be allowed 2069 * to continue however as this is a bad place to stop. 2070 */ 2071 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2072 mtx_lock_spin(&sched_lock); 2073 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2074 thread_unsuspend_one(td); 2075 } 2076 mtx_unlock_spin(&sched_lock); 2077 } 2078 } 2079 2080 2081