1 /* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/lock.h> 35 #include <sys/malloc.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 #include <sys/sysproto.h> 40 #include <sys/filedesc.h> 41 #include <sys/tty.h> 42 #include <sys/signalvar.h> 43 #include <sys/sx.h> 44 #include <sys/user.h> 45 #include <sys/jail.h> 46 #include <sys/kse.h> 47 #include <sys/ktr.h> 48 #include <sys/ucontext.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_object.h> 52 #include <vm/pmap.h> 53 #include <vm/uma.h> 54 #include <vm/vm_map.h> 55 56 #include <machine/frame.h> 57 58 /* 59 * KSEGRP related storage. 60 */ 61 static uma_zone_t ksegrp_zone; 62 static uma_zone_t kse_zone; 63 static uma_zone_t thread_zone; 64 65 /* DEBUG ONLY */ 66 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 67 static int oiks_debug = 1; /* 0 disable, 1 printf, 2 enter debugger */ 68 SYSCTL_INT(_kern_threads, OID_AUTO, oiks, CTLFLAG_RW, 69 &oiks_debug, 0, "OIKS thread debug"); 70 71 static int max_threads_per_proc = 10; 72 SYSCTL_INT(_kern_threads, OID_AUTO, max_per_proc, CTLFLAG_RW, 73 &max_threads_per_proc, 0, "Limit on threads per proc"); 74 75 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 76 77 struct threadqueue zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 78 TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 79 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 80 struct mtx zombie_thread_lock; 81 MTX_SYSINIT(zombie_thread_lock, &zombie_thread_lock, 82 "zombie_thread_lock", MTX_SPIN); 83 84 85 86 void kse_purge(struct proc *p, struct thread *td); 87 /* 88 * Pepare a thread for use. 89 */ 90 static void 91 thread_ctor(void *mem, int size, void *arg) 92 { 93 struct thread *td; 94 95 KASSERT((size == sizeof(struct thread)), 96 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 97 98 td = (struct thread *)mem; 99 td->td_state = TDS_INACTIVE; 100 td->td_flags |= TDF_UNBOUND; 101 } 102 103 /* 104 * Reclaim a thread after use. 105 */ 106 static void 107 thread_dtor(void *mem, int size, void *arg) 108 { 109 struct thread *td; 110 111 KASSERT((size == sizeof(struct thread)), 112 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 113 114 td = (struct thread *)mem; 115 116 #ifdef INVARIANTS 117 /* Verify that this thread is in a safe state to free. */ 118 switch (td->td_state) { 119 case TDS_INHIBITED: 120 case TDS_RUNNING: 121 case TDS_CAN_RUN: 122 case TDS_RUNQ: 123 /* 124 * We must never unlink a thread that is in one of 125 * these states, because it is currently active. 126 */ 127 panic("bad state for thread unlinking"); 128 /* NOTREACHED */ 129 case TDS_INACTIVE: 130 break; 131 default: 132 panic("bad thread state"); 133 /* NOTREACHED */ 134 } 135 #endif 136 } 137 138 /* 139 * Initialize type-stable parts of a thread (when newly created). 140 */ 141 static void 142 thread_init(void *mem, int size) 143 { 144 struct thread *td; 145 146 KASSERT((size == sizeof(struct thread)), 147 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 148 149 td = (struct thread *)mem; 150 mtx_lock(&Giant); 151 pmap_new_thread(td, 0); 152 mtx_unlock(&Giant); 153 cpu_thread_setup(td); 154 } 155 156 /* 157 * Tear down type-stable parts of a thread (just before being discarded). 158 */ 159 static void 160 thread_fini(void *mem, int size) 161 { 162 struct thread *td; 163 164 KASSERT((size == sizeof(struct thread)), 165 ("size mismatch: %d != %d\n", size, (int)sizeof(struct thread))); 166 167 td = (struct thread *)mem; 168 pmap_dispose_thread(td); 169 } 170 171 /* 172 * KSE is linked onto the idle queue. 173 */ 174 void 175 kse_link(struct kse *ke, struct ksegrp *kg) 176 { 177 struct proc *p = kg->kg_proc; 178 179 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 180 kg->kg_kses++; 181 ke->ke_state = KES_UNQUEUED; 182 ke->ke_proc = p; 183 ke->ke_ksegrp = kg; 184 ke->ke_thread = NULL; 185 ke->ke_oncpu = NOCPU; 186 } 187 188 void 189 kse_unlink(struct kse *ke) 190 { 191 struct ksegrp *kg; 192 193 mtx_assert(&sched_lock, MA_OWNED); 194 kg = ke->ke_ksegrp; 195 if (ke->ke_state == KES_IDLE) { 196 kg->kg_idle_kses--; 197 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 198 } 199 200 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 201 if (--kg->kg_kses == 0) { 202 ksegrp_unlink(kg); 203 } 204 /* 205 * Aggregate stats from the KSE 206 */ 207 kse_stash(ke); 208 } 209 210 void 211 ksegrp_link(struct ksegrp *kg, struct proc *p) 212 { 213 214 TAILQ_INIT(&kg->kg_threads); 215 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 216 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 217 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 218 TAILQ_INIT(&kg->kg_iq); /* idle kses in ksegrp */ 219 TAILQ_INIT(&kg->kg_lq); /* loan kses in ksegrp */ 220 kg->kg_proc = p; 221 /* the following counters are in the -zero- section and may not need clearing */ 222 kg->kg_numthreads = 0; 223 kg->kg_runnable = 0; 224 kg->kg_kses = 0; 225 kg->kg_idle_kses = 0; 226 kg->kg_loan_kses = 0; 227 kg->kg_runq_kses = 0; /* XXXKSE change name */ 228 /* link it in now that it's consistent */ 229 p->p_numksegrps++; 230 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 231 } 232 233 void 234 ksegrp_unlink(struct ksegrp *kg) 235 { 236 struct proc *p; 237 238 mtx_assert(&sched_lock, MA_OWNED); 239 p = kg->kg_proc; 240 KASSERT(((kg->kg_numthreads == 0) && (kg->kg_kses == 0)), 241 ("kseg_unlink: residual threads or KSEs")); 242 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 243 p->p_numksegrps--; 244 /* 245 * Aggregate stats from the KSE 246 */ 247 ksegrp_stash(kg); 248 } 249 250 /* 251 * for a newly created process, 252 * link up a the structure and its initial threads etc. 253 */ 254 void 255 proc_linkup(struct proc *p, struct ksegrp *kg, 256 struct kse *ke, struct thread *td) 257 { 258 259 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 260 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 261 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 262 p->p_numksegrps = 0; 263 p->p_numthreads = 0; 264 265 ksegrp_link(kg, p); 266 kse_link(ke, kg); 267 thread_link(td, kg); 268 } 269 270 int 271 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 272 { 273 274 return(ENOSYS); 275 } 276 277 int 278 kse_exit(struct thread *td, struct kse_exit_args *uap) 279 { 280 struct proc *p; 281 struct ksegrp *kg; 282 283 p = td->td_proc; 284 /* KSE-enabled processes only, please. */ 285 if (!(p->p_flag & P_KSES)) 286 return EINVAL; 287 /* must be a bound thread */ 288 if (td->td_flags & TDF_UNBOUND) 289 return EINVAL; 290 kg = td->td_ksegrp; 291 /* serialize killing kse */ 292 PROC_LOCK(p); 293 mtx_lock_spin(&sched_lock); 294 if ((kg->kg_kses == 1) && (kg->kg_numthreads > 1)) { 295 mtx_unlock_spin(&sched_lock); 296 PROC_UNLOCK(p); 297 return (EDEADLK); 298 } 299 if ((p->p_numthreads == 1) && (p->p_numksegrps == 1)) { 300 p->p_flag &= ~P_KSES; 301 mtx_unlock_spin(&sched_lock); 302 PROC_UNLOCK(p); 303 } else { 304 while (mtx_owned(&Giant)) 305 mtx_unlock(&Giant); 306 td->td_kse->ke_flags |= KEF_EXIT; 307 thread_exit(); 308 /* NOTREACHED */ 309 } 310 return 0; 311 } 312 313 int 314 kse_release(struct thread *td, struct kse_release_args *uap) 315 { 316 struct proc *p; 317 318 p = td->td_proc; 319 /* KSE-enabled processes only, please. */ 320 if (p->p_flag & P_KSES) { 321 PROC_LOCK(p); 322 mtx_lock_spin(&sched_lock); 323 thread_exit(); 324 /* NOTREACHED */ 325 } 326 return (EINVAL); 327 } 328 329 /* struct kse_wakeup_args { 330 struct kse_mailbox *mbx; 331 }; */ 332 int 333 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 334 { 335 struct proc *p; 336 struct kse *ke, *ke2; 337 struct ksegrp *kg; 338 339 p = td->td_proc; 340 /* KSE-enabled processes only, please. */ 341 if (!(p->p_flag & P_KSES)) 342 return EINVAL; 343 if (td->td_standin == NULL) 344 td->td_standin = thread_alloc(); 345 ke = NULL; 346 mtx_lock_spin(&sched_lock); 347 if (uap->mbx) { 348 FOREACH_KSEGRP_IN_PROC(p, kg) { 349 FOREACH_KSE_IN_GROUP(kg, ke2) { 350 if (ke2->ke_mailbox != uap->mbx) 351 continue; 352 if (ke2->ke_state == KES_IDLE) { 353 ke = ke2; 354 goto found; 355 } else { 356 mtx_unlock_spin(&sched_lock); 357 td->td_retval[0] = 0; 358 td->td_retval[1] = 0; 359 return 0; 360 } 361 } 362 } 363 } else { 364 kg = td->td_ksegrp; 365 ke = TAILQ_FIRST(&kg->kg_iq); 366 } 367 if (ke == NULL) { 368 mtx_unlock_spin(&sched_lock); 369 return ESRCH; 370 } 371 found: 372 thread_schedule_upcall(td, ke); 373 mtx_unlock_spin(&sched_lock); 374 td->td_retval[0] = 0; 375 td->td_retval[1] = 0; 376 return 0; 377 } 378 379 /* 380 * No new KSEG: first call: use current KSE, don't schedule an upcall 381 * All other situations, do allocate a new KSE and schedule an upcall on it. 382 */ 383 /* struct kse_create_args { 384 struct kse_mailbox *mbx; 385 int newgroup; 386 }; */ 387 int 388 kse_create(struct thread *td, struct kse_create_args *uap) 389 { 390 struct kse *newke; 391 struct kse *ke; 392 struct ksegrp *newkg; 393 struct ksegrp *kg; 394 struct proc *p; 395 struct kse_mailbox mbx; 396 int err; 397 398 p = td->td_proc; 399 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 400 return (err); 401 402 p->p_flag |= P_KSES; /* easier to just set it than to test and set */ 403 kg = td->td_ksegrp; 404 if (uap->newgroup) { 405 /* 406 * If we want a new KSEGRP it doesn't matter whether 407 * we have already fired up KSE mode before or not. 408 * We put the process in KSE mode and create a new KSEGRP 409 * and KSE. If our KSE has not got a mailbox yet then 410 * that doesn't matter, just leave it that way. It will 411 * ensure that this thread stay BOUND. It's possible 412 * that the call came form a threaded library and the main 413 * program knows nothing of threads. 414 */ 415 newkg = ksegrp_alloc(); 416 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 417 kg_startzero, kg_endzero)); 418 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 419 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 420 newke = kse_alloc(); 421 } else { 422 /* 423 * Otherwise, if we have already set this KSE 424 * to have a mailbox, we want to make another KSE here, 425 * but only if there are not already the limit, which 426 * is 1 per CPU max. 427 * 428 * If the current KSE doesn't have a mailbox we just use it 429 * and give it one. 430 * 431 * Because we don't like to access 432 * the KSE outside of schedlock if we are UNBOUND, 433 * (because it can change if we are preempted by an interrupt) 434 * we can deduce it as having a mailbox if we are UNBOUND, 435 * and only need to actually look at it if we are BOUND, 436 * which is safe. 437 */ 438 if ((td->td_flags & TDF_UNBOUND) || td->td_kse->ke_mailbox) { 439 #if 0 /* while debugging */ 440 #ifdef SMP 441 if (kg->kg_kses > mp_ncpus) 442 #endif 443 return (EPROCLIM); 444 #endif 445 newke = kse_alloc(); 446 } else { 447 newke = NULL; 448 } 449 newkg = NULL; 450 } 451 if (newke) { 452 bzero(&newke->ke_startzero, RANGEOF(struct kse, 453 ke_startzero, ke_endzero)); 454 #if 0 455 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 456 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 457 #endif 458 PROC_LOCK(p); 459 if (SIGPENDING(p)) 460 newke->ke_flags |= KEF_ASTPENDING; 461 PROC_UNLOCK(p); 462 /* For the first call this may not have been set */ 463 if (td->td_standin == NULL) { 464 td->td_standin = thread_alloc(); 465 } 466 mtx_lock_spin(&sched_lock); 467 if (newkg) 468 ksegrp_link(newkg, p); 469 else 470 newkg = kg; 471 kse_link(newke, newkg); 472 newke->ke_mailbox = uap->mbx; 473 newke->ke_upcall = mbx.km_func; 474 bcopy(&mbx.km_stack, &newke->ke_stack, sizeof(stack_t)); 475 thread_schedule_upcall(td, newke); 476 mtx_unlock_spin(&sched_lock); 477 } else { 478 /* 479 * If we didn't allocate a new KSE then the we are using 480 * the exisiting (BOUND) kse. 481 */ 482 ke = td->td_kse; 483 ke->ke_mailbox = uap->mbx; 484 ke->ke_upcall = mbx.km_func; 485 bcopy(&mbx.km_stack, &ke->ke_stack, sizeof(stack_t)); 486 } 487 /* 488 * Fill out the KSE-mode specific fields of the new kse. 489 */ 490 491 td->td_retval[0] = 0; 492 td->td_retval[1] = 0; 493 return (0); 494 } 495 496 /* 497 * Fill a ucontext_t with a thread's context information. 498 * 499 * This is an analogue to getcontext(3). 500 */ 501 void 502 thread_getcontext(struct thread *td, ucontext_t *uc) 503 { 504 505 /* 506 * XXX this is declared in a MD include file, i386/include/ucontext.h but 507 * is used in MI code. 508 */ 509 #ifdef __i386__ 510 get_mcontext(td, &uc->uc_mcontext); 511 #endif 512 uc->uc_sigmask = td->td_proc->p_sigmask; 513 } 514 515 /* 516 * Set a thread's context from a ucontext_t. 517 * 518 * This is an analogue to setcontext(3). 519 */ 520 int 521 thread_setcontext(struct thread *td, ucontext_t *uc) 522 { 523 int ret; 524 525 /* 526 * XXX this is declared in a MD include file, i386/include/ucontext.h but 527 * is used in MI code. 528 */ 529 #ifdef __i386__ 530 ret = set_mcontext(td, &uc->uc_mcontext); 531 #else 532 ret = ENOSYS; 533 #endif 534 if (ret == 0) { 535 SIG_CANTMASK(uc->uc_sigmask); 536 PROC_LOCK(td->td_proc); 537 td->td_proc->p_sigmask = uc->uc_sigmask; 538 PROC_UNLOCK(td->td_proc); 539 } 540 return (ret); 541 } 542 543 /* 544 * Initialize global thread allocation resources. 545 */ 546 void 547 threadinit(void) 548 { 549 550 #ifndef __ia64__ 551 thread_zone = uma_zcreate("THREAD", sizeof (struct thread), 552 thread_ctor, thread_dtor, thread_init, thread_fini, 553 UMA_ALIGN_CACHE, 0); 554 #else 555 /* 556 * XXX the ia64 kstack allocator is really lame and is at the mercy 557 * of contigmallloc(). This hackery is to pre-construct a whole 558 * pile of thread structures with associated kernel stacks early 559 * in the system startup while contigmalloc() still works. Once we 560 * have them, keep them. Sigh. 561 */ 562 thread_zone = uma_zcreate("THREAD", sizeof (struct thread), 563 thread_ctor, thread_dtor, thread_init, thread_fini, 564 UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); 565 uma_prealloc(thread_zone, 512); /* XXX arbitary */ 566 #endif 567 ksegrp_zone = uma_zcreate("KSEGRP", sizeof (struct ksegrp), 568 NULL, NULL, NULL, NULL, 569 UMA_ALIGN_CACHE, 0); 570 kse_zone = uma_zcreate("KSE", sizeof (struct kse), 571 NULL, NULL, NULL, NULL, 572 UMA_ALIGN_CACHE, 0); 573 } 574 575 /* 576 * Stash an embarasingly extra thread into the zombie thread queue. 577 */ 578 void 579 thread_stash(struct thread *td) 580 { 581 mtx_lock_spin(&zombie_thread_lock); 582 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 583 mtx_unlock_spin(&zombie_thread_lock); 584 } 585 586 /* 587 * Stash an embarasingly extra kse into the zombie kse queue. 588 */ 589 void 590 kse_stash(struct kse *ke) 591 { 592 mtx_lock_spin(&zombie_thread_lock); 593 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 594 mtx_unlock_spin(&zombie_thread_lock); 595 } 596 597 /* 598 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 599 */ 600 void 601 ksegrp_stash(struct ksegrp *kg) 602 { 603 mtx_lock_spin(&zombie_thread_lock); 604 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 605 mtx_unlock_spin(&zombie_thread_lock); 606 } 607 608 /* 609 * Reap zombie threads. 610 */ 611 void 612 thread_reap(void) 613 { 614 struct thread *td_first, *td_next; 615 struct kse *ke_first, *ke_next; 616 struct ksegrp *kg_first, * kg_next; 617 618 /* 619 * don't even bother to lock if none at this instant 620 * We really don't care about the next instant.. 621 */ 622 if ((!TAILQ_EMPTY(&zombie_threads)) 623 || (!TAILQ_EMPTY(&zombie_kses)) 624 || (!TAILQ_EMPTY(&zombie_ksegrps))) { 625 mtx_lock_spin(&zombie_thread_lock); 626 td_first = TAILQ_FIRST(&zombie_threads); 627 ke_first = TAILQ_FIRST(&zombie_kses); 628 kg_first = TAILQ_FIRST(&zombie_ksegrps); 629 if (td_first) 630 TAILQ_INIT(&zombie_threads); 631 if (ke_first) 632 TAILQ_INIT(&zombie_kses); 633 if (kg_first) 634 TAILQ_INIT(&zombie_ksegrps); 635 mtx_unlock_spin(&zombie_thread_lock); 636 while (td_first) { 637 td_next = TAILQ_NEXT(td_first, td_runq); 638 thread_free(td_first); 639 td_first = td_next; 640 } 641 while (ke_first) { 642 ke_next = TAILQ_NEXT(ke_first, ke_procq); 643 kse_free(ke_first); 644 ke_first = ke_next; 645 } 646 while (kg_first) { 647 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 648 ksegrp_free(kg_first); 649 kg_first = kg_next; 650 } 651 } 652 } 653 654 /* 655 * Allocate a ksegrp. 656 */ 657 struct ksegrp * 658 ksegrp_alloc(void) 659 { 660 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 661 } 662 663 /* 664 * Allocate a kse. 665 */ 666 struct kse * 667 kse_alloc(void) 668 { 669 return (uma_zalloc(kse_zone, M_WAITOK)); 670 } 671 672 /* 673 * Allocate a thread. 674 */ 675 struct thread * 676 thread_alloc(void) 677 { 678 thread_reap(); /* check if any zombies to get */ 679 return (uma_zalloc(thread_zone, M_WAITOK)); 680 } 681 682 /* 683 * Deallocate a ksegrp. 684 */ 685 void 686 ksegrp_free(struct ksegrp *td) 687 { 688 uma_zfree(ksegrp_zone, td); 689 } 690 691 /* 692 * Deallocate a kse. 693 */ 694 void 695 kse_free(struct kse *td) 696 { 697 uma_zfree(kse_zone, td); 698 } 699 700 /* 701 * Deallocate a thread. 702 */ 703 void 704 thread_free(struct thread *td) 705 { 706 uma_zfree(thread_zone, td); 707 } 708 709 /* 710 * Store the thread context in the UTS's mailbox. 711 * then add the mailbox at the head of a list we are building in user space. 712 * The list is anchored in the ksegrp structure. 713 */ 714 int 715 thread_export_context(struct thread *td) 716 { 717 struct proc *p; 718 struct ksegrp *kg; 719 uintptr_t mbx; 720 void *addr; 721 int error; 722 ucontext_t uc; 723 724 p = td->td_proc; 725 kg = td->td_ksegrp; 726 727 /* Export the user/machine context. */ 728 #if 0 729 addr = (caddr_t)td->td_mailbox + 730 offsetof(struct kse_thr_mailbox, tm_context); 731 #else /* if user pointer arithmetic is valid in the kernel */ 732 addr = (void *)(&td->td_mailbox->tm_context); 733 #endif 734 error = copyin(addr, &uc, sizeof(ucontext_t)); 735 if (error == 0) { 736 thread_getcontext(td, &uc); 737 error = copyout(&uc, addr, sizeof(ucontext_t)); 738 739 } 740 if (error) { 741 PROC_LOCK(p); 742 psignal(p, SIGSEGV); 743 PROC_UNLOCK(p); 744 return (error); 745 } 746 /* get address in latest mbox of list pointer */ 747 #if 0 748 addr = (caddr_t)td->td_mailbox 749 + offsetof(struct kse_thr_mailbox , tm_next); 750 #else /* if user pointer arithmetic is valid in the kernel */ 751 addr = (void *)(&td->td_mailbox->tm_next); 752 #endif 753 /* 754 * Put the saved address of the previous first 755 * entry into this one 756 */ 757 for (;;) { 758 mbx = (uintptr_t)kg->kg_completed; 759 if (suword(addr, mbx)) { 760 PROC_LOCK(p); 761 psignal(p, SIGSEGV); 762 PROC_UNLOCK(p); 763 return (EFAULT); 764 } 765 PROC_LOCK(p); 766 if (mbx == (uintptr_t)kg->kg_completed) { 767 kg->kg_completed = td->td_mailbox; 768 PROC_UNLOCK(p); 769 break; 770 } 771 PROC_UNLOCK(p); 772 } 773 return (0); 774 } 775 776 /* 777 * Take the list of completed mailboxes for this KSEGRP and put them on this 778 * KSE's mailbox as it's the next one going up. 779 */ 780 static int 781 thread_link_mboxes(struct ksegrp *kg, struct kse *ke) 782 { 783 struct proc *p = kg->kg_proc; 784 void *addr; 785 uintptr_t mbx; 786 787 #if 0 788 addr = (caddr_t)ke->ke_mailbox 789 + offsetof(struct kse_mailbox, km_completed); 790 #else /* if user pointer arithmetic is valid in the kernel */ 791 addr = (void *)(&ke->ke_mailbox->km_completed); 792 #endif 793 for (;;) { 794 mbx = (uintptr_t)kg->kg_completed; 795 if (suword(addr, mbx)) { 796 PROC_LOCK(p); 797 psignal(p, SIGSEGV); 798 PROC_UNLOCK(p); 799 return (EFAULT); 800 } 801 /* XXXKSE could use atomic CMPXCH here */ 802 PROC_LOCK(p); 803 if (mbx == (uintptr_t)kg->kg_completed) { 804 kg->kg_completed = NULL; 805 PROC_UNLOCK(p); 806 break; 807 } 808 PROC_UNLOCK(p); 809 } 810 return (0); 811 } 812 813 /* 814 * Discard the current thread and exit from its context. 815 * 816 * Because we can't free a thread while we're operating under its context, 817 * push the current thread into our KSE's ke_tdspare slot, freeing the 818 * thread that might be there currently. Because we know that only this 819 * processor will run our KSE, we needn't worry about someone else grabbing 820 * our context before we do a cpu_throw. 821 */ 822 void 823 thread_exit(void) 824 { 825 struct thread *td; 826 struct kse *ke; 827 struct proc *p; 828 struct ksegrp *kg; 829 830 td = curthread; 831 kg = td->td_ksegrp; 832 p = td->td_proc; 833 ke = td->td_kse; 834 835 mtx_assert(&sched_lock, MA_OWNED); 836 KASSERT(p != NULL, ("thread exiting without a process")); 837 KASSERT(ke != NULL, ("thread exiting without a kse")); 838 KASSERT(kg != NULL, ("thread exiting without a kse group")); 839 PROC_LOCK_ASSERT(p, MA_OWNED); 840 CTR1(KTR_PROC, "thread_exit: thread %p", td); 841 KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 842 843 if (ke->ke_tdspare != NULL) { 844 thread_stash(ke->ke_tdspare); 845 ke->ke_tdspare = NULL; 846 } 847 if (td->td_standin != NULL) { 848 thread_stash(td->td_standin); 849 td->td_standin = NULL; 850 } 851 852 cpu_thread_exit(td); /* XXXSMP */ 853 854 /* 855 * The last thread is left attached to the process 856 * So that the whole bundle gets recycled. Skip 857 * all this stuff. 858 */ 859 if (p->p_numthreads > 1) { 860 /* 861 * Unlink this thread from its proc and the kseg. 862 * In keeping with the other structs we probably should 863 * have a thread_unlink() that does some of this but it 864 * would only be called from here (I think) so it would 865 * be a waste. (might be useful for proc_fini() as well.) 866 */ 867 TAILQ_REMOVE(&p->p_threads, td, td_plist); 868 p->p_numthreads--; 869 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 870 kg->kg_numthreads--; 871 /* 872 * The test below is NOT true if we are the 873 * sole exiting thread. P_STOPPED_SNGL is unset 874 * in exit1() after it is the only survivor. 875 */ 876 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 877 if (p->p_numthreads == p->p_suspcount) { 878 thread_unsuspend_one(p->p_singlethread); 879 } 880 } 881 882 /* Reassign this thread's KSE. */ 883 ke->ke_thread = NULL; 884 td->td_kse = NULL; 885 ke->ke_state = KES_UNQUEUED; 886 KASSERT((ke->ke_bound != td), 887 ("thread_exit: entered with ke_bound set")); 888 889 /* 890 * The reason for all this hoopla is 891 * an attempt to stop our thread stack from being freed 892 * until AFTER we have stopped running on it. 893 * Since we are under schedlock, almost any method where 894 * it is eventually freed by someone else is probably ok. 895 * (Especially if they do it under schedlock). We could 896 * almost free it here if we could be certain that 897 * the uma code wouldn't pull it apart immediatly, 898 * but unfortunatly we can not guarantee that. 899 * 900 * For threads that are exiting and NOT killing their 901 * KSEs we can just stash it in the KSE, however 902 * in the case where the KSE is also being deallocated, 903 * we need to store it somewhere else. It turns out that 904 * we will never free the last KSE, so there is always one 905 * other KSE available. We might as well just choose one 906 * and stash it there. Being under schedlock should make that 907 * safe. 908 * 909 * In borrower threads, we can stash it in the lender 910 * Where it won't be needed until this thread is long gone. 911 * Borrower threads can't kill their KSE anyhow, so even 912 * the KSE would be a safe place for them. It is not 913 * necessary to have a KSE (or KSEGRP) at all beyond this 914 * point, while we are under the protection of schedlock. 915 * 916 * Either give the KSE to another thread to use (or make 917 * it idle), or free it entirely, possibly along with its 918 * ksegrp if it's the last one. 919 */ 920 if (ke->ke_flags & KEF_EXIT) { 921 kse_unlink(ke); 922 /* 923 * Designate another KSE to hold our thread. 924 * Safe as long as we abide by whatever lock 925 * we control it with.. The other KSE will not 926 * be able to run it until we release the schelock, 927 * but we need to be careful about it deciding to 928 * write to the stack before then. Luckily 929 * I believe that while another thread's 930 * standin thread can be used in this way, the 931 * spare thread for the KSE cannot be used without 932 * holding schedlock at least once. 933 */ 934 ke = FIRST_KSE_IN_PROC(p); 935 } else { 936 kse_reassign(ke); 937 } 938 if (ke->ke_bound) { 939 /* 940 * WE are a borrower.. 941 * stash our thread with the owner. 942 */ 943 if (ke->ke_bound->td_standin) { 944 thread_stash(ke->ke_bound->td_standin); 945 } 946 ke->ke_bound->td_standin = td; 947 } else { 948 if (ke->ke_tdspare != NULL) { 949 thread_stash(ke->ke_tdspare); 950 ke->ke_tdspare = NULL; 951 } 952 ke->ke_tdspare = td; 953 } 954 PROC_UNLOCK(p); 955 td->td_state = TDS_INACTIVE; 956 td->td_proc = NULL; 957 td->td_ksegrp = NULL; 958 td->td_last_kse = NULL; 959 } else { 960 PROC_UNLOCK(p); 961 } 962 963 cpu_throw(); 964 /* NOTREACHED */ 965 } 966 967 /* 968 * Link a thread to a process. 969 * set up anything that needs to be initialized for it to 970 * be used by the process. 971 * 972 * Note that we do not link to the proc's ucred here. 973 * The thread is linked as if running but no KSE assigned. 974 */ 975 void 976 thread_link(struct thread *td, struct ksegrp *kg) 977 { 978 struct proc *p; 979 980 p = kg->kg_proc; 981 td->td_state = TDS_INACTIVE; 982 td->td_proc = p; 983 td->td_ksegrp = kg; 984 td->td_last_kse = NULL; 985 986 LIST_INIT(&td->td_contested); 987 callout_init(&td->td_slpcallout, 1); 988 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 989 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 990 p->p_numthreads++; 991 kg->kg_numthreads++; 992 if (oiks_debug && p->p_numthreads > max_threads_per_proc) { 993 printf("OIKS %d\n", p->p_numthreads); 994 if (oiks_debug > 1) 995 Debugger("OIKS"); 996 } 997 td->td_kse = NULL; 998 } 999 1000 void 1001 kse_purge(struct proc *p, struct thread *td) 1002 { 1003 struct kse *ke; 1004 struct ksegrp *kg; 1005 1006 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1007 mtx_lock_spin(&sched_lock); 1008 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1009 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1010 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1011 kg->kg_idle_kses--; 1012 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1013 kg->kg_kses--; 1014 if (ke->ke_tdspare) 1015 thread_stash(ke->ke_tdspare); 1016 kse_stash(ke); 1017 } 1018 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1019 p->p_numksegrps--; 1020 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1021 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1022 ("wrong kg_kses")); 1023 if (kg != td->td_ksegrp) { 1024 ksegrp_stash(kg); 1025 } 1026 } 1027 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1028 p->p_numksegrps++; 1029 mtx_unlock_spin(&sched_lock); 1030 } 1031 1032 1033 /* 1034 * Create a thread and schedule it for upcall on the KSE given. 1035 */ 1036 struct thread * 1037 thread_schedule_upcall(struct thread *td, struct kse *ke) 1038 { 1039 struct thread *td2; 1040 struct ksegrp *kg; 1041 int newkse; 1042 1043 mtx_assert(&sched_lock, MA_OWNED); 1044 newkse = (ke != td->td_kse); 1045 1046 /* 1047 * If the kse is already owned by another thread then we can't 1048 * schedule an upcall because the other thread must be BOUND 1049 * which means it is not in a position to take an upcall. 1050 * We must be borrowing the KSE to allow us to complete some in-kernel 1051 * work. When we complete, the Bound thread will have teh chance to 1052 * complete. This thread will sleep as planned. Hopefully there will 1053 * eventually be un unbound thread that can be converted to an 1054 * upcall to report the completion of this thread. 1055 */ 1056 if (ke->ke_bound && ((ke->ke_bound->td_flags & TDF_UNBOUND) == 0)) { 1057 return (NULL); 1058 } 1059 KASSERT((ke->ke_bound == NULL), ("kse already bound")); 1060 1061 if (ke->ke_state == KES_IDLE) { 1062 kg = ke->ke_ksegrp; 1063 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1064 kg->kg_idle_kses--; 1065 ke->ke_state = KES_UNQUEUED; 1066 } 1067 if ((td2 = td->td_standin) != NULL) { 1068 td->td_standin = NULL; 1069 } else { 1070 if (newkse) 1071 panic("no reserve thread when called with a new kse"); 1072 /* 1073 * If called from (e.g.) sleep and we do not have 1074 * a reserve thread, then we've used it, so do not 1075 * create an upcall. 1076 */ 1077 return(NULL); 1078 } 1079 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1080 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1081 bzero(&td2->td_startzero, 1082 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1083 bcopy(&td->td_startcopy, &td2->td_startcopy, 1084 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1085 thread_link(td2, ke->ke_ksegrp); 1086 cpu_set_upcall(td2, td->td_pcb); 1087 1088 /* 1089 * XXXKSE do we really need this? (default values for the 1090 * frame). 1091 */ 1092 bcopy(td->td_frame, td2->td_frame, sizeof(struct trapframe)); 1093 1094 /* 1095 * Bind the new thread to the KSE, 1096 * and if it's our KSE, lend it back to ourself 1097 * so we can continue running. 1098 */ 1099 td2->td_ucred = crhold(td->td_ucred); 1100 td2->td_flags = TDF_UPCALLING; /* note: BOUND */ 1101 td2->td_kse = ke; 1102 td2->td_state = TDS_CAN_RUN; 1103 td2->td_inhibitors = 0; 1104 /* 1105 * If called from msleep(), we are working on the current 1106 * KSE so fake that we borrowed it. If called from 1107 * kse_create(), don't, as we have a new kse too. 1108 */ 1109 if (!newkse) { 1110 /* 1111 * This thread will be scheduled when the current thread 1112 * blocks, exits or tries to enter userspace, (which ever 1113 * happens first). When that happens the KSe will "revert" 1114 * to this thread in a BOUND manner. Since we are called 1115 * from msleep() this is going to be "very soon" in nearly 1116 * all cases. 1117 */ 1118 ke->ke_bound = td2; 1119 TD_SET_LOAN(td2); 1120 } else { 1121 ke->ke_bound = NULL; 1122 ke->ke_thread = td2; 1123 ke->ke_state = KES_THREAD; 1124 setrunqueue(td2); 1125 } 1126 return (td2); /* bogus.. should be a void function */ 1127 } 1128 1129 /* 1130 * Schedule an upcall to notify a KSE process recieved signals. 1131 * 1132 * XXX - Modifying a sigset_t like this is totally bogus. 1133 */ 1134 struct thread * 1135 signal_upcall(struct proc *p, int sig) 1136 { 1137 struct thread *td, *td2; 1138 struct kse *ke; 1139 sigset_t ss; 1140 int error; 1141 1142 PROC_LOCK_ASSERT(p, MA_OWNED); 1143 return (NULL); 1144 1145 td = FIRST_THREAD_IN_PROC(p); 1146 ke = td->td_kse; 1147 PROC_UNLOCK(p); 1148 error = copyin(&ke->ke_mailbox->km_sigscaught, &ss, sizeof(sigset_t)); 1149 PROC_LOCK(p); 1150 if (error) 1151 return (NULL); 1152 SIGADDSET(ss, sig); 1153 PROC_UNLOCK(p); 1154 error = copyout(&ss, &ke->ke_mailbox->km_sigscaught, sizeof(sigset_t)); 1155 PROC_LOCK(p); 1156 if (error) 1157 return (NULL); 1158 if (td->td_standin == NULL) 1159 td->td_standin = thread_alloc(); 1160 mtx_lock_spin(&sched_lock); 1161 td2 = thread_schedule_upcall(td, ke); /* Bogus JRE */ 1162 mtx_unlock_spin(&sched_lock); 1163 return (td2); 1164 } 1165 1166 /* 1167 * setup done on the thread when it enters the kernel. 1168 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1169 */ 1170 void 1171 thread_user_enter(struct proc *p, struct thread *td) 1172 { 1173 struct kse *ke; 1174 1175 /* 1176 * First check that we shouldn't just abort. 1177 * But check if we are the single thread first! 1178 * XXX p_singlethread not locked, but should be safe. 1179 */ 1180 if ((p->p_flag & P_WEXIT) && (p->p_singlethread != td)) { 1181 PROC_LOCK(p); 1182 mtx_lock_spin(&sched_lock); 1183 thread_exit(); 1184 /* NOTREACHED */ 1185 } 1186 1187 /* 1188 * If we are doing a syscall in a KSE environment, 1189 * note where our mailbox is. There is always the 1190 * possibility that we could do this lazily (in sleep()), 1191 * but for now do it every time. 1192 */ 1193 ke = td->td_kse; 1194 if (ke->ke_mailbox != NULL) { 1195 #if 0 1196 td->td_mailbox = (void *)fuword((caddr_t)ke->ke_mailbox 1197 + offsetof(struct kse_mailbox, km_curthread)); 1198 #else /* if user pointer arithmetic is ok in the kernel */ 1199 td->td_mailbox = 1200 (void *)fuword( (void *)&ke->ke_mailbox->km_curthread); 1201 #endif 1202 if ((td->td_mailbox == NULL) || 1203 (td->td_mailbox == (void *)-1)) { 1204 td->td_mailbox = NULL; /* single thread it.. */ 1205 td->td_flags &= ~TDF_UNBOUND; 1206 } else { 1207 if (td->td_standin == NULL) 1208 td->td_standin = thread_alloc(); 1209 td->td_flags |= TDF_UNBOUND; 1210 } 1211 } 1212 } 1213 1214 /* 1215 * The extra work we go through if we are a threaded process when we 1216 * return to userland. 1217 * 1218 * If we are a KSE process and returning to user mode, check for 1219 * extra work to do before we return (e.g. for more syscalls 1220 * to complete first). If we were in a critical section, we should 1221 * just return to let it finish. Same if we were in the UTS (in 1222 * which case the mailbox's context's busy indicator will be set). 1223 * The only traps we suport will have set the mailbox. 1224 * We will clear it here. 1225 */ 1226 int 1227 thread_userret(struct thread *td, struct trapframe *frame) 1228 { 1229 int error; 1230 int unbound; 1231 struct kse *ke; 1232 struct ksegrp *kg; 1233 struct thread *td2; 1234 struct proc *p; 1235 1236 error = 0; 1237 1238 unbound = td->td_flags & TDF_UNBOUND; 1239 1240 kg = td->td_ksegrp; 1241 p = td->td_proc; 1242 1243 /* 1244 * Originally bound threads never upcall but they may 1245 * loan out their KSE at this point. 1246 * Upcalls imply bound.. They also may want to do some Philantropy. 1247 * Unbound threads on the other hand either yield to other work 1248 * or transform into an upcall. 1249 * (having saved their context to user space in both cases) 1250 */ 1251 if (unbound ) { 1252 /* 1253 * We are an unbound thread, looking to return to 1254 * user space. 1255 * THere are several possibilities: 1256 * 1) we are using a borrowed KSE. save state and exit. 1257 * kse_reassign() will recycle the kse as needed, 1258 * 2) we are not.. save state, and then convert ourself 1259 * to be an upcall, bound to the KSE. 1260 * if there are others that need the kse, 1261 * give them a chance by doing an mi_switch(). 1262 * Because we are bound, control will eventually return 1263 * to us here. 1264 * *** 1265 * Save the thread's context, and link it 1266 * into the KSEGRP's list of completed threads. 1267 */ 1268 error = thread_export_context(td); 1269 td->td_mailbox = NULL; 1270 if (error) { 1271 /* 1272 * If we are not running on a borrowed KSE, then 1273 * failing to do the KSE operation just defaults 1274 * back to synchonous operation, so just return from 1275 * the syscall. If it IS borrowed, there is nothing 1276 * we can do. We just lose that context. We 1277 * probably should note this somewhere and send 1278 * the process a signal. 1279 */ 1280 PROC_LOCK(td->td_proc); 1281 psignal(td->td_proc, SIGSEGV); 1282 mtx_lock_spin(&sched_lock); 1283 if (td->td_kse->ke_bound == NULL) { 1284 td->td_flags &= ~TDF_UNBOUND; 1285 PROC_UNLOCK(td->td_proc); 1286 mtx_unlock_spin(&sched_lock); 1287 return (error); /* go sync */ 1288 } 1289 thread_exit(); 1290 } 1291 1292 /* 1293 * if the KSE is owned and we are borrowing it, 1294 * don't make an upcall, just exit so that the owner 1295 * can get its KSE if it wants it. 1296 * Our context is already safely stored for later 1297 * use by the UTS. 1298 */ 1299 PROC_LOCK(p); 1300 mtx_lock_spin(&sched_lock); 1301 if (td->td_kse->ke_bound) { 1302 thread_exit(); 1303 } 1304 PROC_UNLOCK(p); 1305 1306 /* 1307 * Turn ourself into a bound upcall. 1308 * We will rely on kse_reassign() 1309 * to make us run at a later time. 1310 * We should look just like a sheduled upcall 1311 * from msleep() or cv_wait(). 1312 */ 1313 td->td_flags &= ~TDF_UNBOUND; 1314 td->td_flags |= TDF_UPCALLING; 1315 /* Only get here if we have become an upcall */ 1316 1317 } else { 1318 mtx_lock_spin(&sched_lock); 1319 } 1320 /* 1321 * We ARE going back to userland with this KSE. 1322 * Check for threads that need to borrow it. 1323 * Optimisation: don't call mi_switch if no-one wants the KSE. 1324 * Any other thread that comes ready after this missed the boat. 1325 */ 1326 ke = td->td_kse; 1327 if ((td2 = kg->kg_last_assigned)) 1328 td2 = TAILQ_NEXT(td2, td_runq); 1329 else 1330 td2 = TAILQ_FIRST(&kg->kg_runq); 1331 if (td2) { 1332 /* 1333 * force a switch to more urgent 'in kernel' 1334 * work. Control will return to this thread 1335 * when there is no more work to do. 1336 * kse_reassign() will do tha for us. 1337 */ 1338 TD_SET_LOAN(td); 1339 ke->ke_bound = td; 1340 ke->ke_thread = NULL; 1341 mi_switch(); /* kse_reassign() will (re)find td2 */ 1342 } 1343 mtx_unlock_spin(&sched_lock); 1344 1345 /* 1346 * Optimisation: 1347 * Ensure that we have a spare thread available, 1348 * for when we re-enter the kernel. 1349 */ 1350 if (td->td_standin == NULL) { 1351 if (ke->ke_tdspare) { 1352 td->td_standin = ke->ke_tdspare; 1353 ke->ke_tdspare = NULL; 1354 } else { 1355 td->td_standin = thread_alloc(); 1356 } 1357 } 1358 1359 /* 1360 * To get here, we know there is no other need for our 1361 * KSE so we can proceed. If not upcalling, go back to 1362 * userspace. If we are, get the upcall set up. 1363 */ 1364 if ((td->td_flags & TDF_UPCALLING) == 0) 1365 return (0); 1366 1367 /* 1368 * We must be an upcall to get this far. 1369 * There is no more work to do and we are going to ride 1370 * this thead/KSE up to userland as an upcall. 1371 * Do the last parts of the setup needed for the upcall. 1372 */ 1373 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1374 td, td->td_proc->p_pid, td->td_proc->p_comm); 1375 1376 /* 1377 * Set user context to the UTS. 1378 */ 1379 cpu_set_upcall_kse(td, ke); 1380 1381 /* 1382 * Put any completed mailboxes on this KSE's list. 1383 */ 1384 error = thread_link_mboxes(kg, ke); 1385 if (error) 1386 goto bad; 1387 1388 /* 1389 * Set state and mailbox. 1390 * From now on we are just a bound outgoing process. 1391 * **Problem** userret is often called several times. 1392 * it would be nice if this all happenned only on the first time 1393 * through. (the scan for extra work etc.) 1394 */ 1395 td->td_flags &= ~TDF_UPCALLING; 1396 #if 0 1397 error = suword((caddr_t)ke->ke_mailbox + 1398 offsetof(struct kse_mailbox, km_curthread), 0); 1399 #else /* if user pointer arithmetic is ok in the kernel */ 1400 error = suword((caddr_t)&ke->ke_mailbox->km_curthread, 0); 1401 #endif 1402 if (!error) 1403 return (0); 1404 1405 bad: 1406 /* 1407 * Things are going to be so screwed we should just kill the process. 1408 * how do we do that? 1409 */ 1410 PROC_LOCK(td->td_proc); 1411 psignal(td->td_proc, SIGSEGV); 1412 PROC_UNLOCK(td->td_proc); 1413 return (error); /* go sync */ 1414 } 1415 1416 /* 1417 * Enforce single-threading. 1418 * 1419 * Returns 1 if the caller must abort (another thread is waiting to 1420 * exit the process or similar). Process is locked! 1421 * Returns 0 when you are successfully the only thread running. 1422 * A process has successfully single threaded in the suspend mode when 1423 * There are no threads in user mode. Threads in the kernel must be 1424 * allowed to continue until they get to the user boundary. They may even 1425 * copy out their return values and data before suspending. They may however be 1426 * accellerated in reaching the user boundary as we will wake up 1427 * any sleeping threads that are interruptable. (PCATCH). 1428 */ 1429 int 1430 thread_single(int force_exit) 1431 { 1432 struct thread *td; 1433 struct thread *td2; 1434 struct proc *p; 1435 1436 td = curthread; 1437 p = td->td_proc; 1438 PROC_LOCK_ASSERT(p, MA_OWNED); 1439 KASSERT((td != NULL), ("curthread is NULL")); 1440 1441 if ((p->p_flag & P_KSES) == 0) 1442 return (0); 1443 1444 /* Is someone already single threading? */ 1445 if (p->p_singlethread) 1446 return (1); 1447 1448 if (force_exit == SINGLE_EXIT) 1449 p->p_flag |= P_SINGLE_EXIT; 1450 else 1451 p->p_flag &= ~P_SINGLE_EXIT; 1452 p->p_flag |= P_STOPPED_SINGLE; 1453 p->p_singlethread = td; 1454 /* XXXKSE Which lock protects the below values? */ 1455 while ((p->p_numthreads - p->p_suspcount) != 1) { 1456 mtx_lock_spin(&sched_lock); 1457 FOREACH_THREAD_IN_PROC(p, td2) { 1458 if (td2 == td) 1459 continue; 1460 if (TD_IS_INHIBITED(td2)) { 1461 if (force_exit == SINGLE_EXIT) { 1462 if (TD_IS_SUSPENDED(td2)) { 1463 thread_unsuspend_one(td2); 1464 } 1465 if (TD_ON_SLEEPQ(td2) && 1466 (td2->td_flags & TDF_SINTR)) { 1467 if (td2->td_flags & TDF_CVWAITQ) 1468 cv_abort(td2); 1469 else 1470 abortsleep(td2); 1471 } 1472 } else { 1473 if (TD_IS_SUSPENDED(td2)) 1474 continue; 1475 /* maybe other inhibitted states too? */ 1476 if (TD_IS_SLEEPING(td2)) 1477 thread_suspend_one(td2); 1478 } 1479 } 1480 } 1481 /* 1482 * Maybe we suspended some threads.. was it enough? 1483 */ 1484 if ((p->p_numthreads - p->p_suspcount) == 1) { 1485 mtx_unlock_spin(&sched_lock); 1486 break; 1487 } 1488 1489 /* 1490 * Wake us up when everyone else has suspended. 1491 * In the mean time we suspend as well. 1492 */ 1493 thread_suspend_one(td); 1494 mtx_unlock(&Giant); 1495 PROC_UNLOCK(p); 1496 mi_switch(); 1497 mtx_unlock_spin(&sched_lock); 1498 mtx_lock(&Giant); 1499 PROC_LOCK(p); 1500 } 1501 if (force_exit == SINGLE_EXIT) 1502 kse_purge(p, td); 1503 return (0); 1504 } 1505 1506 /* 1507 * Called in from locations that can safely check to see 1508 * whether we have to suspend or at least throttle for a 1509 * single-thread event (e.g. fork). 1510 * 1511 * Such locations include userret(). 1512 * If the "return_instead" argument is non zero, the thread must be able to 1513 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 1514 * 1515 * The 'return_instead' argument tells the function if it may do a 1516 * thread_exit() or suspend, or whether the caller must abort and back 1517 * out instead. 1518 * 1519 * If the thread that set the single_threading request has set the 1520 * P_SINGLE_EXIT bit in the process flags then this call will never return 1521 * if 'return_instead' is false, but will exit. 1522 * 1523 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 1524 *---------------+--------------------+--------------------- 1525 * 0 | returns 0 | returns 0 or 1 1526 * | when ST ends | immediatly 1527 *---------------+--------------------+--------------------- 1528 * 1 | thread exits | returns 1 1529 * | | immediatly 1530 * 0 = thread_exit() or suspension ok, 1531 * other = return error instead of stopping the thread. 1532 * 1533 * While a full suspension is under effect, even a single threading 1534 * thread would be suspended if it made this call (but it shouldn't). 1535 * This call should only be made from places where 1536 * thread_exit() would be safe as that may be the outcome unless 1537 * return_instead is set. 1538 */ 1539 int 1540 thread_suspend_check(int return_instead) 1541 { 1542 struct thread *td; 1543 struct proc *p; 1544 struct kse *ke; 1545 struct ksegrp *kg; 1546 1547 td = curthread; 1548 p = td->td_proc; 1549 kg = td->td_ksegrp; 1550 PROC_LOCK_ASSERT(p, MA_OWNED); 1551 while (P_SHOULDSTOP(p)) { 1552 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1553 KASSERT(p->p_singlethread != NULL, 1554 ("singlethread not set")); 1555 /* 1556 * The only suspension in action is a 1557 * single-threading. Single threader need not stop. 1558 * XXX Should be safe to access unlocked 1559 * as it can only be set to be true by us. 1560 */ 1561 if (p->p_singlethread == td) 1562 return (0); /* Exempt from stopping. */ 1563 } 1564 if (return_instead) 1565 return (1); 1566 1567 /* 1568 * If the process is waiting for us to exit, 1569 * this thread should just suicide. 1570 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 1571 */ 1572 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1573 mtx_lock_spin(&sched_lock); 1574 while (mtx_owned(&Giant)) 1575 mtx_unlock(&Giant); 1576 /* 1577 * free extra kses and ksegrps, we needn't worry 1578 * about if current thread is in same ksegrp as 1579 * p_singlethread and last kse in the group 1580 * could be killed, this is protected by kg_numthreads, 1581 * in this case, we deduce that kg_numthreads must > 1. 1582 */ 1583 ke = td->td_kse; 1584 if (ke->ke_bound == NULL && 1585 ((kg->kg_kses != 1) || (kg->kg_numthreads == 1))) 1586 ke->ke_flags |= KEF_EXIT; 1587 thread_exit(); 1588 } 1589 1590 /* 1591 * When a thread suspends, it just 1592 * moves to the processes's suspend queue 1593 * and stays there. 1594 * 1595 * XXXKSE if TDF_BOUND is true 1596 * it will not release it's KSE which might 1597 * lead to deadlock if there are not enough KSEs 1598 * to complete all waiting threads. 1599 * Maybe be able to 'lend' it out again. 1600 * (lent kse's can not go back to userland?) 1601 * and can only be lent in STOPPED state. 1602 */ 1603 mtx_lock_spin(&sched_lock); 1604 if ((p->p_flag & P_STOPPED_SIG) && 1605 (p->p_suspcount+1 == p->p_numthreads)) { 1606 mtx_unlock_spin(&sched_lock); 1607 PROC_LOCK(p->p_pptr); 1608 if ((p->p_pptr->p_procsig->ps_flag & 1609 PS_NOCLDSTOP) == 0) { 1610 psignal(p->p_pptr, SIGCHLD); 1611 } 1612 PROC_UNLOCK(p->p_pptr); 1613 mtx_lock_spin(&sched_lock); 1614 } 1615 mtx_assert(&Giant, MA_NOTOWNED); 1616 thread_suspend_one(td); 1617 PROC_UNLOCK(p); 1618 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1619 if (p->p_numthreads == p->p_suspcount) { 1620 thread_unsuspend_one(p->p_singlethread); 1621 } 1622 } 1623 p->p_stats->p_ru.ru_nivcsw++; 1624 mi_switch(); 1625 mtx_unlock_spin(&sched_lock); 1626 PROC_LOCK(p); 1627 } 1628 return (0); 1629 } 1630 1631 void 1632 thread_suspend_one(struct thread *td) 1633 { 1634 struct proc *p = td->td_proc; 1635 1636 mtx_assert(&sched_lock, MA_OWNED); 1637 p->p_suspcount++; 1638 TD_SET_SUSPENDED(td); 1639 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1640 /* 1641 * Hack: If we are suspending but are on the sleep queue 1642 * then we are in msleep or the cv equivalent. We 1643 * want to look like we have two Inhibitors. 1644 * May already be set.. doesn't matter. 1645 */ 1646 if (TD_ON_SLEEPQ(td)) 1647 TD_SET_SLEEPING(td); 1648 } 1649 1650 void 1651 thread_unsuspend_one(struct thread *td) 1652 { 1653 struct proc *p = td->td_proc; 1654 1655 mtx_assert(&sched_lock, MA_OWNED); 1656 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 1657 TD_CLR_SUSPENDED(td); 1658 p->p_suspcount--; 1659 setrunnable(td); 1660 } 1661 1662 /* 1663 * Allow all threads blocked by single threading to continue running. 1664 */ 1665 void 1666 thread_unsuspend(struct proc *p) 1667 { 1668 struct thread *td; 1669 1670 mtx_assert(&sched_lock, MA_OWNED); 1671 PROC_LOCK_ASSERT(p, MA_OWNED); 1672 if (!P_SHOULDSTOP(p)) { 1673 while (( td = TAILQ_FIRST(&p->p_suspended))) { 1674 thread_unsuspend_one(td); 1675 } 1676 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 1677 (p->p_numthreads == p->p_suspcount)) { 1678 /* 1679 * Stopping everything also did the job for the single 1680 * threading request. Now we've downgraded to single-threaded, 1681 * let it continue. 1682 */ 1683 thread_unsuspend_one(p->p_singlethread); 1684 } 1685 } 1686 1687 void 1688 thread_single_end(void) 1689 { 1690 struct thread *td; 1691 struct proc *p; 1692 1693 td = curthread; 1694 p = td->td_proc; 1695 PROC_LOCK_ASSERT(p, MA_OWNED); 1696 p->p_flag &= ~P_STOPPED_SINGLE; 1697 p->p_singlethread = NULL; 1698 /* 1699 * If there are other threads they mey now run, 1700 * unless of course there is a blanket 'stop order' 1701 * on the process. The single threader must be allowed 1702 * to continue however as this is a bad place to stop. 1703 */ 1704 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 1705 mtx_lock_spin(&sched_lock); 1706 while (( td = TAILQ_FIRST(&p->p_suspended))) { 1707 thread_unsuspend_one(td); 1708 } 1709 mtx_unlock_spin(&sched_lock); 1710 } 1711 } 1712 1713 1714