1 /* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/mutex.h> 38 #include <sys/proc.h> 39 #include <sys/smp.h> 40 #include <sys/sysctl.h> 41 #include <sys/sysproto.h> 42 #include <sys/filedesc.h> 43 #include <sys/sched.h> 44 #include <sys/signalvar.h> 45 #include <sys/sleepqueue.h> 46 #include <sys/sx.h> 47 #include <sys/tty.h> 48 #include <sys/turnstile.h> 49 #include <sys/user.h> 50 #include <sys/kse.h> 51 #include <sys/ktr.h> 52 #include <sys/ucontext.h> 53 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_object.h> 57 #include <vm/pmap.h> 58 #include <vm/uma.h> 59 #include <vm/vm_map.h> 60 61 #include <machine/frame.h> 62 63 /* 64 * KSEGRP related storage. 65 */ 66 static uma_zone_t ksegrp_zone; 67 static uma_zone_t kse_zone; 68 static uma_zone_t thread_zone; 69 static uma_zone_t upcall_zone; 70 71 /* DEBUG ONLY */ 72 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 73 static int thread_debug = 0; 74 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 75 &thread_debug, 0, "thread debug"); 76 77 static int max_threads_per_proc = 150; 78 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 79 &max_threads_per_proc, 0, "Limit on threads per proc"); 80 81 static int max_groups_per_proc = 50; 82 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 83 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 84 85 static int max_threads_hits; 86 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 87 &max_threads_hits, 0, ""); 88 89 static int virtual_cpu; 90 91 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 92 93 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 94 TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 95 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 96 TAILQ_HEAD(, kse_upcall) zombie_upcalls = 97 TAILQ_HEAD_INITIALIZER(zombie_upcalls); 98 struct mtx kse_zombie_lock; 99 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 100 101 static void kse_purge(struct proc *p, struct thread *td); 102 static void kse_purge_group(struct thread *td); 103 static int thread_update_usr_ticks(struct thread *td, int user); 104 static void thread_alloc_spare(struct thread *td, struct thread *spare); 105 106 static int 107 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 108 { 109 int error, new_val; 110 int def_val; 111 112 #ifdef SMP 113 def_val = mp_ncpus; 114 #else 115 def_val = 1; 116 #endif 117 if (virtual_cpu == 0) 118 new_val = def_val; 119 else 120 new_val = virtual_cpu; 121 error = sysctl_handle_int(oidp, &new_val, 0, req); 122 if (error != 0 || req->newptr == NULL) 123 return (error); 124 if (new_val < 0) 125 return (EINVAL); 126 virtual_cpu = new_val; 127 return (0); 128 } 129 130 /* DEBUG ONLY */ 131 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 132 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 133 "debug virtual cpus"); 134 135 /* 136 * Prepare a thread for use. 137 */ 138 static void 139 thread_ctor(void *mem, int size, void *arg) 140 { 141 struct thread *td; 142 143 td = (struct thread *)mem; 144 td->td_state = TDS_INACTIVE; 145 td->td_oncpu = NOCPU; 146 td->td_critnest = 1; 147 } 148 149 /* 150 * Reclaim a thread after use. 151 */ 152 static void 153 thread_dtor(void *mem, int size, void *arg) 154 { 155 struct thread *td; 156 157 td = (struct thread *)mem; 158 159 #ifdef INVARIANTS 160 /* Verify that this thread is in a safe state to free. */ 161 switch (td->td_state) { 162 case TDS_INHIBITED: 163 case TDS_RUNNING: 164 case TDS_CAN_RUN: 165 case TDS_RUNQ: 166 /* 167 * We must never unlink a thread that is in one of 168 * these states, because it is currently active. 169 */ 170 panic("bad state for thread unlinking"); 171 /* NOTREACHED */ 172 case TDS_INACTIVE: 173 break; 174 default: 175 panic("bad thread state"); 176 /* NOTREACHED */ 177 } 178 #endif 179 } 180 181 /* 182 * Initialize type-stable parts of a thread (when newly created). 183 */ 184 static void 185 thread_init(void *mem, int size) 186 { 187 struct thread *td; 188 189 td = (struct thread *)mem; 190 vm_thread_new(td, 0); 191 cpu_thread_setup(td); 192 td->td_sleepqueue = sleepq_alloc(); 193 td->td_turnstile = turnstile_alloc(); 194 td->td_sched = (struct td_sched *)&td[1]; 195 } 196 197 /* 198 * Tear down type-stable parts of a thread (just before being discarded). 199 */ 200 static void 201 thread_fini(void *mem, int size) 202 { 203 struct thread *td; 204 205 td = (struct thread *)mem; 206 turnstile_free(td->td_turnstile); 207 sleepq_free(td->td_sleepqueue); 208 vm_thread_dispose(td); 209 } 210 211 /* 212 * Initialize type-stable parts of a kse (when newly created). 213 */ 214 static void 215 kse_init(void *mem, int size) 216 { 217 struct kse *ke; 218 219 ke = (struct kse *)mem; 220 ke->ke_sched = (struct ke_sched *)&ke[1]; 221 } 222 223 /* 224 * Initialize type-stable parts of a ksegrp (when newly created). 225 */ 226 static void 227 ksegrp_init(void *mem, int size) 228 { 229 struct ksegrp *kg; 230 231 kg = (struct ksegrp *)mem; 232 kg->kg_sched = (struct kg_sched *)&kg[1]; 233 } 234 235 /* 236 * KSE is linked into kse group. 237 */ 238 void 239 kse_link(struct kse *ke, struct ksegrp *kg) 240 { 241 struct proc *p = kg->kg_proc; 242 243 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 244 kg->kg_kses++; 245 ke->ke_state = KES_UNQUEUED; 246 ke->ke_proc = p; 247 ke->ke_ksegrp = kg; 248 ke->ke_thread = NULL; 249 ke->ke_oncpu = NOCPU; 250 ke->ke_flags = 0; 251 } 252 253 void 254 kse_unlink(struct kse *ke) 255 { 256 struct ksegrp *kg; 257 258 mtx_assert(&sched_lock, MA_OWNED); 259 kg = ke->ke_ksegrp; 260 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 261 if (ke->ke_state == KES_IDLE) { 262 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 263 kg->kg_idle_kses--; 264 } 265 --kg->kg_kses; 266 /* 267 * Aggregate stats from the KSE 268 */ 269 kse_stash(ke); 270 } 271 272 void 273 ksegrp_link(struct ksegrp *kg, struct proc *p) 274 { 275 276 TAILQ_INIT(&kg->kg_threads); 277 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 278 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 279 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 280 TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 281 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 282 kg->kg_proc = p; 283 /* 284 * the following counters are in the -zero- section 285 * and may not need clearing 286 */ 287 kg->kg_numthreads = 0; 288 kg->kg_runnable = 0; 289 kg->kg_kses = 0; 290 kg->kg_runq_kses = 0; /* XXXKSE change name */ 291 kg->kg_idle_kses = 0; 292 kg->kg_numupcalls = 0; 293 /* link it in now that it's consistent */ 294 p->p_numksegrps++; 295 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 296 } 297 298 void 299 ksegrp_unlink(struct ksegrp *kg) 300 { 301 struct proc *p; 302 303 mtx_assert(&sched_lock, MA_OWNED); 304 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 305 KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 306 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 307 308 p = kg->kg_proc; 309 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 310 p->p_numksegrps--; 311 /* 312 * Aggregate stats from the KSE 313 */ 314 ksegrp_stash(kg); 315 } 316 317 struct kse_upcall * 318 upcall_alloc(void) 319 { 320 struct kse_upcall *ku; 321 322 ku = uma_zalloc(upcall_zone, M_WAITOK); 323 bzero(ku, sizeof(*ku)); 324 return (ku); 325 } 326 327 void 328 upcall_free(struct kse_upcall *ku) 329 { 330 331 uma_zfree(upcall_zone, ku); 332 } 333 334 void 335 upcall_link(struct kse_upcall *ku, struct ksegrp *kg) 336 { 337 338 mtx_assert(&sched_lock, MA_OWNED); 339 TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 340 ku->ku_ksegrp = kg; 341 kg->kg_numupcalls++; 342 } 343 344 void 345 upcall_unlink(struct kse_upcall *ku) 346 { 347 struct ksegrp *kg = ku->ku_ksegrp; 348 349 mtx_assert(&sched_lock, MA_OWNED); 350 KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 351 TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 352 kg->kg_numupcalls--; 353 upcall_stash(ku); 354 } 355 356 void 357 upcall_remove(struct thread *td) 358 { 359 360 if (td->td_upcall) { 361 td->td_upcall->ku_owner = NULL; 362 upcall_unlink(td->td_upcall); 363 td->td_upcall = 0; 364 } 365 } 366 367 /* 368 * For a newly created process, 369 * link up all the structures and its initial threads etc. 370 */ 371 void 372 proc_linkup(struct proc *p, struct ksegrp *kg, 373 struct kse *ke, struct thread *td) 374 { 375 376 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 377 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 378 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 379 p->p_numksegrps = 0; 380 p->p_numthreads = 0; 381 382 ksegrp_link(kg, p); 383 kse_link(ke, kg); 384 thread_link(td, kg); 385 } 386 387 #ifndef _SYS_SYSPROTO_H_ 388 struct kse_switchin_args { 389 const struct __mcontext *mcp; 390 long val; 391 long *loc; 392 }; 393 #endif 394 395 int 396 kse_switchin(struct thread *td, struct kse_switchin_args *uap) 397 { 398 mcontext_t mc; 399 int error; 400 401 error = (uap->mcp == NULL) ? EINVAL : 0; 402 if (!error) 403 error = copyin(uap->mcp, &mc, sizeof(mc)); 404 if (!error && uap->loc != NULL) 405 error = (suword(uap->loc, uap->val) != 0) ? EINVAL : 0; 406 if (!error) 407 error = set_mcontext(td, &mc); 408 return ((error == 0) ? EJUSTRETURN : error); 409 } 410 411 /* 412 struct kse_thr_interrupt_args { 413 struct kse_thr_mailbox * tmbx; 414 int cmd; 415 long data; 416 }; 417 */ 418 int 419 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 420 { 421 struct proc *p; 422 struct thread *td2; 423 424 p = td->td_proc; 425 426 if (!(p->p_flag & P_SA)) 427 return (EINVAL); 428 429 switch (uap->cmd) { 430 case KSE_INTR_SENDSIG: 431 if (uap->data < 0 || uap->data > _SIG_MAXSIG) 432 return (EINVAL); 433 case KSE_INTR_INTERRUPT: 434 case KSE_INTR_RESTART: 435 PROC_LOCK(p); 436 mtx_lock_spin(&sched_lock); 437 FOREACH_THREAD_IN_PROC(p, td2) { 438 if (td2->td_mailbox == uap->tmbx) 439 break; 440 } 441 if (td2 == NULL) { 442 mtx_unlock_spin(&sched_lock); 443 PROC_UNLOCK(p); 444 return (ESRCH); 445 } 446 if (uap->cmd == KSE_INTR_SENDSIG) { 447 if (uap->data > 0) { 448 td2->td_flags &= ~TDF_INTERRUPT; 449 mtx_unlock_spin(&sched_lock); 450 tdsignal(td2, (int)uap->data, SIGTARGET_TD); 451 } else { 452 mtx_unlock_spin(&sched_lock); 453 } 454 } else { 455 td2->td_flags |= TDF_INTERRUPT | TDF_ASTPENDING; 456 if (TD_CAN_UNBIND(td2)) 457 td2->td_upcall->ku_flags |= KUF_DOUPCALL; 458 if (uap->cmd == KSE_INTR_INTERRUPT) 459 td2->td_intrval = EINTR; 460 else 461 td2->td_intrval = ERESTART; 462 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) 463 sleepq_abort(td2); 464 mtx_unlock_spin(&sched_lock); 465 } 466 PROC_UNLOCK(p); 467 break; 468 case KSE_INTR_SIGEXIT: 469 if (uap->data < 1 || uap->data > _SIG_MAXSIG) 470 return (EINVAL); 471 PROC_LOCK(p); 472 sigexit(td, (int)uap->data); 473 break; 474 default: 475 return (EINVAL); 476 } 477 return (0); 478 } 479 480 /* 481 struct kse_exit_args { 482 register_t dummy; 483 }; 484 */ 485 int 486 kse_exit(struct thread *td, struct kse_exit_args *uap) 487 { 488 struct proc *p; 489 struct ksegrp *kg; 490 struct kse *ke; 491 struct kse_upcall *ku, *ku2; 492 int error, count; 493 494 p = td->td_proc; 495 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 496 return (EINVAL); 497 kg = td->td_ksegrp; 498 count = 0; 499 PROC_LOCK(p); 500 mtx_lock_spin(&sched_lock); 501 FOREACH_UPCALL_IN_GROUP(kg, ku2) { 502 if (ku2->ku_flags & KUF_EXITING) 503 count++; 504 } 505 if ((kg->kg_numupcalls - count) == 1 && 506 (kg->kg_numthreads > 1)) { 507 mtx_unlock_spin(&sched_lock); 508 PROC_UNLOCK(p); 509 return (EDEADLK); 510 } 511 ku->ku_flags |= KUF_EXITING; 512 mtx_unlock_spin(&sched_lock); 513 PROC_UNLOCK(p); 514 error = suword(&ku->ku_mailbox->km_flags, ku->ku_mflags|KMF_DONE); 515 PROC_LOCK(p); 516 if (error) 517 psignal(p, SIGSEGV); 518 mtx_lock_spin(&sched_lock); 519 upcall_remove(td); 520 ke = td->td_kse; 521 if (p->p_numthreads == 1) { 522 kse_purge(p, td); 523 p->p_flag &= ~P_SA; 524 mtx_unlock_spin(&sched_lock); 525 PROC_UNLOCK(p); 526 } else { 527 if (kg->kg_numthreads == 1) { /* Shutdown a group */ 528 kse_purge_group(td); 529 ke->ke_flags |= KEF_EXIT; 530 } 531 thread_stopped(p); 532 thread_exit(); 533 /* NOTREACHED */ 534 } 535 return (0); 536 } 537 538 /* 539 * Either becomes an upcall or waits for an awakening event and 540 * then becomes an upcall. Only error cases return. 541 */ 542 /* 543 struct kse_release_args { 544 struct timespec *timeout; 545 }; 546 */ 547 int 548 kse_release(struct thread *td, struct kse_release_args *uap) 549 { 550 struct proc *p; 551 struct ksegrp *kg; 552 struct kse_upcall *ku; 553 struct timespec timeout; 554 struct timeval tv; 555 sigset_t sigset; 556 int error; 557 558 p = td->td_proc; 559 kg = td->td_ksegrp; 560 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 561 return (EINVAL); 562 if (uap->timeout != NULL) { 563 if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 564 return (error); 565 TIMESPEC_TO_TIMEVAL(&tv, &timeout); 566 } 567 if (td->td_flags & TDF_SA) 568 td->td_pflags |= TDP_UPCALLING; 569 else { 570 ku->ku_mflags = fuword(&ku->ku_mailbox->km_flags); 571 if (ku->ku_mflags == -1) { 572 PROC_LOCK(p); 573 sigexit(td, SIGSEGV); 574 } 575 } 576 PROC_LOCK(p); 577 if (ku->ku_mflags & KMF_WAITSIGEVENT) { 578 /* UTS wants to wait for signal event */ 579 if (!(p->p_flag & P_SIGEVENT) && !(ku->ku_flags & KUF_DOUPCALL)) 580 error = msleep(&p->p_siglist, &p->p_mtx, PPAUSE|PCATCH, 581 "ksesigwait", (uap->timeout ? tvtohz(&tv) : 0)); 582 p->p_flag &= ~P_SIGEVENT; 583 sigset = p->p_siglist; 584 PROC_UNLOCK(p); 585 error = copyout(&sigset, &ku->ku_mailbox->km_sigscaught, 586 sizeof(sigset)); 587 } else { 588 if (! kg->kg_completed && !(ku->ku_flags & KUF_DOUPCALL)) { 589 kg->kg_upsleeps++; 590 error = msleep(&kg->kg_completed, &p->p_mtx, 591 PPAUSE|PCATCH, "kserel", 592 (uap->timeout ? tvtohz(&tv) : 0)); 593 kg->kg_upsleeps--; 594 } 595 PROC_UNLOCK(p); 596 } 597 if (ku->ku_flags & KUF_DOUPCALL) { 598 mtx_lock_spin(&sched_lock); 599 ku->ku_flags &= ~KUF_DOUPCALL; 600 mtx_unlock_spin(&sched_lock); 601 } 602 return (0); 603 } 604 605 /* struct kse_wakeup_args { 606 struct kse_mailbox *mbx; 607 }; */ 608 int 609 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 610 { 611 struct proc *p; 612 struct ksegrp *kg; 613 struct kse_upcall *ku; 614 struct thread *td2; 615 616 p = td->td_proc; 617 td2 = NULL; 618 ku = NULL; 619 /* KSE-enabled processes only, please. */ 620 if (!(p->p_flag & P_SA)) 621 return (EINVAL); 622 PROC_LOCK(p); 623 mtx_lock_spin(&sched_lock); 624 if (uap->mbx) { 625 FOREACH_KSEGRP_IN_PROC(p, kg) { 626 FOREACH_UPCALL_IN_GROUP(kg, ku) { 627 if (ku->ku_mailbox == uap->mbx) 628 break; 629 } 630 if (ku) 631 break; 632 } 633 } else { 634 kg = td->td_ksegrp; 635 if (kg->kg_upsleeps) { 636 wakeup_one(&kg->kg_completed); 637 mtx_unlock_spin(&sched_lock); 638 PROC_UNLOCK(p); 639 return (0); 640 } 641 ku = TAILQ_FIRST(&kg->kg_upcalls); 642 } 643 if (ku) { 644 if ((td2 = ku->ku_owner) == NULL) { 645 panic("%s: no owner", __func__); 646 } else if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR) && 647 ((td2->td_wchan == &kg->kg_completed) || 648 (td2->td_wchan == &p->p_siglist && 649 (ku->ku_mflags & KMF_WAITSIGEVENT)))) { 650 sleepq_abort(td2); 651 } else { 652 ku->ku_flags |= KUF_DOUPCALL; 653 } 654 mtx_unlock_spin(&sched_lock); 655 PROC_UNLOCK(p); 656 return (0); 657 } 658 mtx_unlock_spin(&sched_lock); 659 PROC_UNLOCK(p); 660 return (ESRCH); 661 } 662 663 /* 664 * No new KSEG: first call: use current KSE, don't schedule an upcall 665 * All other situations, do allocate max new KSEs and schedule an upcall. 666 */ 667 /* struct kse_create_args { 668 struct kse_mailbox *mbx; 669 int newgroup; 670 }; */ 671 int 672 kse_create(struct thread *td, struct kse_create_args *uap) 673 { 674 struct kse *newke; 675 struct ksegrp *newkg; 676 struct ksegrp *kg; 677 struct proc *p; 678 struct kse_mailbox mbx; 679 struct kse_upcall *newku; 680 int err, ncpus, sa = 0, first = 0; 681 struct thread *newtd; 682 683 p = td->td_proc; 684 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 685 return (err); 686 687 /* Too bad, why hasn't kernel always a cpu counter !? */ 688 #ifdef SMP 689 ncpus = mp_ncpus; 690 #else 691 ncpus = 1; 692 #endif 693 if (virtual_cpu != 0) 694 ncpus = virtual_cpu; 695 if (!(mbx.km_flags & KMF_BOUND)) 696 sa = TDF_SA; 697 else 698 ncpus = 1; 699 PROC_LOCK(p); 700 if (!(p->p_flag & P_SA)) { 701 first = 1; 702 p->p_flag |= P_SA; 703 } 704 PROC_UNLOCK(p); 705 if (!sa && !uap->newgroup && !first) 706 return (EINVAL); 707 kg = td->td_ksegrp; 708 if (uap->newgroup) { 709 /* Have race condition but it is cheap */ 710 if (p->p_numksegrps >= max_groups_per_proc) 711 return (EPROCLIM); 712 /* 713 * If we want a new KSEGRP it doesn't matter whether 714 * we have already fired up KSE mode before or not. 715 * We put the process in KSE mode and create a new KSEGRP. 716 */ 717 newkg = ksegrp_alloc(); 718 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 719 kg_startzero, kg_endzero)); 720 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 721 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 722 PROC_LOCK(p); 723 mtx_lock_spin(&sched_lock); 724 if (p->p_numksegrps >= max_groups_per_proc) { 725 mtx_unlock_spin(&sched_lock); 726 PROC_UNLOCK(p); 727 ksegrp_free(newkg); 728 return (EPROCLIM); 729 } 730 ksegrp_link(newkg, p); 731 sched_fork_ksegrp(kg, newkg); 732 mtx_unlock_spin(&sched_lock); 733 PROC_UNLOCK(p); 734 } else { 735 if (!first && ((td->td_flags & TDF_SA) ^ sa) != 0) 736 return (EINVAL); 737 newkg = kg; 738 } 739 740 /* 741 * Creating upcalls more than number of physical cpu does 742 * not help performance. 743 */ 744 if (newkg->kg_numupcalls >= ncpus) 745 return (EPROCLIM); 746 747 if (newkg->kg_numupcalls == 0) { 748 /* 749 * Initialize KSE group 750 * 751 * For multiplxed group, create KSEs as many as physical 752 * cpus. This increases concurrent even if userland 753 * is not MP safe and can only run on single CPU. 754 * In ideal world, every physical cpu should execute a thread. 755 * If there is enough KSEs, threads in kernel can be 756 * executed parallel on different cpus with full speed, 757 * Concurrent in kernel shouldn't be restricted by number of 758 * upcalls userland provides. Adding more upcall structures 759 * only increases concurrent in userland. 760 * 761 * For bound thread group, because there is only thread in the 762 * group, we only create one KSE for the group. Thread in this 763 * kind of group will never schedule an upcall when blocked, 764 * this intends to simulate pthread system scope thread. 765 */ 766 while (newkg->kg_kses < ncpus) { 767 newke = kse_alloc(); 768 bzero(&newke->ke_startzero, RANGEOF(struct kse, 769 ke_startzero, ke_endzero)); 770 #if 0 771 mtx_lock_spin(&sched_lock); 772 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 773 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 774 mtx_unlock_spin(&sched_lock); 775 #endif 776 mtx_lock_spin(&sched_lock); 777 kse_link(newke, newkg); 778 sched_fork_kse(td->td_kse, newke); 779 /* Add engine */ 780 kse_reassign(newke); 781 mtx_unlock_spin(&sched_lock); 782 } 783 } 784 newku = upcall_alloc(); 785 newku->ku_mailbox = uap->mbx; 786 newku->ku_func = mbx.km_func; 787 bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 788 789 /* For the first call this may not have been set */ 790 if (td->td_standin == NULL) 791 thread_alloc_spare(td, NULL); 792 793 PROC_LOCK(p); 794 if (newkg->kg_numupcalls >= ncpus) { 795 PROC_UNLOCK(p); 796 upcall_free(newku); 797 return (EPROCLIM); 798 } 799 if (first && sa) { 800 SIGSETOR(p->p_siglist, td->td_siglist); 801 SIGEMPTYSET(td->td_siglist); 802 SIGFILLSET(td->td_sigmask); 803 SIG_CANTMASK(td->td_sigmask); 804 } 805 mtx_lock_spin(&sched_lock); 806 PROC_UNLOCK(p); 807 upcall_link(newku, newkg); 808 if (mbx.km_quantum) 809 newkg->kg_upquantum = max(1, mbx.km_quantum/tick); 810 811 /* 812 * Each upcall structure has an owner thread, find which 813 * one owns it. 814 */ 815 if (uap->newgroup) { 816 /* 817 * Because new ksegrp hasn't thread, 818 * create an initial upcall thread to own it. 819 */ 820 newtd = thread_schedule_upcall(td, newku); 821 } else { 822 /* 823 * If current thread hasn't an upcall structure, 824 * just assign the upcall to it. 825 */ 826 if (td->td_upcall == NULL) { 827 newku->ku_owner = td; 828 td->td_upcall = newku; 829 newtd = td; 830 } else { 831 /* 832 * Create a new upcall thread to own it. 833 */ 834 newtd = thread_schedule_upcall(td, newku); 835 } 836 } 837 if (!sa) { 838 newtd->td_mailbox = mbx.km_curthread; 839 newtd->td_flags &= ~TDF_SA; 840 if (newtd != td) { 841 mtx_unlock_spin(&sched_lock); 842 cpu_set_upcall_kse(newtd, newku); 843 mtx_lock_spin(&sched_lock); 844 } 845 } else { 846 newtd->td_flags |= TDF_SA; 847 } 848 if (newtd != td) 849 setrunqueue(newtd); 850 mtx_unlock_spin(&sched_lock); 851 return (0); 852 } 853 854 /* 855 * Initialize global thread allocation resources. 856 */ 857 void 858 threadinit(void) 859 { 860 861 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 862 thread_ctor, thread_dtor, thread_init, thread_fini, 863 UMA_ALIGN_CACHE, 0); 864 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 865 NULL, NULL, ksegrp_init, NULL, 866 UMA_ALIGN_CACHE, 0); 867 kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 868 NULL, NULL, kse_init, NULL, 869 UMA_ALIGN_CACHE, 0); 870 upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 871 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 872 } 873 874 /* 875 * Stash an embarasingly extra thread into the zombie thread queue. 876 */ 877 void 878 thread_stash(struct thread *td) 879 { 880 mtx_lock_spin(&kse_zombie_lock); 881 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 882 mtx_unlock_spin(&kse_zombie_lock); 883 } 884 885 /* 886 * Stash an embarasingly extra kse into the zombie kse queue. 887 */ 888 void 889 kse_stash(struct kse *ke) 890 { 891 mtx_lock_spin(&kse_zombie_lock); 892 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 893 mtx_unlock_spin(&kse_zombie_lock); 894 } 895 896 /* 897 * Stash an embarasingly extra upcall into the zombie upcall queue. 898 */ 899 900 void 901 upcall_stash(struct kse_upcall *ku) 902 { 903 mtx_lock_spin(&kse_zombie_lock); 904 TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 905 mtx_unlock_spin(&kse_zombie_lock); 906 } 907 908 /* 909 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 910 */ 911 void 912 ksegrp_stash(struct ksegrp *kg) 913 { 914 mtx_lock_spin(&kse_zombie_lock); 915 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 916 mtx_unlock_spin(&kse_zombie_lock); 917 } 918 919 /* 920 * Reap zombie kse resource. 921 */ 922 void 923 thread_reap(void) 924 { 925 struct thread *td_first, *td_next; 926 struct kse *ke_first, *ke_next; 927 struct ksegrp *kg_first, * kg_next; 928 struct kse_upcall *ku_first, *ku_next; 929 930 /* 931 * Don't even bother to lock if none at this instant, 932 * we really don't care about the next instant.. 933 */ 934 if ((!TAILQ_EMPTY(&zombie_threads)) 935 || (!TAILQ_EMPTY(&zombie_kses)) 936 || (!TAILQ_EMPTY(&zombie_ksegrps)) 937 || (!TAILQ_EMPTY(&zombie_upcalls))) { 938 mtx_lock_spin(&kse_zombie_lock); 939 td_first = TAILQ_FIRST(&zombie_threads); 940 ke_first = TAILQ_FIRST(&zombie_kses); 941 kg_first = TAILQ_FIRST(&zombie_ksegrps); 942 ku_first = TAILQ_FIRST(&zombie_upcalls); 943 if (td_first) 944 TAILQ_INIT(&zombie_threads); 945 if (ke_first) 946 TAILQ_INIT(&zombie_kses); 947 if (kg_first) 948 TAILQ_INIT(&zombie_ksegrps); 949 if (ku_first) 950 TAILQ_INIT(&zombie_upcalls); 951 mtx_unlock_spin(&kse_zombie_lock); 952 while (td_first) { 953 td_next = TAILQ_NEXT(td_first, td_runq); 954 if (td_first->td_ucred) 955 crfree(td_first->td_ucred); 956 thread_free(td_first); 957 td_first = td_next; 958 } 959 while (ke_first) { 960 ke_next = TAILQ_NEXT(ke_first, ke_procq); 961 kse_free(ke_first); 962 ke_first = ke_next; 963 } 964 while (kg_first) { 965 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 966 ksegrp_free(kg_first); 967 kg_first = kg_next; 968 } 969 while (ku_first) { 970 ku_next = TAILQ_NEXT(ku_first, ku_link); 971 upcall_free(ku_first); 972 ku_first = ku_next; 973 } 974 } 975 } 976 977 /* 978 * Allocate a ksegrp. 979 */ 980 struct ksegrp * 981 ksegrp_alloc(void) 982 { 983 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 984 } 985 986 /* 987 * Allocate a kse. 988 */ 989 struct kse * 990 kse_alloc(void) 991 { 992 return (uma_zalloc(kse_zone, M_WAITOK)); 993 } 994 995 /* 996 * Allocate a thread. 997 */ 998 struct thread * 999 thread_alloc(void) 1000 { 1001 thread_reap(); /* check if any zombies to get */ 1002 return (uma_zalloc(thread_zone, M_WAITOK)); 1003 } 1004 1005 /* 1006 * Deallocate a ksegrp. 1007 */ 1008 void 1009 ksegrp_free(struct ksegrp *td) 1010 { 1011 uma_zfree(ksegrp_zone, td); 1012 } 1013 1014 /* 1015 * Deallocate a kse. 1016 */ 1017 void 1018 kse_free(struct kse *td) 1019 { 1020 uma_zfree(kse_zone, td); 1021 } 1022 1023 /* 1024 * Deallocate a thread. 1025 */ 1026 void 1027 thread_free(struct thread *td) 1028 { 1029 1030 cpu_thread_clean(td); 1031 uma_zfree(thread_zone, td); 1032 } 1033 1034 /* 1035 * Store the thread context in the UTS's mailbox. 1036 * then add the mailbox at the head of a list we are building in user space. 1037 * The list is anchored in the ksegrp structure. 1038 */ 1039 int 1040 thread_export_context(struct thread *td, int willexit) 1041 { 1042 struct proc *p; 1043 struct ksegrp *kg; 1044 uintptr_t mbx; 1045 void *addr; 1046 int error = 0, temp, sig; 1047 mcontext_t mc; 1048 1049 p = td->td_proc; 1050 kg = td->td_ksegrp; 1051 1052 /* Export the user/machine context. */ 1053 get_mcontext(td, &mc, 0); 1054 addr = (void *)(&td->td_mailbox->tm_context.uc_mcontext); 1055 error = copyout(&mc, addr, sizeof(mcontext_t)); 1056 if (error) 1057 goto bad; 1058 1059 /* Exports clock ticks in kernel mode */ 1060 addr = (caddr_t)(&td->td_mailbox->tm_sticks); 1061 temp = fuword32(addr) + td->td_usticks; 1062 if (suword32(addr, temp)) { 1063 error = EFAULT; 1064 goto bad; 1065 } 1066 1067 /* 1068 * Post sync signal, or process SIGKILL and SIGSTOP. 1069 * For sync signal, it is only possible when the signal is not 1070 * caught by userland or process is being debugged. 1071 */ 1072 PROC_LOCK(p); 1073 if (td->td_flags & TDF_NEEDSIGCHK) { 1074 mtx_lock_spin(&sched_lock); 1075 td->td_flags &= ~TDF_NEEDSIGCHK; 1076 mtx_unlock_spin(&sched_lock); 1077 mtx_lock(&p->p_sigacts->ps_mtx); 1078 while ((sig = cursig(td)) != 0) 1079 postsig(sig); 1080 mtx_unlock(&p->p_sigacts->ps_mtx); 1081 } 1082 if (willexit) 1083 SIGFILLSET(td->td_sigmask); 1084 PROC_UNLOCK(p); 1085 1086 /* Get address in latest mbox of list pointer */ 1087 addr = (void *)(&td->td_mailbox->tm_next); 1088 /* 1089 * Put the saved address of the previous first 1090 * entry into this one 1091 */ 1092 for (;;) { 1093 mbx = (uintptr_t)kg->kg_completed; 1094 if (suword(addr, mbx)) { 1095 error = EFAULT; 1096 goto bad; 1097 } 1098 PROC_LOCK(p); 1099 if (mbx == (uintptr_t)kg->kg_completed) { 1100 kg->kg_completed = td->td_mailbox; 1101 /* 1102 * The thread context may be taken away by 1103 * other upcall threads when we unlock 1104 * process lock. it's no longer valid to 1105 * use it again in any other places. 1106 */ 1107 td->td_mailbox = NULL; 1108 PROC_UNLOCK(p); 1109 break; 1110 } 1111 PROC_UNLOCK(p); 1112 } 1113 td->td_usticks = 0; 1114 return (0); 1115 1116 bad: 1117 PROC_LOCK(p); 1118 sigexit(td, SIGILL); 1119 return (error); 1120 } 1121 1122 /* 1123 * Take the list of completed mailboxes for this KSEGRP and put them on this 1124 * upcall's mailbox as it's the next one going up. 1125 */ 1126 static int 1127 thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1128 { 1129 struct proc *p = kg->kg_proc; 1130 void *addr; 1131 uintptr_t mbx; 1132 1133 addr = (void *)(&ku->ku_mailbox->km_completed); 1134 for (;;) { 1135 mbx = (uintptr_t)kg->kg_completed; 1136 if (suword(addr, mbx)) { 1137 PROC_LOCK(p); 1138 psignal(p, SIGSEGV); 1139 PROC_UNLOCK(p); 1140 return (EFAULT); 1141 } 1142 PROC_LOCK(p); 1143 if (mbx == (uintptr_t)kg->kg_completed) { 1144 kg->kg_completed = NULL; 1145 PROC_UNLOCK(p); 1146 break; 1147 } 1148 PROC_UNLOCK(p); 1149 } 1150 return (0); 1151 } 1152 1153 /* 1154 * This function should be called at statclock interrupt time 1155 */ 1156 int 1157 thread_statclock(int user) 1158 { 1159 struct thread *td = curthread; 1160 struct ksegrp *kg = td->td_ksegrp; 1161 1162 if (kg->kg_numupcalls == 0 || !(td->td_flags & TDF_SA)) 1163 return (0); 1164 if (user) { 1165 /* Current always do via ast() */ 1166 mtx_lock_spin(&sched_lock); 1167 td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1168 mtx_unlock_spin(&sched_lock); 1169 td->td_uuticks++; 1170 } else { 1171 if (td->td_mailbox != NULL) 1172 td->td_usticks++; 1173 else { 1174 /* XXXKSE 1175 * We will call thread_user_enter() for every 1176 * kernel entry in future, so if the thread mailbox 1177 * is NULL, it must be a UTS kernel, don't account 1178 * clock ticks for it. 1179 */ 1180 } 1181 } 1182 return (0); 1183 } 1184 1185 /* 1186 * Export state clock ticks for userland 1187 */ 1188 static int 1189 thread_update_usr_ticks(struct thread *td, int user) 1190 { 1191 struct proc *p = td->td_proc; 1192 struct kse_thr_mailbox *tmbx; 1193 struct kse_upcall *ku; 1194 struct ksegrp *kg; 1195 caddr_t addr; 1196 u_int uticks; 1197 1198 if ((ku = td->td_upcall) == NULL) 1199 return (-1); 1200 1201 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1202 if ((tmbx == NULL) || (tmbx == (void *)-1)) 1203 return (-1); 1204 if (user) { 1205 uticks = td->td_uuticks; 1206 td->td_uuticks = 0; 1207 addr = (caddr_t)&tmbx->tm_uticks; 1208 } else { 1209 uticks = td->td_usticks; 1210 td->td_usticks = 0; 1211 addr = (caddr_t)&tmbx->tm_sticks; 1212 } 1213 if (uticks) { 1214 if (suword32(addr, uticks+fuword32(addr))) { 1215 PROC_LOCK(p); 1216 psignal(p, SIGSEGV); 1217 PROC_UNLOCK(p); 1218 return (-2); 1219 } 1220 } 1221 kg = td->td_ksegrp; 1222 if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { 1223 mtx_lock_spin(&sched_lock); 1224 td->td_upcall->ku_flags |= KUF_DOUPCALL; 1225 mtx_unlock_spin(&sched_lock); 1226 } 1227 return (0); 1228 } 1229 1230 /* 1231 * Discard the current thread and exit from its context. 1232 * 1233 * Because we can't free a thread while we're operating under its context, 1234 * push the current thread into our CPU's deadthread holder. This means 1235 * we needn't worry about someone else grabbing our context before we 1236 * do a cpu_throw(). 1237 */ 1238 void 1239 thread_exit(void) 1240 { 1241 struct thread *td; 1242 struct kse *ke; 1243 struct proc *p; 1244 struct ksegrp *kg; 1245 1246 td = curthread; 1247 kg = td->td_ksegrp; 1248 p = td->td_proc; 1249 ke = td->td_kse; 1250 1251 mtx_assert(&sched_lock, MA_OWNED); 1252 KASSERT(p != NULL, ("thread exiting without a process")); 1253 KASSERT(ke != NULL, ("thread exiting without a kse")); 1254 KASSERT(kg != NULL, ("thread exiting without a kse group")); 1255 PROC_LOCK_ASSERT(p, MA_OWNED); 1256 CTR1(KTR_PROC, "thread_exit: thread %p", td); 1257 mtx_assert(&Giant, MA_NOTOWNED); 1258 1259 if (td->td_standin != NULL) { 1260 thread_stash(td->td_standin); 1261 td->td_standin = NULL; 1262 } 1263 1264 cpu_thread_exit(td); /* XXXSMP */ 1265 1266 /* 1267 * The last thread is left attached to the process 1268 * So that the whole bundle gets recycled. Skip 1269 * all this stuff. 1270 */ 1271 if (p->p_numthreads > 1) { 1272 thread_unlink(td); 1273 if (p->p_maxthrwaits) 1274 wakeup(&p->p_numthreads); 1275 /* 1276 * The test below is NOT true if we are the 1277 * sole exiting thread. P_STOPPED_SNGL is unset 1278 * in exit1() after it is the only survivor. 1279 */ 1280 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1281 if (p->p_numthreads == p->p_suspcount) { 1282 thread_unsuspend_one(p->p_singlethread); 1283 } 1284 } 1285 1286 /* 1287 * Because each upcall structure has an owner thread, 1288 * owner thread exits only when process is in exiting 1289 * state, so upcall to userland is no longer needed, 1290 * deleting upcall structure is safe here. 1291 * So when all threads in a group is exited, all upcalls 1292 * in the group should be automatically freed. 1293 */ 1294 if (td->td_upcall) 1295 upcall_remove(td); 1296 1297 sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 1298 sched_exit_kse(FIRST_KSE_IN_PROC(p), ke); 1299 ke->ke_state = KES_UNQUEUED; 1300 ke->ke_thread = NULL; 1301 /* 1302 * Decide what to do with the KSE attached to this thread. 1303 */ 1304 if (ke->ke_flags & KEF_EXIT) { 1305 kse_unlink(ke); 1306 if (kg->kg_kses == 0) { 1307 sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), kg); 1308 ksegrp_unlink(kg); 1309 } 1310 } 1311 else 1312 kse_reassign(ke); 1313 PROC_UNLOCK(p); 1314 td->td_kse = NULL; 1315 td->td_state = TDS_INACTIVE; 1316 #if 0 1317 td->td_proc = NULL; 1318 #endif 1319 td->td_ksegrp = NULL; 1320 td->td_last_kse = NULL; 1321 PCPU_SET(deadthread, td); 1322 } else { 1323 PROC_UNLOCK(p); 1324 } 1325 /* XXX Shouldn't cpu_throw() here. */ 1326 mtx_assert(&sched_lock, MA_OWNED); 1327 cpu_throw(td, choosethread()); 1328 panic("I'm a teapot!"); 1329 /* NOTREACHED */ 1330 } 1331 1332 /* 1333 * Do any thread specific cleanups that may be needed in wait() 1334 * called with Giant held, proc and schedlock not held. 1335 */ 1336 void 1337 thread_wait(struct proc *p) 1338 { 1339 struct thread *td; 1340 1341 KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 1342 KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); 1343 FOREACH_THREAD_IN_PROC(p, td) { 1344 if (td->td_standin != NULL) { 1345 thread_free(td->td_standin); 1346 td->td_standin = NULL; 1347 } 1348 cpu_thread_clean(td); 1349 } 1350 thread_reap(); /* check for zombie threads etc. */ 1351 } 1352 1353 /* 1354 * Link a thread to a process. 1355 * set up anything that needs to be initialized for it to 1356 * be used by the process. 1357 * 1358 * Note that we do not link to the proc's ucred here. 1359 * The thread is linked as if running but no KSE assigned. 1360 */ 1361 void 1362 thread_link(struct thread *td, struct ksegrp *kg) 1363 { 1364 struct proc *p; 1365 1366 p = kg->kg_proc; 1367 td->td_state = TDS_INACTIVE; 1368 td->td_proc = p; 1369 td->td_ksegrp = kg; 1370 td->td_last_kse = NULL; 1371 td->td_flags = 0; 1372 td->td_kse = NULL; 1373 1374 LIST_INIT(&td->td_contested); 1375 callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 1376 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 1377 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 1378 p->p_numthreads++; 1379 kg->kg_numthreads++; 1380 } 1381 1382 void 1383 thread_unlink(struct thread *td) 1384 { 1385 struct proc *p = td->td_proc; 1386 struct ksegrp *kg = td->td_ksegrp; 1387 1388 mtx_assert(&sched_lock, MA_OWNED); 1389 TAILQ_REMOVE(&p->p_threads, td, td_plist); 1390 p->p_numthreads--; 1391 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1392 kg->kg_numthreads--; 1393 /* could clear a few other things here */ 1394 } 1395 1396 /* 1397 * Purge a ksegrp resource. When a ksegrp is preparing to 1398 * exit, it calls this function. 1399 */ 1400 static void 1401 kse_purge_group(struct thread *td) 1402 { 1403 struct ksegrp *kg; 1404 struct kse *ke; 1405 1406 kg = td->td_ksegrp; 1407 KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1408 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1409 KASSERT(ke->ke_state == KES_IDLE, 1410 ("%s: wrong idle KSE state", __func__)); 1411 kse_unlink(ke); 1412 } 1413 KASSERT((kg->kg_kses == 1), 1414 ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1415 KASSERT((kg->kg_numupcalls == 0), 1416 ("%s: ksegrp still has %d upcall datas", 1417 __func__, kg->kg_numupcalls)); 1418 } 1419 1420 /* 1421 * Purge a process's KSE resource. When a process is preparing to 1422 * exit, it calls kse_purge to release any extra KSE resources in 1423 * the process. 1424 */ 1425 static void 1426 kse_purge(struct proc *p, struct thread *td) 1427 { 1428 struct ksegrp *kg; 1429 struct kse *ke; 1430 1431 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1432 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1433 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1434 p->p_numksegrps--; 1435 /* 1436 * There is no ownership for KSE, after all threads 1437 * in the group exited, it is possible that some KSEs 1438 * were left in idle queue, gc them now. 1439 */ 1440 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1441 KASSERT(ke->ke_state == KES_IDLE, 1442 ("%s: wrong idle KSE state", __func__)); 1443 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1444 kg->kg_idle_kses--; 1445 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1446 kg->kg_kses--; 1447 kse_stash(ke); 1448 } 1449 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1450 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1451 ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1452 KASSERT((kg->kg_numupcalls == 0), 1453 ("%s: ksegrp still has %d upcall datas", 1454 __func__, kg->kg_numupcalls)); 1455 1456 if (kg != td->td_ksegrp) 1457 ksegrp_stash(kg); 1458 } 1459 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1460 p->p_numksegrps++; 1461 } 1462 1463 /* 1464 * This function is intended to be used to initialize a spare thread 1465 * for upcall. Initialize thread's large data area outside sched_lock 1466 * for thread_schedule_upcall(). 1467 */ 1468 void 1469 thread_alloc_spare(struct thread *td, struct thread *spare) 1470 { 1471 if (td->td_standin) 1472 return; 1473 if (spare == NULL) 1474 spare = thread_alloc(); 1475 td->td_standin = spare; 1476 bzero(&spare->td_startzero, 1477 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1478 spare->td_proc = td->td_proc; 1479 spare->td_ucred = crhold(td->td_ucred); 1480 } 1481 1482 /* 1483 * Create a thread and schedule it for upcall on the KSE given. 1484 * Use our thread's standin so that we don't have to allocate one. 1485 */ 1486 struct thread * 1487 thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 1488 { 1489 struct thread *td2; 1490 1491 mtx_assert(&sched_lock, MA_OWNED); 1492 1493 /* 1494 * Schedule an upcall thread on specified kse_upcall, 1495 * the kse_upcall must be free. 1496 * td must have a spare thread. 1497 */ 1498 KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1499 if ((td2 = td->td_standin) != NULL) { 1500 td->td_standin = NULL; 1501 } else { 1502 panic("no reserve thread when scheduling an upcall"); 1503 return (NULL); 1504 } 1505 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1506 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1507 bcopy(&td->td_startcopy, &td2->td_startcopy, 1508 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1509 thread_link(td2, ku->ku_ksegrp); 1510 /* inherit blocked thread's context */ 1511 cpu_set_upcall(td2, td); 1512 /* Let the new thread become owner of the upcall */ 1513 ku->ku_owner = td2; 1514 td2->td_upcall = ku; 1515 td2->td_flags = TDF_SA; 1516 td2->td_pflags = TDP_UPCALLING; 1517 td2->td_kse = NULL; 1518 td2->td_state = TDS_CAN_RUN; 1519 td2->td_inhibitors = 0; 1520 SIGFILLSET(td2->td_sigmask); 1521 SIG_CANTMASK(td2->td_sigmask); 1522 sched_fork_thread(td, td2); 1523 return (td2); /* bogus.. should be a void function */ 1524 } 1525 1526 /* 1527 * It is only used when thread generated a trap and process is being 1528 * debugged. 1529 */ 1530 void 1531 thread_signal_add(struct thread *td, int sig) 1532 { 1533 struct proc *p; 1534 siginfo_t siginfo; 1535 struct sigacts *ps; 1536 int error; 1537 1538 p = td->td_proc; 1539 PROC_LOCK_ASSERT(p, MA_OWNED); 1540 ps = p->p_sigacts; 1541 mtx_assert(&ps->ps_mtx, MA_OWNED); 1542 1543 cpu_thread_siginfo(sig, 0, &siginfo); 1544 mtx_unlock(&ps->ps_mtx); 1545 PROC_UNLOCK(p); 1546 error = copyout(&siginfo, &td->td_mailbox->tm_syncsig, sizeof(siginfo)); 1547 if (error) { 1548 PROC_LOCK(p); 1549 sigexit(td, SIGILL); 1550 } 1551 PROC_LOCK(p); 1552 SIGADDSET(td->td_sigmask, sig); 1553 mtx_lock(&ps->ps_mtx); 1554 } 1555 1556 void 1557 thread_switchout(struct thread *td) 1558 { 1559 struct kse_upcall *ku; 1560 struct thread *td2; 1561 1562 mtx_assert(&sched_lock, MA_OWNED); 1563 1564 /* 1565 * If the outgoing thread is in threaded group and has never 1566 * scheduled an upcall, decide whether this is a short 1567 * or long term event and thus whether or not to schedule 1568 * an upcall. 1569 * If it is a short term event, just suspend it in 1570 * a way that takes its KSE with it. 1571 * Select the events for which we want to schedule upcalls. 1572 * For now it's just sleep. 1573 * XXXKSE eventually almost any inhibition could do. 1574 */ 1575 if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { 1576 /* 1577 * Release ownership of upcall, and schedule an upcall 1578 * thread, this new upcall thread becomes the owner of 1579 * the upcall structure. 1580 */ 1581 ku = td->td_upcall; 1582 ku->ku_owner = NULL; 1583 td->td_upcall = NULL; 1584 td->td_flags &= ~TDF_CAN_UNBIND; 1585 td2 = thread_schedule_upcall(td, ku); 1586 setrunqueue(td2); 1587 } 1588 } 1589 1590 /* 1591 * Setup done on the thread when it enters the kernel. 1592 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1593 */ 1594 void 1595 thread_user_enter(struct proc *p, struct thread *td) 1596 { 1597 struct ksegrp *kg; 1598 struct kse_upcall *ku; 1599 struct kse_thr_mailbox *tmbx; 1600 uint32_t tflags; 1601 1602 kg = td->td_ksegrp; 1603 1604 /* 1605 * First check that we shouldn't just abort. 1606 * But check if we are the single thread first! 1607 */ 1608 if (p->p_flag & P_SINGLE_EXIT) { 1609 PROC_LOCK(p); 1610 mtx_lock_spin(&sched_lock); 1611 thread_stopped(p); 1612 thread_exit(); 1613 /* NOTREACHED */ 1614 } 1615 1616 /* 1617 * If we are doing a syscall in a KSE environment, 1618 * note where our mailbox is. There is always the 1619 * possibility that we could do this lazily (in kse_reassign()), 1620 * but for now do it every time. 1621 */ 1622 kg = td->td_ksegrp; 1623 if (td->td_flags & TDF_SA) { 1624 ku = td->td_upcall; 1625 KASSERT(ku, ("%s: no upcall owned", __func__)); 1626 KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1627 KASSERT(!TD_CAN_UNBIND(td), ("%s: can unbind", __func__)); 1628 ku->ku_mflags = fuword32((void *)&ku->ku_mailbox->km_flags); 1629 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1630 if ((tmbx == NULL) || (tmbx == (void *)-1L) || 1631 (ku->ku_mflags & KMF_NOUPCALL)) { 1632 td->td_mailbox = NULL; 1633 } else { 1634 if (td->td_standin == NULL) 1635 thread_alloc_spare(td, NULL); 1636 tflags = fuword32(&tmbx->tm_flags); 1637 /* 1638 * On some architectures, TP register points to thread 1639 * mailbox but not points to kse mailbox, and userland 1640 * can not atomically clear km_curthread, but can 1641 * use TP register, and set TMF_NOUPCALL in thread 1642 * flag to indicate a critical region. 1643 */ 1644 if (tflags & TMF_NOUPCALL) { 1645 td->td_mailbox = NULL; 1646 } else { 1647 td->td_mailbox = tmbx; 1648 mtx_lock_spin(&sched_lock); 1649 td->td_flags |= TDF_CAN_UNBIND; 1650 mtx_unlock_spin(&sched_lock); 1651 } 1652 } 1653 } 1654 } 1655 1656 /* 1657 * The extra work we go through if we are a threaded process when we 1658 * return to userland. 1659 * 1660 * If we are a KSE process and returning to user mode, check for 1661 * extra work to do before we return (e.g. for more syscalls 1662 * to complete first). If we were in a critical section, we should 1663 * just return to let it finish. Same if we were in the UTS (in 1664 * which case the mailbox's context's busy indicator will be set). 1665 * The only traps we suport will have set the mailbox. 1666 * We will clear it here. 1667 */ 1668 int 1669 thread_userret(struct thread *td, struct trapframe *frame) 1670 { 1671 int error = 0, upcalls, uts_crit; 1672 struct kse_upcall *ku; 1673 struct ksegrp *kg, *kg2; 1674 struct proc *p; 1675 struct timespec ts; 1676 1677 p = td->td_proc; 1678 kg = td->td_ksegrp; 1679 ku = td->td_upcall; 1680 1681 /* Nothing to do with bound thread */ 1682 if (!(td->td_flags & TDF_SA)) 1683 return (0); 1684 1685 /* 1686 * Stat clock interrupt hit in userland, it 1687 * is returning from interrupt, charge thread's 1688 * userland time for UTS. 1689 */ 1690 if (td->td_flags & TDF_USTATCLOCK) { 1691 thread_update_usr_ticks(td, 1); 1692 mtx_lock_spin(&sched_lock); 1693 td->td_flags &= ~TDF_USTATCLOCK; 1694 mtx_unlock_spin(&sched_lock); 1695 if (kg->kg_completed || 1696 (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1697 thread_user_enter(p, td); 1698 } 1699 1700 uts_crit = (td->td_mailbox == NULL); 1701 /* 1702 * Optimisation: 1703 * This thread has not started any upcall. 1704 * If there is no work to report other than ourself, 1705 * then it can return direct to userland. 1706 */ 1707 if (TD_CAN_UNBIND(td)) { 1708 mtx_lock_spin(&sched_lock); 1709 td->td_flags &= ~TDF_CAN_UNBIND; 1710 if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1711 (kg->kg_completed == NULL) && 1712 (ku->ku_flags & KUF_DOUPCALL) == 0 && 1713 (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { 1714 mtx_unlock_spin(&sched_lock); 1715 thread_update_usr_ticks(td, 0); 1716 nanotime(&ts); 1717 error = copyout(&ts, 1718 (caddr_t)&ku->ku_mailbox->km_timeofday, 1719 sizeof(ts)); 1720 td->td_mailbox = 0; 1721 ku->ku_mflags = 0; 1722 if (error) 1723 goto out; 1724 return (0); 1725 } 1726 mtx_unlock_spin(&sched_lock); 1727 thread_export_context(td, 0); 1728 /* 1729 * There is something to report, and we own an upcall 1730 * strucuture, we can go to userland. 1731 * Turn ourself into an upcall thread. 1732 */ 1733 td->td_pflags |= TDP_UPCALLING; 1734 } else if (td->td_mailbox && (ku == NULL)) { 1735 thread_export_context(td, 1); 1736 PROC_LOCK(p); 1737 /* 1738 * There are upcall threads waiting for 1739 * work to do, wake one of them up. 1740 * XXXKSE Maybe wake all of them up. 1741 */ 1742 if (kg->kg_upsleeps) 1743 wakeup_one(&kg->kg_completed); 1744 mtx_lock_spin(&sched_lock); 1745 thread_stopped(p); 1746 thread_exit(); 1747 /* NOTREACHED */ 1748 } 1749 1750 KASSERT(ku != NULL, ("upcall is NULL\n")); 1751 KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1752 1753 if (p->p_numthreads > max_threads_per_proc) { 1754 max_threads_hits++; 1755 PROC_LOCK(p); 1756 mtx_lock_spin(&sched_lock); 1757 p->p_maxthrwaits++; 1758 while (p->p_numthreads > max_threads_per_proc) { 1759 upcalls = 0; 1760 FOREACH_KSEGRP_IN_PROC(p, kg2) { 1761 if (kg2->kg_numupcalls == 0) 1762 upcalls++; 1763 else 1764 upcalls += kg2->kg_numupcalls; 1765 } 1766 if (upcalls >= max_threads_per_proc) 1767 break; 1768 mtx_unlock_spin(&sched_lock); 1769 if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1770 "maxthreads", 0)) { 1771 mtx_lock_spin(&sched_lock); 1772 break; 1773 } else { 1774 mtx_lock_spin(&sched_lock); 1775 } 1776 } 1777 p->p_maxthrwaits--; 1778 mtx_unlock_spin(&sched_lock); 1779 PROC_UNLOCK(p); 1780 } 1781 1782 if (td->td_pflags & TDP_UPCALLING) { 1783 uts_crit = 0; 1784 kg->kg_nextupcall = ticks+kg->kg_upquantum; 1785 /* 1786 * There is no more work to do and we are going to ride 1787 * this thread up to userland as an upcall. 1788 * Do the last parts of the setup needed for the upcall. 1789 */ 1790 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1791 td, td->td_proc->p_pid, td->td_proc->p_comm); 1792 1793 td->td_pflags &= ~TDP_UPCALLING; 1794 if (ku->ku_flags & KUF_DOUPCALL) { 1795 mtx_lock_spin(&sched_lock); 1796 ku->ku_flags &= ~KUF_DOUPCALL; 1797 mtx_unlock_spin(&sched_lock); 1798 } 1799 /* 1800 * Set user context to the UTS 1801 */ 1802 if (!(ku->ku_mflags & KMF_NOUPCALL)) { 1803 cpu_set_upcall_kse(td, ku); 1804 error = suword(&ku->ku_mailbox->km_curthread, 0); 1805 if (error) 1806 goto out; 1807 } 1808 1809 /* 1810 * Unhook the list of completed threads. 1811 * anything that completes after this gets to 1812 * come in next time. 1813 * Put the list of completed thread mailboxes on 1814 * this KSE's mailbox. 1815 */ 1816 if (!(ku->ku_mflags & KMF_NOCOMPLETED) && 1817 (error = thread_link_mboxes(kg, ku)) != 0) 1818 goto out; 1819 } 1820 if (!uts_crit) { 1821 nanotime(&ts); 1822 error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); 1823 } 1824 1825 out: 1826 if (error) { 1827 /* 1828 * Things are going to be so screwed we should just kill 1829 * the process. 1830 * how do we do that? 1831 */ 1832 PROC_LOCK(td->td_proc); 1833 psignal(td->td_proc, SIGSEGV); 1834 PROC_UNLOCK(td->td_proc); 1835 } else { 1836 /* 1837 * Optimisation: 1838 * Ensure that we have a spare thread available, 1839 * for when we re-enter the kernel. 1840 */ 1841 if (td->td_standin == NULL) 1842 thread_alloc_spare(td, NULL); 1843 } 1844 1845 ku->ku_mflags = 0; 1846 /* 1847 * Clear thread mailbox first, then clear system tick count. 1848 * The order is important because thread_statclock() use 1849 * mailbox pointer to see if it is an userland thread or 1850 * an UTS kernel thread. 1851 */ 1852 td->td_mailbox = NULL; 1853 td->td_usticks = 0; 1854 return (error); /* go sync */ 1855 } 1856 1857 /* 1858 * Enforce single-threading. 1859 * 1860 * Returns 1 if the caller must abort (another thread is waiting to 1861 * exit the process or similar). Process is locked! 1862 * Returns 0 when you are successfully the only thread running. 1863 * A process has successfully single threaded in the suspend mode when 1864 * There are no threads in user mode. Threads in the kernel must be 1865 * allowed to continue until they get to the user boundary. They may even 1866 * copy out their return values and data before suspending. They may however be 1867 * accellerated in reaching the user boundary as we will wake up 1868 * any sleeping threads that are interruptable. (PCATCH). 1869 */ 1870 int 1871 thread_single(int force_exit) 1872 { 1873 struct thread *td; 1874 struct thread *td2; 1875 struct proc *p; 1876 1877 td = curthread; 1878 p = td->td_proc; 1879 mtx_assert(&Giant, MA_OWNED); 1880 PROC_LOCK_ASSERT(p, MA_OWNED); 1881 KASSERT((td != NULL), ("curthread is NULL")); 1882 1883 if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1) 1884 return (0); 1885 1886 /* Is someone already single threading? */ 1887 if (p->p_singlethread) 1888 return (1); 1889 1890 if (force_exit == SINGLE_EXIT) { 1891 p->p_flag |= P_SINGLE_EXIT; 1892 } else 1893 p->p_flag &= ~P_SINGLE_EXIT; 1894 p->p_flag |= P_STOPPED_SINGLE; 1895 mtx_lock_spin(&sched_lock); 1896 p->p_singlethread = td; 1897 while ((p->p_numthreads - p->p_suspcount) != 1) { 1898 FOREACH_THREAD_IN_PROC(p, td2) { 1899 if (td2 == td) 1900 continue; 1901 td2->td_flags |= TDF_ASTPENDING; 1902 if (TD_IS_INHIBITED(td2)) { 1903 if (force_exit == SINGLE_EXIT) { 1904 if (TD_IS_SUSPENDED(td2)) { 1905 thread_unsuspend_one(td2); 1906 } 1907 if (TD_ON_SLEEPQ(td2) && 1908 (td2->td_flags & TDF_SINTR)) { 1909 sleepq_abort(td2); 1910 } 1911 } else { 1912 if (TD_IS_SUSPENDED(td2)) 1913 continue; 1914 /* 1915 * maybe other inhibitted states too? 1916 * XXXKSE Is it totally safe to 1917 * suspend a non-interruptable thread? 1918 */ 1919 if (td2->td_inhibitors & 1920 (TDI_SLEEPING | TDI_SWAPPED)) 1921 thread_suspend_one(td2); 1922 } 1923 } 1924 } 1925 /* 1926 * Maybe we suspended some threads.. was it enough? 1927 */ 1928 if ((p->p_numthreads - p->p_suspcount) == 1) 1929 break; 1930 1931 /* 1932 * Wake us up when everyone else has suspended. 1933 * In the mean time we suspend as well. 1934 */ 1935 thread_suspend_one(td); 1936 DROP_GIANT(); 1937 PROC_UNLOCK(p); 1938 mi_switch(SW_VOL); 1939 mtx_unlock_spin(&sched_lock); 1940 PICKUP_GIANT(); 1941 PROC_LOCK(p); 1942 mtx_lock_spin(&sched_lock); 1943 } 1944 if (force_exit == SINGLE_EXIT) { 1945 if (td->td_upcall) 1946 upcall_remove(td); 1947 kse_purge(p, td); 1948 } 1949 mtx_unlock_spin(&sched_lock); 1950 return (0); 1951 } 1952 1953 /* 1954 * Called in from locations that can safely check to see 1955 * whether we have to suspend or at least throttle for a 1956 * single-thread event (e.g. fork). 1957 * 1958 * Such locations include userret(). 1959 * If the "return_instead" argument is non zero, the thread must be able to 1960 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 1961 * 1962 * The 'return_instead' argument tells the function if it may do a 1963 * thread_exit() or suspend, or whether the caller must abort and back 1964 * out instead. 1965 * 1966 * If the thread that set the single_threading request has set the 1967 * P_SINGLE_EXIT bit in the process flags then this call will never return 1968 * if 'return_instead' is false, but will exit. 1969 * 1970 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 1971 *---------------+--------------------+--------------------- 1972 * 0 | returns 0 | returns 0 or 1 1973 * | when ST ends | immediatly 1974 *---------------+--------------------+--------------------- 1975 * 1 | thread exits | returns 1 1976 * | | immediatly 1977 * 0 = thread_exit() or suspension ok, 1978 * other = return error instead of stopping the thread. 1979 * 1980 * While a full suspension is under effect, even a single threading 1981 * thread would be suspended if it made this call (but it shouldn't). 1982 * This call should only be made from places where 1983 * thread_exit() would be safe as that may be the outcome unless 1984 * return_instead is set. 1985 */ 1986 int 1987 thread_suspend_check(int return_instead) 1988 { 1989 struct thread *td; 1990 struct proc *p; 1991 1992 td = curthread; 1993 p = td->td_proc; 1994 PROC_LOCK_ASSERT(p, MA_OWNED); 1995 while (P_SHOULDSTOP(p)) { 1996 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1997 KASSERT(p->p_singlethread != NULL, 1998 ("singlethread not set")); 1999 /* 2000 * The only suspension in action is a 2001 * single-threading. Single threader need not stop. 2002 * XXX Should be safe to access unlocked 2003 * as it can only be set to be true by us. 2004 */ 2005 if (p->p_singlethread == td) 2006 return (0); /* Exempt from stopping. */ 2007 } 2008 if (return_instead) 2009 return (1); 2010 2011 mtx_lock_spin(&sched_lock); 2012 thread_stopped(p); 2013 /* 2014 * If the process is waiting for us to exit, 2015 * this thread should just suicide. 2016 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 2017 */ 2018 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 2019 while (mtx_owned(&Giant)) 2020 mtx_unlock(&Giant); 2021 if (p->p_flag & P_SA) 2022 thread_exit(); 2023 else 2024 thr_exit1(); 2025 } 2026 2027 /* 2028 * When a thread suspends, it just 2029 * moves to the processes's suspend queue 2030 * and stays there. 2031 */ 2032 thread_suspend_one(td); 2033 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 2034 if (p->p_numthreads == p->p_suspcount) { 2035 thread_unsuspend_one(p->p_singlethread); 2036 } 2037 } 2038 DROP_GIANT(); 2039 PROC_UNLOCK(p); 2040 mi_switch(SW_INVOL); 2041 mtx_unlock_spin(&sched_lock); 2042 PICKUP_GIANT(); 2043 PROC_LOCK(p); 2044 } 2045 return (0); 2046 } 2047 2048 void 2049 thread_suspend_one(struct thread *td) 2050 { 2051 struct proc *p = td->td_proc; 2052 2053 mtx_assert(&sched_lock, MA_OWNED); 2054 PROC_LOCK_ASSERT(p, MA_OWNED); 2055 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 2056 p->p_suspcount++; 2057 TD_SET_SUSPENDED(td); 2058 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 2059 /* 2060 * Hack: If we are suspending but are on the sleep queue 2061 * then we are in msleep or the cv equivalent. We 2062 * want to look like we have two Inhibitors. 2063 * May already be set.. doesn't matter. 2064 */ 2065 if (TD_ON_SLEEPQ(td)) 2066 TD_SET_SLEEPING(td); 2067 } 2068 2069 void 2070 thread_unsuspend_one(struct thread *td) 2071 { 2072 struct proc *p = td->td_proc; 2073 2074 mtx_assert(&sched_lock, MA_OWNED); 2075 PROC_LOCK_ASSERT(p, MA_OWNED); 2076 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 2077 TD_CLR_SUSPENDED(td); 2078 p->p_suspcount--; 2079 setrunnable(td); 2080 } 2081 2082 /* 2083 * Allow all threads blocked by single threading to continue running. 2084 */ 2085 void 2086 thread_unsuspend(struct proc *p) 2087 { 2088 struct thread *td; 2089 2090 mtx_assert(&sched_lock, MA_OWNED); 2091 PROC_LOCK_ASSERT(p, MA_OWNED); 2092 if (!P_SHOULDSTOP(p)) { 2093 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2094 thread_unsuspend_one(td); 2095 } 2096 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 2097 (p->p_numthreads == p->p_suspcount)) { 2098 /* 2099 * Stopping everything also did the job for the single 2100 * threading request. Now we've downgraded to single-threaded, 2101 * let it continue. 2102 */ 2103 thread_unsuspend_one(p->p_singlethread); 2104 } 2105 } 2106 2107 void 2108 thread_single_end(void) 2109 { 2110 struct thread *td; 2111 struct proc *p; 2112 2113 td = curthread; 2114 p = td->td_proc; 2115 PROC_LOCK_ASSERT(p, MA_OWNED); 2116 p->p_flag &= ~P_STOPPED_SINGLE; 2117 mtx_lock_spin(&sched_lock); 2118 p->p_singlethread = NULL; 2119 /* 2120 * If there are other threads they mey now run, 2121 * unless of course there is a blanket 'stop order' 2122 * on the process. The single threader must be allowed 2123 * to continue however as this is a bad place to stop. 2124 */ 2125 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2126 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2127 thread_unsuspend_one(td); 2128 } 2129 } 2130 mtx_unlock_spin(&sched_lock); 2131 } 2132