1 /* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/mutex.h> 38 #include <sys/proc.h> 39 #include <sys/smp.h> 40 #include <sys/sysctl.h> 41 #include <sys/sysproto.h> 42 #include <sys/filedesc.h> 43 #include <sys/sched.h> 44 #include <sys/signalvar.h> 45 #include <sys/sleepqueue.h> 46 #include <sys/sx.h> 47 #include <sys/tty.h> 48 #include <sys/turnstile.h> 49 #include <sys/user.h> 50 #include <sys/kse.h> 51 #include <sys/ktr.h> 52 #include <sys/ucontext.h> 53 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_object.h> 57 #include <vm/pmap.h> 58 #include <vm/uma.h> 59 #include <vm/vm_map.h> 60 61 #include <machine/frame.h> 62 63 /* 64 * KSEGRP related storage. 65 */ 66 static uma_zone_t ksegrp_zone; 67 static uma_zone_t kse_zone; 68 static uma_zone_t thread_zone; 69 static uma_zone_t upcall_zone; 70 71 /* DEBUG ONLY */ 72 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 73 static int thread_debug = 0; 74 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 75 &thread_debug, 0, "thread debug"); 76 77 static int max_threads_per_proc = 1500; 78 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 79 &max_threads_per_proc, 0, "Limit on threads per proc"); 80 81 static int max_groups_per_proc = 500; 82 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 83 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 84 85 static int max_threads_hits; 86 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 87 &max_threads_hits, 0, ""); 88 89 static int virtual_cpu; 90 91 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 92 93 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 94 TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 95 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 96 TAILQ_HEAD(, kse_upcall) zombie_upcalls = 97 TAILQ_HEAD_INITIALIZER(zombie_upcalls); 98 struct mtx kse_zombie_lock; 99 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 100 101 static void kse_purge(struct proc *p, struct thread *td); 102 static void kse_purge_group(struct thread *td); 103 static int thread_update_usr_ticks(struct thread *td, int user); 104 static void thread_alloc_spare(struct thread *td, struct thread *spare); 105 106 static int 107 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 108 { 109 int error, new_val; 110 int def_val; 111 112 #ifdef SMP 113 def_val = mp_ncpus; 114 #else 115 def_val = 1; 116 #endif 117 if (virtual_cpu == 0) 118 new_val = def_val; 119 else 120 new_val = virtual_cpu; 121 error = sysctl_handle_int(oidp, &new_val, 0, req); 122 if (error != 0 || req->newptr == NULL) 123 return (error); 124 if (new_val < 0) 125 return (EINVAL); 126 virtual_cpu = new_val; 127 return (0); 128 } 129 130 /* DEBUG ONLY */ 131 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 132 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 133 "debug virtual cpus"); 134 135 /* 136 * Thread ID allocator. The allocator keeps track of assigned IDs by 137 * using a bitmap. The bitmap is created in parts. The parts are linked 138 * together. 139 */ 140 typedef u_long tid_bitmap_word; 141 142 #define TID_IDS_PER_PART 1024 143 #define TID_IDS_PER_IDX (sizeof(tid_bitmap_word) << 3) 144 #define TID_BITMAP_SIZE (TID_IDS_PER_PART / TID_IDS_PER_IDX) 145 #define TID_MIN (PID_MAX + 1) 146 147 struct tid_bitmap_part { 148 STAILQ_ENTRY(tid_bitmap_part) bmp_next; 149 tid_bitmap_word bmp_bitmap[TID_BITMAP_SIZE]; 150 int bmp_base; 151 int bmp_free; 152 }; 153 154 static STAILQ_HEAD(, tid_bitmap_part) tid_bitmap = 155 STAILQ_HEAD_INITIALIZER(tid_bitmap); 156 static uma_zone_t tid_zone; 157 158 struct mtx tid_lock; 159 MTX_SYSINIT(tid_lock, &tid_lock, "TID lock", MTX_DEF); 160 161 /* 162 * Prepare a thread for use. 163 */ 164 static void 165 thread_ctor(void *mem, int size, void *arg) 166 { 167 struct thread *td; 168 169 td = (struct thread *)mem; 170 td->td_tid = 0; 171 td->td_state = TDS_INACTIVE; 172 td->td_oncpu = NOCPU; 173 td->td_critnest = 1; 174 } 175 176 /* 177 * Reclaim a thread after use. 178 */ 179 static void 180 thread_dtor(void *mem, int size, void *arg) 181 { 182 struct thread *td; 183 struct tid_bitmap_part *bmp; 184 int bit, idx, tid; 185 186 td = (struct thread *)mem; 187 188 if (td->td_tid > PID_MAX) { 189 STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { 190 if (td->td_tid >= bmp->bmp_base && 191 td->td_tid < bmp->bmp_base + TID_IDS_PER_PART) 192 break; 193 } 194 KASSERT(bmp != NULL, ("No TID bitmap?")); 195 mtx_lock(&tid_lock); 196 tid = td->td_tid - bmp->bmp_base; 197 idx = tid / TID_IDS_PER_IDX; 198 bit = 1UL << (tid % TID_IDS_PER_IDX); 199 bmp->bmp_bitmap[idx] |= bit; 200 bmp->bmp_free++; 201 mtx_unlock(&tid_lock); 202 } 203 204 #ifdef INVARIANTS 205 /* Verify that this thread is in a safe state to free. */ 206 switch (td->td_state) { 207 case TDS_INHIBITED: 208 case TDS_RUNNING: 209 case TDS_CAN_RUN: 210 case TDS_RUNQ: 211 /* 212 * We must never unlink a thread that is in one of 213 * these states, because it is currently active. 214 */ 215 panic("bad state for thread unlinking"); 216 /* NOTREACHED */ 217 case TDS_INACTIVE: 218 break; 219 default: 220 panic("bad thread state"); 221 /* NOTREACHED */ 222 } 223 #endif 224 } 225 226 /* 227 * Initialize type-stable parts of a thread (when newly created). 228 */ 229 static void 230 thread_init(void *mem, int size) 231 { 232 struct thread *td; 233 234 td = (struct thread *)mem; 235 vm_thread_new(td, 0); 236 cpu_thread_setup(td); 237 td->td_sleepqueue = sleepq_alloc(); 238 td->td_turnstile = turnstile_alloc(); 239 td->td_sched = (struct td_sched *)&td[1]; 240 } 241 242 /* 243 * Tear down type-stable parts of a thread (just before being discarded). 244 */ 245 static void 246 thread_fini(void *mem, int size) 247 { 248 struct thread *td; 249 250 td = (struct thread *)mem; 251 turnstile_free(td->td_turnstile); 252 sleepq_free(td->td_sleepqueue); 253 vm_thread_dispose(td); 254 } 255 256 /* 257 * Initialize type-stable parts of a kse (when newly created). 258 */ 259 static void 260 kse_init(void *mem, int size) 261 { 262 struct kse *ke; 263 264 ke = (struct kse *)mem; 265 ke->ke_sched = (struct ke_sched *)&ke[1]; 266 } 267 268 /* 269 * Initialize type-stable parts of a ksegrp (when newly created). 270 */ 271 static void 272 ksegrp_init(void *mem, int size) 273 { 274 struct ksegrp *kg; 275 276 kg = (struct ksegrp *)mem; 277 kg->kg_sched = (struct kg_sched *)&kg[1]; 278 } 279 280 /* 281 * KSE is linked into kse group. 282 */ 283 void 284 kse_link(struct kse *ke, struct ksegrp *kg) 285 { 286 struct proc *p = kg->kg_proc; 287 288 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 289 kg->kg_kses++; 290 ke->ke_state = KES_UNQUEUED; 291 ke->ke_proc = p; 292 ke->ke_ksegrp = kg; 293 ke->ke_thread = NULL; 294 ke->ke_oncpu = NOCPU; 295 ke->ke_flags = 0; 296 } 297 298 void 299 kse_unlink(struct kse *ke) 300 { 301 struct ksegrp *kg; 302 303 mtx_assert(&sched_lock, MA_OWNED); 304 kg = ke->ke_ksegrp; 305 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 306 if (ke->ke_state == KES_IDLE) { 307 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 308 kg->kg_idle_kses--; 309 } 310 --kg->kg_kses; 311 /* 312 * Aggregate stats from the KSE 313 */ 314 kse_stash(ke); 315 } 316 317 void 318 ksegrp_link(struct ksegrp *kg, struct proc *p) 319 { 320 321 TAILQ_INIT(&kg->kg_threads); 322 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 323 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 324 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 325 TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 326 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 327 kg->kg_proc = p; 328 /* 329 * the following counters are in the -zero- section 330 * and may not need clearing 331 */ 332 kg->kg_numthreads = 0; 333 kg->kg_runnable = 0; 334 kg->kg_kses = 0; 335 kg->kg_runq_kses = 0; /* XXXKSE change name */ 336 kg->kg_idle_kses = 0; 337 kg->kg_numupcalls = 0; 338 /* link it in now that it's consistent */ 339 p->p_numksegrps++; 340 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 341 } 342 343 void 344 ksegrp_unlink(struct ksegrp *kg) 345 { 346 struct proc *p; 347 348 mtx_assert(&sched_lock, MA_OWNED); 349 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 350 KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 351 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 352 353 p = kg->kg_proc; 354 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 355 p->p_numksegrps--; 356 /* 357 * Aggregate stats from the KSE 358 */ 359 ksegrp_stash(kg); 360 } 361 362 struct kse_upcall * 363 upcall_alloc(void) 364 { 365 struct kse_upcall *ku; 366 367 ku = uma_zalloc(upcall_zone, M_WAITOK); 368 bzero(ku, sizeof(*ku)); 369 return (ku); 370 } 371 372 void 373 upcall_free(struct kse_upcall *ku) 374 { 375 376 uma_zfree(upcall_zone, ku); 377 } 378 379 void 380 upcall_link(struct kse_upcall *ku, struct ksegrp *kg) 381 { 382 383 mtx_assert(&sched_lock, MA_OWNED); 384 TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 385 ku->ku_ksegrp = kg; 386 kg->kg_numupcalls++; 387 } 388 389 void 390 upcall_unlink(struct kse_upcall *ku) 391 { 392 struct ksegrp *kg = ku->ku_ksegrp; 393 394 mtx_assert(&sched_lock, MA_OWNED); 395 KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 396 TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 397 kg->kg_numupcalls--; 398 upcall_stash(ku); 399 } 400 401 void 402 upcall_remove(struct thread *td) 403 { 404 405 if (td->td_upcall) { 406 td->td_upcall->ku_owner = NULL; 407 upcall_unlink(td->td_upcall); 408 td->td_upcall = 0; 409 } 410 } 411 412 /* 413 * For a newly created process, 414 * link up all the structures and its initial threads etc. 415 */ 416 void 417 proc_linkup(struct proc *p, struct ksegrp *kg, 418 struct kse *ke, struct thread *td) 419 { 420 421 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 422 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 423 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 424 p->p_numksegrps = 0; 425 p->p_numthreads = 0; 426 427 ksegrp_link(kg, p); 428 kse_link(ke, kg); 429 thread_link(td, kg); 430 } 431 432 #ifndef _SYS_SYSPROTO_H_ 433 struct kse_switchin_args { 434 const struct __mcontext *mcp; 435 long val; 436 long *loc; 437 }; 438 #endif 439 440 int 441 kse_switchin(struct thread *td, struct kse_switchin_args *uap) 442 { 443 mcontext_t mc; 444 int error; 445 446 error = (uap->mcp == NULL) ? EINVAL : 0; 447 if (!error) 448 error = copyin(uap->mcp, &mc, sizeof(mc)); 449 if (!error && uap->loc != NULL) 450 error = (suword(uap->loc, uap->val) != 0) ? EINVAL : 0; 451 if (!error) 452 error = set_mcontext(td, &mc); 453 return ((error == 0) ? EJUSTRETURN : error); 454 } 455 456 /* 457 struct kse_thr_interrupt_args { 458 struct kse_thr_mailbox * tmbx; 459 int cmd; 460 long data; 461 }; 462 */ 463 int 464 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 465 { 466 struct proc *p; 467 struct thread *td2; 468 469 p = td->td_proc; 470 471 if (!(p->p_flag & P_SA)) 472 return (EINVAL); 473 474 switch (uap->cmd) { 475 case KSE_INTR_SENDSIG: 476 if (uap->data < 0 || uap->data > _SIG_MAXSIG) 477 return (EINVAL); 478 case KSE_INTR_INTERRUPT: 479 case KSE_INTR_RESTART: 480 PROC_LOCK(p); 481 mtx_lock_spin(&sched_lock); 482 FOREACH_THREAD_IN_PROC(p, td2) { 483 if (td2->td_mailbox == uap->tmbx) 484 break; 485 } 486 if (td2 == NULL) { 487 mtx_unlock_spin(&sched_lock); 488 PROC_UNLOCK(p); 489 return (ESRCH); 490 } 491 if (uap->cmd == KSE_INTR_SENDSIG) { 492 if (uap->data > 0) { 493 td2->td_flags &= ~TDF_INTERRUPT; 494 mtx_unlock_spin(&sched_lock); 495 tdsignal(td2, (int)uap->data, SIGTARGET_TD); 496 } else { 497 mtx_unlock_spin(&sched_lock); 498 } 499 } else { 500 td2->td_flags |= TDF_INTERRUPT | TDF_ASTPENDING; 501 if (TD_CAN_UNBIND(td2)) 502 td2->td_upcall->ku_flags |= KUF_DOUPCALL; 503 if (uap->cmd == KSE_INTR_INTERRUPT) 504 td2->td_intrval = EINTR; 505 else 506 td2->td_intrval = ERESTART; 507 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) 508 sleepq_abort(td2); 509 mtx_unlock_spin(&sched_lock); 510 } 511 PROC_UNLOCK(p); 512 break; 513 case KSE_INTR_SIGEXIT: 514 if (uap->data < 1 || uap->data > _SIG_MAXSIG) 515 return (EINVAL); 516 PROC_LOCK(p); 517 sigexit(td, (int)uap->data); 518 break; 519 default: 520 return (EINVAL); 521 } 522 return (0); 523 } 524 525 /* 526 struct kse_exit_args { 527 register_t dummy; 528 }; 529 */ 530 int 531 kse_exit(struct thread *td, struct kse_exit_args *uap) 532 { 533 struct proc *p; 534 struct ksegrp *kg; 535 struct kse *ke; 536 struct kse_upcall *ku, *ku2; 537 int error, count; 538 539 p = td->td_proc; 540 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 541 return (EINVAL); 542 kg = td->td_ksegrp; 543 count = 0; 544 PROC_LOCK(p); 545 mtx_lock_spin(&sched_lock); 546 FOREACH_UPCALL_IN_GROUP(kg, ku2) { 547 if (ku2->ku_flags & KUF_EXITING) 548 count++; 549 } 550 if ((kg->kg_numupcalls - count) == 1 && 551 (kg->kg_numthreads > 1)) { 552 mtx_unlock_spin(&sched_lock); 553 PROC_UNLOCK(p); 554 return (EDEADLK); 555 } 556 ku->ku_flags |= KUF_EXITING; 557 mtx_unlock_spin(&sched_lock); 558 PROC_UNLOCK(p); 559 error = suword(&ku->ku_mailbox->km_flags, ku->ku_mflags|KMF_DONE); 560 PROC_LOCK(p); 561 if (error) 562 psignal(p, SIGSEGV); 563 mtx_lock_spin(&sched_lock); 564 upcall_remove(td); 565 ke = td->td_kse; 566 if (p->p_numthreads == 1) { 567 kse_purge(p, td); 568 p->p_flag &= ~P_SA; 569 mtx_unlock_spin(&sched_lock); 570 PROC_UNLOCK(p); 571 } else { 572 if (kg->kg_numthreads == 1) { /* Shutdown a group */ 573 kse_purge_group(td); 574 ke->ke_flags |= KEF_EXIT; 575 } 576 thread_stopped(p); 577 thread_exit(); 578 /* NOTREACHED */ 579 } 580 return (0); 581 } 582 583 /* 584 * Either becomes an upcall or waits for an awakening event and 585 * then becomes an upcall. Only error cases return. 586 */ 587 /* 588 struct kse_release_args { 589 struct timespec *timeout; 590 }; 591 */ 592 int 593 kse_release(struct thread *td, struct kse_release_args *uap) 594 { 595 struct proc *p; 596 struct ksegrp *kg; 597 struct kse_upcall *ku; 598 struct timespec timeout; 599 struct timeval tv; 600 sigset_t sigset; 601 int error; 602 603 p = td->td_proc; 604 kg = td->td_ksegrp; 605 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 606 return (EINVAL); 607 if (uap->timeout != NULL) { 608 if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 609 return (error); 610 TIMESPEC_TO_TIMEVAL(&tv, &timeout); 611 } 612 if (td->td_flags & TDF_SA) 613 td->td_pflags |= TDP_UPCALLING; 614 else { 615 ku->ku_mflags = fuword(&ku->ku_mailbox->km_flags); 616 if (ku->ku_mflags == -1) { 617 PROC_LOCK(p); 618 sigexit(td, SIGSEGV); 619 } 620 } 621 PROC_LOCK(p); 622 if (ku->ku_mflags & KMF_WAITSIGEVENT) { 623 /* UTS wants to wait for signal event */ 624 if (!(p->p_flag & P_SIGEVENT) && !(ku->ku_flags & KUF_DOUPCALL)) { 625 td->td_kflags |= TDK_KSERELSIG; 626 error = msleep(&p->p_siglist, &p->p_mtx, PPAUSE|PCATCH, 627 "ksesigwait", (uap->timeout ? tvtohz(&tv) : 0)); 628 td->td_kflags &= ~(TDK_KSERELSIG | TDK_WAKEUP); 629 } 630 p->p_flag &= ~P_SIGEVENT; 631 sigset = p->p_siglist; 632 PROC_UNLOCK(p); 633 error = copyout(&sigset, &ku->ku_mailbox->km_sigscaught, 634 sizeof(sigset)); 635 } else { 636 if (! kg->kg_completed && !(ku->ku_flags & KUF_DOUPCALL)) { 637 kg->kg_upsleeps++; 638 td->td_kflags |= TDK_KSEREL; 639 error = msleep(&kg->kg_completed, &p->p_mtx, 640 PPAUSE|PCATCH, "kserel", 641 (uap->timeout ? tvtohz(&tv) : 0)); 642 td->td_kflags &= ~(TDK_KSEREL | TDK_WAKEUP); 643 kg->kg_upsleeps--; 644 } 645 PROC_UNLOCK(p); 646 } 647 if (ku->ku_flags & KUF_DOUPCALL) { 648 mtx_lock_spin(&sched_lock); 649 ku->ku_flags &= ~KUF_DOUPCALL; 650 mtx_unlock_spin(&sched_lock); 651 } 652 return (0); 653 } 654 655 /* struct kse_wakeup_args { 656 struct kse_mailbox *mbx; 657 }; */ 658 int 659 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 660 { 661 struct proc *p; 662 struct ksegrp *kg; 663 struct kse_upcall *ku; 664 struct thread *td2; 665 666 p = td->td_proc; 667 td2 = NULL; 668 ku = NULL; 669 /* KSE-enabled processes only, please. */ 670 if (!(p->p_flag & P_SA)) 671 return (EINVAL); 672 PROC_LOCK(p); 673 mtx_lock_spin(&sched_lock); 674 if (uap->mbx) { 675 FOREACH_KSEGRP_IN_PROC(p, kg) { 676 FOREACH_UPCALL_IN_GROUP(kg, ku) { 677 if (ku->ku_mailbox == uap->mbx) 678 break; 679 } 680 if (ku) 681 break; 682 } 683 } else { 684 kg = td->td_ksegrp; 685 if (kg->kg_upsleeps) { 686 mtx_unlock_spin(&sched_lock); 687 wakeup_one(&kg->kg_completed); 688 PROC_UNLOCK(p); 689 return (0); 690 } 691 ku = TAILQ_FIRST(&kg->kg_upcalls); 692 } 693 if (ku == NULL) { 694 mtx_unlock_spin(&sched_lock); 695 PROC_UNLOCK(p); 696 return (ESRCH); 697 } 698 if ((td2 = ku->ku_owner) == NULL) { 699 mtx_unlock_spin(&sched_lock); 700 panic("%s: no owner", __func__); 701 } else if (td2->td_kflags & (TDK_KSEREL | TDK_KSERELSIG)) { 702 mtx_unlock_spin(&sched_lock); 703 if (!(td2->td_kflags & TDK_WAKEUP)) { 704 td2->td_kflags |= TDK_WAKEUP; 705 if (td2->td_kflags & TDK_KSEREL) 706 sleepq_remove(td2, &kg->kg_completed); 707 else 708 sleepq_remove(td2, &p->p_siglist); 709 } 710 } else { 711 ku->ku_flags |= KUF_DOUPCALL; 712 mtx_unlock_spin(&sched_lock); 713 } 714 PROC_UNLOCK(p); 715 return (0); 716 } 717 718 /* 719 * No new KSEG: first call: use current KSE, don't schedule an upcall 720 * All other situations, do allocate max new KSEs and schedule an upcall. 721 */ 722 /* struct kse_create_args { 723 struct kse_mailbox *mbx; 724 int newgroup; 725 }; */ 726 int 727 kse_create(struct thread *td, struct kse_create_args *uap) 728 { 729 struct kse *newke; 730 struct ksegrp *newkg; 731 struct ksegrp *kg; 732 struct proc *p; 733 struct kse_mailbox mbx; 734 struct kse_upcall *newku; 735 int err, ncpus, sa = 0, first = 0; 736 struct thread *newtd; 737 738 p = td->td_proc; 739 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 740 return (err); 741 742 /* Too bad, why hasn't kernel always a cpu counter !? */ 743 #ifdef SMP 744 ncpus = mp_ncpus; 745 #else 746 ncpus = 1; 747 #endif 748 if (virtual_cpu != 0) 749 ncpus = virtual_cpu; 750 if (!(mbx.km_flags & KMF_BOUND)) 751 sa = TDF_SA; 752 else 753 ncpus = 1; 754 PROC_LOCK(p); 755 if (!(p->p_flag & P_SA)) { 756 first = 1; 757 p->p_flag |= P_SA; 758 } 759 PROC_UNLOCK(p); 760 if (!sa && !uap->newgroup && !first) 761 return (EINVAL); 762 kg = td->td_ksegrp; 763 if (uap->newgroup) { 764 /* Have race condition but it is cheap */ 765 if (p->p_numksegrps >= max_groups_per_proc) 766 return (EPROCLIM); 767 /* 768 * If we want a new KSEGRP it doesn't matter whether 769 * we have already fired up KSE mode before or not. 770 * We put the process in KSE mode and create a new KSEGRP. 771 */ 772 newkg = ksegrp_alloc(); 773 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 774 kg_startzero, kg_endzero)); 775 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 776 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 777 PROC_LOCK(p); 778 mtx_lock_spin(&sched_lock); 779 if (p->p_numksegrps >= max_groups_per_proc) { 780 mtx_unlock_spin(&sched_lock); 781 PROC_UNLOCK(p); 782 ksegrp_free(newkg); 783 return (EPROCLIM); 784 } 785 ksegrp_link(newkg, p); 786 sched_fork_ksegrp(kg, newkg); 787 mtx_unlock_spin(&sched_lock); 788 PROC_UNLOCK(p); 789 } else { 790 if (!first && ((td->td_flags & TDF_SA) ^ sa) != 0) 791 return (EINVAL); 792 newkg = kg; 793 } 794 795 /* 796 * Creating upcalls more than number of physical cpu does 797 * not help performance. 798 */ 799 if (newkg->kg_numupcalls >= ncpus) 800 return (EPROCLIM); 801 802 if (newkg->kg_numupcalls == 0) { 803 /* 804 * Initialize KSE group 805 * 806 * For multiplxed group, create KSEs as many as physical 807 * cpus. This increases concurrent even if userland 808 * is not MP safe and can only run on single CPU. 809 * In ideal world, every physical cpu should execute a thread. 810 * If there is enough KSEs, threads in kernel can be 811 * executed parallel on different cpus with full speed, 812 * Concurrent in kernel shouldn't be restricted by number of 813 * upcalls userland provides. Adding more upcall structures 814 * only increases concurrent in userland. 815 * 816 * For bound thread group, because there is only thread in the 817 * group, we only create one KSE for the group. Thread in this 818 * kind of group will never schedule an upcall when blocked, 819 * this intends to simulate pthread system scope thread. 820 */ 821 while (newkg->kg_kses < ncpus) { 822 newke = kse_alloc(); 823 bzero(&newke->ke_startzero, RANGEOF(struct kse, 824 ke_startzero, ke_endzero)); 825 #if 0 826 mtx_lock_spin(&sched_lock); 827 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 828 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 829 mtx_unlock_spin(&sched_lock); 830 #endif 831 mtx_lock_spin(&sched_lock); 832 kse_link(newke, newkg); 833 sched_fork_kse(td->td_kse, newke); 834 /* Add engine */ 835 kse_reassign(newke); 836 mtx_unlock_spin(&sched_lock); 837 } 838 } 839 newku = upcall_alloc(); 840 newku->ku_mailbox = uap->mbx; 841 newku->ku_func = mbx.km_func; 842 bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 843 844 /* For the first call this may not have been set */ 845 if (td->td_standin == NULL) 846 thread_alloc_spare(td, NULL); 847 848 PROC_LOCK(p); 849 if (newkg->kg_numupcalls >= ncpus) { 850 PROC_UNLOCK(p); 851 upcall_free(newku); 852 return (EPROCLIM); 853 } 854 if (first && sa) { 855 SIGSETOR(p->p_siglist, td->td_siglist); 856 SIGEMPTYSET(td->td_siglist); 857 SIGFILLSET(td->td_sigmask); 858 SIG_CANTMASK(td->td_sigmask); 859 } 860 mtx_lock_spin(&sched_lock); 861 PROC_UNLOCK(p); 862 upcall_link(newku, newkg); 863 if (mbx.km_quantum) 864 newkg->kg_upquantum = max(1, mbx.km_quantum/tick); 865 866 /* 867 * Each upcall structure has an owner thread, find which 868 * one owns it. 869 */ 870 if (uap->newgroup) { 871 /* 872 * Because new ksegrp hasn't thread, 873 * create an initial upcall thread to own it. 874 */ 875 newtd = thread_schedule_upcall(td, newku); 876 } else { 877 /* 878 * If current thread hasn't an upcall structure, 879 * just assign the upcall to it. 880 */ 881 if (td->td_upcall == NULL) { 882 newku->ku_owner = td; 883 td->td_upcall = newku; 884 newtd = td; 885 } else { 886 /* 887 * Create a new upcall thread to own it. 888 */ 889 newtd = thread_schedule_upcall(td, newku); 890 } 891 } 892 if (!sa) { 893 newtd->td_mailbox = mbx.km_curthread; 894 newtd->td_flags &= ~TDF_SA; 895 if (newtd != td) { 896 mtx_unlock_spin(&sched_lock); 897 cpu_set_upcall_kse(newtd, newku); 898 mtx_lock_spin(&sched_lock); 899 } 900 } else { 901 newtd->td_flags |= TDF_SA; 902 } 903 if (newtd != td) 904 setrunqueue(newtd); 905 mtx_unlock_spin(&sched_lock); 906 return (0); 907 } 908 909 /* 910 * Initialize global thread allocation resources. 911 */ 912 void 913 threadinit(void) 914 { 915 916 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 917 thread_ctor, thread_dtor, thread_init, thread_fini, 918 UMA_ALIGN_CACHE, 0); 919 tid_zone = uma_zcreate("TID", sizeof(struct tid_bitmap_part), 920 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 921 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 922 NULL, NULL, ksegrp_init, NULL, 923 UMA_ALIGN_CACHE, 0); 924 kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 925 NULL, NULL, kse_init, NULL, 926 UMA_ALIGN_CACHE, 0); 927 upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 928 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 929 } 930 931 /* 932 * Stash an embarasingly extra thread into the zombie thread queue. 933 */ 934 void 935 thread_stash(struct thread *td) 936 { 937 mtx_lock_spin(&kse_zombie_lock); 938 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 939 mtx_unlock_spin(&kse_zombie_lock); 940 } 941 942 /* 943 * Stash an embarasingly extra kse into the zombie kse queue. 944 */ 945 void 946 kse_stash(struct kse *ke) 947 { 948 mtx_lock_spin(&kse_zombie_lock); 949 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 950 mtx_unlock_spin(&kse_zombie_lock); 951 } 952 953 /* 954 * Stash an embarasingly extra upcall into the zombie upcall queue. 955 */ 956 957 void 958 upcall_stash(struct kse_upcall *ku) 959 { 960 mtx_lock_spin(&kse_zombie_lock); 961 TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 962 mtx_unlock_spin(&kse_zombie_lock); 963 } 964 965 /* 966 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 967 */ 968 void 969 ksegrp_stash(struct ksegrp *kg) 970 { 971 mtx_lock_spin(&kse_zombie_lock); 972 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 973 mtx_unlock_spin(&kse_zombie_lock); 974 } 975 976 /* 977 * Reap zombie kse resource. 978 */ 979 void 980 thread_reap(void) 981 { 982 struct thread *td_first, *td_next; 983 struct kse *ke_first, *ke_next; 984 struct ksegrp *kg_first, * kg_next; 985 struct kse_upcall *ku_first, *ku_next; 986 987 /* 988 * Don't even bother to lock if none at this instant, 989 * we really don't care about the next instant.. 990 */ 991 if ((!TAILQ_EMPTY(&zombie_threads)) 992 || (!TAILQ_EMPTY(&zombie_kses)) 993 || (!TAILQ_EMPTY(&zombie_ksegrps)) 994 || (!TAILQ_EMPTY(&zombie_upcalls))) { 995 mtx_lock_spin(&kse_zombie_lock); 996 td_first = TAILQ_FIRST(&zombie_threads); 997 ke_first = TAILQ_FIRST(&zombie_kses); 998 kg_first = TAILQ_FIRST(&zombie_ksegrps); 999 ku_first = TAILQ_FIRST(&zombie_upcalls); 1000 if (td_first) 1001 TAILQ_INIT(&zombie_threads); 1002 if (ke_first) 1003 TAILQ_INIT(&zombie_kses); 1004 if (kg_first) 1005 TAILQ_INIT(&zombie_ksegrps); 1006 if (ku_first) 1007 TAILQ_INIT(&zombie_upcalls); 1008 mtx_unlock_spin(&kse_zombie_lock); 1009 while (td_first) { 1010 td_next = TAILQ_NEXT(td_first, td_runq); 1011 if (td_first->td_ucred) 1012 crfree(td_first->td_ucred); 1013 thread_free(td_first); 1014 td_first = td_next; 1015 } 1016 while (ke_first) { 1017 ke_next = TAILQ_NEXT(ke_first, ke_procq); 1018 kse_free(ke_first); 1019 ke_first = ke_next; 1020 } 1021 while (kg_first) { 1022 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 1023 ksegrp_free(kg_first); 1024 kg_first = kg_next; 1025 } 1026 while (ku_first) { 1027 ku_next = TAILQ_NEXT(ku_first, ku_link); 1028 upcall_free(ku_first); 1029 ku_first = ku_next; 1030 } 1031 } 1032 } 1033 1034 /* 1035 * Allocate a ksegrp. 1036 */ 1037 struct ksegrp * 1038 ksegrp_alloc(void) 1039 { 1040 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 1041 } 1042 1043 /* 1044 * Allocate a kse. 1045 */ 1046 struct kse * 1047 kse_alloc(void) 1048 { 1049 return (uma_zalloc(kse_zone, M_WAITOK)); 1050 } 1051 1052 /* 1053 * Allocate a thread. 1054 */ 1055 struct thread * 1056 thread_alloc(void) 1057 { 1058 thread_reap(); /* check if any zombies to get */ 1059 return (uma_zalloc(thread_zone, M_WAITOK)); 1060 } 1061 1062 /* 1063 * Deallocate a ksegrp. 1064 */ 1065 void 1066 ksegrp_free(struct ksegrp *td) 1067 { 1068 uma_zfree(ksegrp_zone, td); 1069 } 1070 1071 /* 1072 * Deallocate a kse. 1073 */ 1074 void 1075 kse_free(struct kse *td) 1076 { 1077 uma_zfree(kse_zone, td); 1078 } 1079 1080 /* 1081 * Deallocate a thread. 1082 */ 1083 void 1084 thread_free(struct thread *td) 1085 { 1086 1087 cpu_thread_clean(td); 1088 uma_zfree(thread_zone, td); 1089 } 1090 1091 /* 1092 * Assign a thread ID. 1093 */ 1094 int 1095 thread_new_tid(void) 1096 { 1097 struct tid_bitmap_part *bmp, *new; 1098 int bit, idx, tid; 1099 1100 mtx_lock(&tid_lock); 1101 STAILQ_FOREACH(bmp, &tid_bitmap, bmp_next) { 1102 if (bmp->bmp_free) 1103 break; 1104 } 1105 /* Create a new bitmap if we run out of free bits. */ 1106 if (bmp == NULL) { 1107 mtx_unlock(&tid_lock); 1108 new = uma_zalloc(tid_zone, M_WAITOK); 1109 mtx_lock(&tid_lock); 1110 bmp = STAILQ_LAST(&tid_bitmap, tid_bitmap_part, bmp_next); 1111 if (bmp == NULL || bmp->bmp_free < TID_IDS_PER_PART/2) { 1112 /* 1=free, 0=assigned. This way we can use ffsl(). */ 1113 memset(new->bmp_bitmap, ~0U, sizeof(new->bmp_bitmap)); 1114 new->bmp_base = (bmp == NULL) ? TID_MIN : 1115 bmp->bmp_base + TID_IDS_PER_PART; 1116 new->bmp_free = TID_IDS_PER_PART; 1117 STAILQ_INSERT_TAIL(&tid_bitmap, new, bmp_next); 1118 bmp = new; 1119 new = NULL; 1120 } 1121 } else 1122 new = NULL; 1123 /* We have a bitmap with available IDs. */ 1124 idx = 0; 1125 while (idx < TID_BITMAP_SIZE && bmp->bmp_bitmap[idx] == 0UL) 1126 idx++; 1127 bit = ffsl(bmp->bmp_bitmap[idx]) - 1; 1128 tid = bmp->bmp_base + idx * TID_IDS_PER_IDX + bit; 1129 bmp->bmp_bitmap[idx] &= ~(1UL << bit); 1130 bmp->bmp_free--; 1131 mtx_unlock(&tid_lock); 1132 1133 if (new != NULL) 1134 uma_zfree(tid_zone, new); 1135 return (tid); 1136 } 1137 1138 /* 1139 * Store the thread context in the UTS's mailbox. 1140 * then add the mailbox at the head of a list we are building in user space. 1141 * The list is anchored in the ksegrp structure. 1142 */ 1143 int 1144 thread_export_context(struct thread *td, int willexit) 1145 { 1146 struct proc *p; 1147 struct ksegrp *kg; 1148 uintptr_t mbx; 1149 void *addr; 1150 int error = 0, temp, sig; 1151 mcontext_t mc; 1152 1153 p = td->td_proc; 1154 kg = td->td_ksegrp; 1155 1156 /* Export the user/machine context. */ 1157 get_mcontext(td, &mc, 0); 1158 addr = (void *)(&td->td_mailbox->tm_context.uc_mcontext); 1159 error = copyout(&mc, addr, sizeof(mcontext_t)); 1160 if (error) 1161 goto bad; 1162 1163 /* Exports clock ticks in kernel mode */ 1164 addr = (caddr_t)(&td->td_mailbox->tm_sticks); 1165 temp = fuword32(addr) + td->td_usticks; 1166 if (suword32(addr, temp)) { 1167 error = EFAULT; 1168 goto bad; 1169 } 1170 1171 /* 1172 * Post sync signal, or process SIGKILL and SIGSTOP. 1173 * For sync signal, it is only possible when the signal is not 1174 * caught by userland or process is being debugged. 1175 */ 1176 PROC_LOCK(p); 1177 if (td->td_flags & TDF_NEEDSIGCHK) { 1178 mtx_lock_spin(&sched_lock); 1179 td->td_flags &= ~TDF_NEEDSIGCHK; 1180 mtx_unlock_spin(&sched_lock); 1181 mtx_lock(&p->p_sigacts->ps_mtx); 1182 while ((sig = cursig(td)) != 0) 1183 postsig(sig); 1184 mtx_unlock(&p->p_sigacts->ps_mtx); 1185 } 1186 if (willexit) 1187 SIGFILLSET(td->td_sigmask); 1188 PROC_UNLOCK(p); 1189 1190 /* Get address in latest mbox of list pointer */ 1191 addr = (void *)(&td->td_mailbox->tm_next); 1192 /* 1193 * Put the saved address of the previous first 1194 * entry into this one 1195 */ 1196 for (;;) { 1197 mbx = (uintptr_t)kg->kg_completed; 1198 if (suword(addr, mbx)) { 1199 error = EFAULT; 1200 goto bad; 1201 } 1202 PROC_LOCK(p); 1203 if (mbx == (uintptr_t)kg->kg_completed) { 1204 kg->kg_completed = td->td_mailbox; 1205 /* 1206 * The thread context may be taken away by 1207 * other upcall threads when we unlock 1208 * process lock. it's no longer valid to 1209 * use it again in any other places. 1210 */ 1211 td->td_mailbox = NULL; 1212 PROC_UNLOCK(p); 1213 break; 1214 } 1215 PROC_UNLOCK(p); 1216 } 1217 td->td_usticks = 0; 1218 return (0); 1219 1220 bad: 1221 PROC_LOCK(p); 1222 sigexit(td, SIGILL); 1223 return (error); 1224 } 1225 1226 /* 1227 * Take the list of completed mailboxes for this KSEGRP and put them on this 1228 * upcall's mailbox as it's the next one going up. 1229 */ 1230 static int 1231 thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1232 { 1233 struct proc *p = kg->kg_proc; 1234 void *addr; 1235 uintptr_t mbx; 1236 1237 addr = (void *)(&ku->ku_mailbox->km_completed); 1238 for (;;) { 1239 mbx = (uintptr_t)kg->kg_completed; 1240 if (suword(addr, mbx)) { 1241 PROC_LOCK(p); 1242 psignal(p, SIGSEGV); 1243 PROC_UNLOCK(p); 1244 return (EFAULT); 1245 } 1246 PROC_LOCK(p); 1247 if (mbx == (uintptr_t)kg->kg_completed) { 1248 kg->kg_completed = NULL; 1249 PROC_UNLOCK(p); 1250 break; 1251 } 1252 PROC_UNLOCK(p); 1253 } 1254 return (0); 1255 } 1256 1257 /* 1258 * This function should be called at statclock interrupt time 1259 */ 1260 int 1261 thread_statclock(int user) 1262 { 1263 struct thread *td = curthread; 1264 struct ksegrp *kg = td->td_ksegrp; 1265 1266 if (kg->kg_numupcalls == 0 || !(td->td_flags & TDF_SA)) 1267 return (0); 1268 if (user) { 1269 /* Current always do via ast() */ 1270 mtx_lock_spin(&sched_lock); 1271 td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1272 mtx_unlock_spin(&sched_lock); 1273 td->td_uuticks++; 1274 } else { 1275 if (td->td_mailbox != NULL) 1276 td->td_usticks++; 1277 else { 1278 /* XXXKSE 1279 * We will call thread_user_enter() for every 1280 * kernel entry in future, so if the thread mailbox 1281 * is NULL, it must be a UTS kernel, don't account 1282 * clock ticks for it. 1283 */ 1284 } 1285 } 1286 return (0); 1287 } 1288 1289 /* 1290 * Export state clock ticks for userland 1291 */ 1292 static int 1293 thread_update_usr_ticks(struct thread *td, int user) 1294 { 1295 struct proc *p = td->td_proc; 1296 struct kse_thr_mailbox *tmbx; 1297 struct kse_upcall *ku; 1298 struct ksegrp *kg; 1299 caddr_t addr; 1300 u_int uticks; 1301 1302 if ((ku = td->td_upcall) == NULL) 1303 return (-1); 1304 1305 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1306 if ((tmbx == NULL) || (tmbx == (void *)-1)) 1307 return (-1); 1308 if (user) { 1309 uticks = td->td_uuticks; 1310 td->td_uuticks = 0; 1311 addr = (caddr_t)&tmbx->tm_uticks; 1312 } else { 1313 uticks = td->td_usticks; 1314 td->td_usticks = 0; 1315 addr = (caddr_t)&tmbx->tm_sticks; 1316 } 1317 if (uticks) { 1318 if (suword32(addr, uticks+fuword32(addr))) { 1319 PROC_LOCK(p); 1320 psignal(p, SIGSEGV); 1321 PROC_UNLOCK(p); 1322 return (-2); 1323 } 1324 } 1325 kg = td->td_ksegrp; 1326 if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { 1327 mtx_lock_spin(&sched_lock); 1328 td->td_upcall->ku_flags |= KUF_DOUPCALL; 1329 mtx_unlock_spin(&sched_lock); 1330 } 1331 return (0); 1332 } 1333 1334 /* 1335 * Discard the current thread and exit from its context. 1336 * 1337 * Because we can't free a thread while we're operating under its context, 1338 * push the current thread into our CPU's deadthread holder. This means 1339 * we needn't worry about someone else grabbing our context before we 1340 * do a cpu_throw(). 1341 */ 1342 void 1343 thread_exit(void) 1344 { 1345 struct thread *td; 1346 struct kse *ke; 1347 struct proc *p; 1348 struct ksegrp *kg; 1349 1350 td = curthread; 1351 kg = td->td_ksegrp; 1352 p = td->td_proc; 1353 ke = td->td_kse; 1354 1355 mtx_assert(&sched_lock, MA_OWNED); 1356 KASSERT(p != NULL, ("thread exiting without a process")); 1357 KASSERT(ke != NULL, ("thread exiting without a kse")); 1358 KASSERT(kg != NULL, ("thread exiting without a kse group")); 1359 PROC_LOCK_ASSERT(p, MA_OWNED); 1360 CTR1(KTR_PROC, "thread_exit: thread %p", td); 1361 mtx_assert(&Giant, MA_NOTOWNED); 1362 1363 if (td->td_standin != NULL) { 1364 thread_stash(td->td_standin); 1365 td->td_standin = NULL; 1366 } 1367 1368 cpu_thread_exit(td); /* XXXSMP */ 1369 1370 /* 1371 * The last thread is left attached to the process 1372 * So that the whole bundle gets recycled. Skip 1373 * all this stuff. 1374 */ 1375 if (p->p_numthreads > 1) { 1376 thread_unlink(td); 1377 if (p->p_maxthrwaits) 1378 wakeup(&p->p_numthreads); 1379 /* 1380 * The test below is NOT true if we are the 1381 * sole exiting thread. P_STOPPED_SNGL is unset 1382 * in exit1() after it is the only survivor. 1383 */ 1384 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1385 if (p->p_numthreads == p->p_suspcount) { 1386 thread_unsuspend_one(p->p_singlethread); 1387 } 1388 } 1389 1390 /* 1391 * Because each upcall structure has an owner thread, 1392 * owner thread exits only when process is in exiting 1393 * state, so upcall to userland is no longer needed, 1394 * deleting upcall structure is safe here. 1395 * So when all threads in a group is exited, all upcalls 1396 * in the group should be automatically freed. 1397 */ 1398 if (td->td_upcall) 1399 upcall_remove(td); 1400 1401 sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 1402 sched_exit_kse(FIRST_KSE_IN_PROC(p), ke); 1403 ke->ke_state = KES_UNQUEUED; 1404 ke->ke_thread = NULL; 1405 /* 1406 * Decide what to do with the KSE attached to this thread. 1407 */ 1408 if (ke->ke_flags & KEF_EXIT) { 1409 kse_unlink(ke); 1410 if (kg->kg_kses == 0) { 1411 sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), kg); 1412 ksegrp_unlink(kg); 1413 } 1414 } 1415 else 1416 kse_reassign(ke); 1417 PROC_UNLOCK(p); 1418 td->td_kse = NULL; 1419 td->td_state = TDS_INACTIVE; 1420 #if 0 1421 td->td_proc = NULL; 1422 #endif 1423 td->td_ksegrp = NULL; 1424 td->td_last_kse = NULL; 1425 PCPU_SET(deadthread, td); 1426 } else { 1427 PROC_UNLOCK(p); 1428 } 1429 /* XXX Shouldn't cpu_throw() here. */ 1430 mtx_assert(&sched_lock, MA_OWNED); 1431 cpu_throw(td, choosethread()); 1432 panic("I'm a teapot!"); 1433 /* NOTREACHED */ 1434 } 1435 1436 /* 1437 * Do any thread specific cleanups that may be needed in wait() 1438 * called with Giant, proc and schedlock not held. 1439 */ 1440 void 1441 thread_wait(struct proc *p) 1442 { 1443 struct thread *td; 1444 1445 mtx_assert(&Giant, MA_NOTOWNED); 1446 KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); 1447 KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); 1448 FOREACH_THREAD_IN_PROC(p, td) { 1449 if (td->td_standin != NULL) { 1450 thread_free(td->td_standin); 1451 td->td_standin = NULL; 1452 } 1453 cpu_thread_clean(td); 1454 } 1455 thread_reap(); /* check for zombie threads etc. */ 1456 } 1457 1458 /* 1459 * Link a thread to a process. 1460 * set up anything that needs to be initialized for it to 1461 * be used by the process. 1462 * 1463 * Note that we do not link to the proc's ucred here. 1464 * The thread is linked as if running but no KSE assigned. 1465 */ 1466 void 1467 thread_link(struct thread *td, struct ksegrp *kg) 1468 { 1469 struct proc *p; 1470 1471 p = kg->kg_proc; 1472 td->td_state = TDS_INACTIVE; 1473 td->td_proc = p; 1474 td->td_ksegrp = kg; 1475 td->td_last_kse = NULL; 1476 td->td_flags = 0; 1477 td->td_kflags = 0; 1478 td->td_kse = NULL; 1479 1480 LIST_INIT(&td->td_contested); 1481 callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 1482 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 1483 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 1484 p->p_numthreads++; 1485 kg->kg_numthreads++; 1486 } 1487 1488 void 1489 thread_unlink(struct thread *td) 1490 { 1491 struct proc *p = td->td_proc; 1492 struct ksegrp *kg = td->td_ksegrp; 1493 1494 mtx_assert(&sched_lock, MA_OWNED); 1495 TAILQ_REMOVE(&p->p_threads, td, td_plist); 1496 p->p_numthreads--; 1497 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1498 kg->kg_numthreads--; 1499 /* could clear a few other things here */ 1500 } 1501 1502 /* 1503 * Purge a ksegrp resource. When a ksegrp is preparing to 1504 * exit, it calls this function. 1505 */ 1506 static void 1507 kse_purge_group(struct thread *td) 1508 { 1509 struct ksegrp *kg; 1510 struct kse *ke; 1511 1512 kg = td->td_ksegrp; 1513 KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1514 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1515 KASSERT(ke->ke_state == KES_IDLE, 1516 ("%s: wrong idle KSE state", __func__)); 1517 kse_unlink(ke); 1518 } 1519 KASSERT((kg->kg_kses == 1), 1520 ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1521 KASSERT((kg->kg_numupcalls == 0), 1522 ("%s: ksegrp still has %d upcall datas", 1523 __func__, kg->kg_numupcalls)); 1524 } 1525 1526 /* 1527 * Purge a process's KSE resource. When a process is preparing to 1528 * exit, it calls kse_purge to release any extra KSE resources in 1529 * the process. 1530 */ 1531 static void 1532 kse_purge(struct proc *p, struct thread *td) 1533 { 1534 struct ksegrp *kg; 1535 struct kse *ke; 1536 1537 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1538 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1539 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1540 p->p_numksegrps--; 1541 /* 1542 * There is no ownership for KSE, after all threads 1543 * in the group exited, it is possible that some KSEs 1544 * were left in idle queue, gc them now. 1545 */ 1546 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1547 KASSERT(ke->ke_state == KES_IDLE, 1548 ("%s: wrong idle KSE state", __func__)); 1549 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1550 kg->kg_idle_kses--; 1551 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1552 kg->kg_kses--; 1553 kse_stash(ke); 1554 } 1555 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1556 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1557 ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1558 KASSERT((kg->kg_numupcalls == 0), 1559 ("%s: ksegrp still has %d upcall datas", 1560 __func__, kg->kg_numupcalls)); 1561 1562 if (kg != td->td_ksegrp) 1563 ksegrp_stash(kg); 1564 } 1565 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1566 p->p_numksegrps++; 1567 } 1568 1569 /* 1570 * This function is intended to be used to initialize a spare thread 1571 * for upcall. Initialize thread's large data area outside sched_lock 1572 * for thread_schedule_upcall(). 1573 */ 1574 void 1575 thread_alloc_spare(struct thread *td, struct thread *spare) 1576 { 1577 1578 if (td->td_standin) 1579 return; 1580 if (spare == NULL) { 1581 spare = thread_alloc(); 1582 spare->td_tid = thread_new_tid(); 1583 } 1584 td->td_standin = spare; 1585 bzero(&spare->td_startzero, 1586 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1587 spare->td_proc = td->td_proc; 1588 spare->td_ucred = crhold(td->td_ucred); 1589 } 1590 1591 /* 1592 * Create a thread and schedule it for upcall on the KSE given. 1593 * Use our thread's standin so that we don't have to allocate one. 1594 */ 1595 struct thread * 1596 thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 1597 { 1598 struct thread *td2; 1599 1600 mtx_assert(&sched_lock, MA_OWNED); 1601 1602 /* 1603 * Schedule an upcall thread on specified kse_upcall, 1604 * the kse_upcall must be free. 1605 * td must have a spare thread. 1606 */ 1607 KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1608 if ((td2 = td->td_standin) != NULL) { 1609 td->td_standin = NULL; 1610 } else { 1611 panic("no reserve thread when scheduling an upcall"); 1612 return (NULL); 1613 } 1614 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1615 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1616 bcopy(&td->td_startcopy, &td2->td_startcopy, 1617 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1618 thread_link(td2, ku->ku_ksegrp); 1619 /* inherit blocked thread's context */ 1620 cpu_set_upcall(td2, td); 1621 /* Let the new thread become owner of the upcall */ 1622 ku->ku_owner = td2; 1623 td2->td_upcall = ku; 1624 td2->td_flags = TDF_SA; 1625 td2->td_pflags = TDP_UPCALLING; 1626 td2->td_kse = NULL; 1627 td2->td_state = TDS_CAN_RUN; 1628 td2->td_inhibitors = 0; 1629 SIGFILLSET(td2->td_sigmask); 1630 SIG_CANTMASK(td2->td_sigmask); 1631 sched_fork_thread(td, td2); 1632 return (td2); /* bogus.. should be a void function */ 1633 } 1634 1635 /* 1636 * It is only used when thread generated a trap and process is being 1637 * debugged. 1638 */ 1639 void 1640 thread_signal_add(struct thread *td, int sig) 1641 { 1642 struct proc *p; 1643 siginfo_t siginfo; 1644 struct sigacts *ps; 1645 int error; 1646 1647 p = td->td_proc; 1648 PROC_LOCK_ASSERT(p, MA_OWNED); 1649 ps = p->p_sigacts; 1650 mtx_assert(&ps->ps_mtx, MA_OWNED); 1651 1652 cpu_thread_siginfo(sig, 0, &siginfo); 1653 mtx_unlock(&ps->ps_mtx); 1654 PROC_UNLOCK(p); 1655 error = copyout(&siginfo, &td->td_mailbox->tm_syncsig, sizeof(siginfo)); 1656 if (error) { 1657 PROC_LOCK(p); 1658 sigexit(td, SIGILL); 1659 } 1660 PROC_LOCK(p); 1661 SIGADDSET(td->td_sigmask, sig); 1662 mtx_lock(&ps->ps_mtx); 1663 } 1664 1665 void 1666 thread_switchout(struct thread *td) 1667 { 1668 struct kse_upcall *ku; 1669 struct thread *td2; 1670 1671 mtx_assert(&sched_lock, MA_OWNED); 1672 1673 /* 1674 * If the outgoing thread is in threaded group and has never 1675 * scheduled an upcall, decide whether this is a short 1676 * or long term event and thus whether or not to schedule 1677 * an upcall. 1678 * If it is a short term event, just suspend it in 1679 * a way that takes its KSE with it. 1680 * Select the events for which we want to schedule upcalls. 1681 * For now it's just sleep. 1682 * XXXKSE eventually almost any inhibition could do. 1683 */ 1684 if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { 1685 /* 1686 * Release ownership of upcall, and schedule an upcall 1687 * thread, this new upcall thread becomes the owner of 1688 * the upcall structure. 1689 */ 1690 ku = td->td_upcall; 1691 ku->ku_owner = NULL; 1692 td->td_upcall = NULL; 1693 td->td_flags &= ~TDF_CAN_UNBIND; 1694 td2 = thread_schedule_upcall(td, ku); 1695 setrunqueue(td2); 1696 } 1697 } 1698 1699 /* 1700 * Setup done on the thread when it enters the kernel. 1701 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1702 */ 1703 void 1704 thread_user_enter(struct proc *p, struct thread *td) 1705 { 1706 struct ksegrp *kg; 1707 struct kse_upcall *ku; 1708 struct kse_thr_mailbox *tmbx; 1709 uint32_t tflags; 1710 1711 kg = td->td_ksegrp; 1712 1713 /* 1714 * First check that we shouldn't just abort. 1715 * But check if we are the single thread first! 1716 */ 1717 if (p->p_flag & P_SINGLE_EXIT) { 1718 PROC_LOCK(p); 1719 mtx_lock_spin(&sched_lock); 1720 thread_stopped(p); 1721 thread_exit(); 1722 /* NOTREACHED */ 1723 } 1724 1725 /* 1726 * If we are doing a syscall in a KSE environment, 1727 * note where our mailbox is. There is always the 1728 * possibility that we could do this lazily (in kse_reassign()), 1729 * but for now do it every time. 1730 */ 1731 kg = td->td_ksegrp; 1732 if (td->td_flags & TDF_SA) { 1733 ku = td->td_upcall; 1734 KASSERT(ku, ("%s: no upcall owned", __func__)); 1735 KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1736 KASSERT(!TD_CAN_UNBIND(td), ("%s: can unbind", __func__)); 1737 ku->ku_mflags = fuword32((void *)&ku->ku_mailbox->km_flags); 1738 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1739 if ((tmbx == NULL) || (tmbx == (void *)-1L) || 1740 (ku->ku_mflags & KMF_NOUPCALL)) { 1741 td->td_mailbox = NULL; 1742 } else { 1743 if (td->td_standin == NULL) 1744 thread_alloc_spare(td, NULL); 1745 tflags = fuword32(&tmbx->tm_flags); 1746 /* 1747 * On some architectures, TP register points to thread 1748 * mailbox but not points to kse mailbox, and userland 1749 * can not atomically clear km_curthread, but can 1750 * use TP register, and set TMF_NOUPCALL in thread 1751 * flag to indicate a critical region. 1752 */ 1753 if (tflags & TMF_NOUPCALL) { 1754 td->td_mailbox = NULL; 1755 } else { 1756 td->td_mailbox = tmbx; 1757 mtx_lock_spin(&sched_lock); 1758 td->td_flags |= TDF_CAN_UNBIND; 1759 mtx_unlock_spin(&sched_lock); 1760 } 1761 } 1762 } 1763 } 1764 1765 /* 1766 * The extra work we go through if we are a threaded process when we 1767 * return to userland. 1768 * 1769 * If we are a KSE process and returning to user mode, check for 1770 * extra work to do before we return (e.g. for more syscalls 1771 * to complete first). If we were in a critical section, we should 1772 * just return to let it finish. Same if we were in the UTS (in 1773 * which case the mailbox's context's busy indicator will be set). 1774 * The only traps we suport will have set the mailbox. 1775 * We will clear it here. 1776 */ 1777 int 1778 thread_userret(struct thread *td, struct trapframe *frame) 1779 { 1780 int error = 0, upcalls, uts_crit; 1781 struct kse_upcall *ku; 1782 struct ksegrp *kg, *kg2; 1783 struct proc *p; 1784 struct timespec ts; 1785 1786 p = td->td_proc; 1787 kg = td->td_ksegrp; 1788 ku = td->td_upcall; 1789 1790 /* Nothing to do with bound thread */ 1791 if (!(td->td_flags & TDF_SA)) 1792 return (0); 1793 1794 /* 1795 * Stat clock interrupt hit in userland, it 1796 * is returning from interrupt, charge thread's 1797 * userland time for UTS. 1798 */ 1799 if (td->td_flags & TDF_USTATCLOCK) { 1800 thread_update_usr_ticks(td, 1); 1801 mtx_lock_spin(&sched_lock); 1802 td->td_flags &= ~TDF_USTATCLOCK; 1803 mtx_unlock_spin(&sched_lock); 1804 if (kg->kg_completed || 1805 (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1806 thread_user_enter(p, td); 1807 } 1808 1809 uts_crit = (td->td_mailbox == NULL); 1810 /* 1811 * Optimisation: 1812 * This thread has not started any upcall. 1813 * If there is no work to report other than ourself, 1814 * then it can return direct to userland. 1815 */ 1816 if (TD_CAN_UNBIND(td)) { 1817 mtx_lock_spin(&sched_lock); 1818 td->td_flags &= ~TDF_CAN_UNBIND; 1819 if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1820 (kg->kg_completed == NULL) && 1821 (ku->ku_flags & KUF_DOUPCALL) == 0 && 1822 (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { 1823 mtx_unlock_spin(&sched_lock); 1824 thread_update_usr_ticks(td, 0); 1825 nanotime(&ts); 1826 error = copyout(&ts, 1827 (caddr_t)&ku->ku_mailbox->km_timeofday, 1828 sizeof(ts)); 1829 td->td_mailbox = 0; 1830 ku->ku_mflags = 0; 1831 if (error) 1832 goto out; 1833 return (0); 1834 } 1835 mtx_unlock_spin(&sched_lock); 1836 thread_export_context(td, 0); 1837 /* 1838 * There is something to report, and we own an upcall 1839 * strucuture, we can go to userland. 1840 * Turn ourself into an upcall thread. 1841 */ 1842 td->td_pflags |= TDP_UPCALLING; 1843 } else if (td->td_mailbox && (ku == NULL)) { 1844 thread_export_context(td, 1); 1845 PROC_LOCK(p); 1846 /* 1847 * There are upcall threads waiting for 1848 * work to do, wake one of them up. 1849 * XXXKSE Maybe wake all of them up. 1850 */ 1851 if (kg->kg_upsleeps) 1852 wakeup_one(&kg->kg_completed); 1853 mtx_lock_spin(&sched_lock); 1854 thread_stopped(p); 1855 thread_exit(); 1856 /* NOTREACHED */ 1857 } 1858 1859 KASSERT(ku != NULL, ("upcall is NULL\n")); 1860 KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1861 1862 if (p->p_numthreads > max_threads_per_proc) { 1863 max_threads_hits++; 1864 PROC_LOCK(p); 1865 mtx_lock_spin(&sched_lock); 1866 p->p_maxthrwaits++; 1867 while (p->p_numthreads > max_threads_per_proc) { 1868 upcalls = 0; 1869 FOREACH_KSEGRP_IN_PROC(p, kg2) { 1870 if (kg2->kg_numupcalls == 0) 1871 upcalls++; 1872 else 1873 upcalls += kg2->kg_numupcalls; 1874 } 1875 if (upcalls >= max_threads_per_proc) 1876 break; 1877 mtx_unlock_spin(&sched_lock); 1878 if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1879 "maxthreads", 0)) { 1880 mtx_lock_spin(&sched_lock); 1881 break; 1882 } else { 1883 mtx_lock_spin(&sched_lock); 1884 } 1885 } 1886 p->p_maxthrwaits--; 1887 mtx_unlock_spin(&sched_lock); 1888 PROC_UNLOCK(p); 1889 } 1890 1891 if (td->td_pflags & TDP_UPCALLING) { 1892 uts_crit = 0; 1893 kg->kg_nextupcall = ticks+kg->kg_upquantum; 1894 /* 1895 * There is no more work to do and we are going to ride 1896 * this thread up to userland as an upcall. 1897 * Do the last parts of the setup needed for the upcall. 1898 */ 1899 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1900 td, td->td_proc->p_pid, td->td_proc->p_comm); 1901 1902 td->td_pflags &= ~TDP_UPCALLING; 1903 if (ku->ku_flags & KUF_DOUPCALL) { 1904 mtx_lock_spin(&sched_lock); 1905 ku->ku_flags &= ~KUF_DOUPCALL; 1906 mtx_unlock_spin(&sched_lock); 1907 } 1908 /* 1909 * Set user context to the UTS 1910 */ 1911 if (!(ku->ku_mflags & KMF_NOUPCALL)) { 1912 cpu_set_upcall_kse(td, ku); 1913 error = suword(&ku->ku_mailbox->km_curthread, 0); 1914 if (error) 1915 goto out; 1916 } 1917 1918 /* 1919 * Unhook the list of completed threads. 1920 * anything that completes after this gets to 1921 * come in next time. 1922 * Put the list of completed thread mailboxes on 1923 * this KSE's mailbox. 1924 */ 1925 if (!(ku->ku_mflags & KMF_NOCOMPLETED) && 1926 (error = thread_link_mboxes(kg, ku)) != 0) 1927 goto out; 1928 } 1929 if (!uts_crit) { 1930 nanotime(&ts); 1931 error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); 1932 } 1933 1934 out: 1935 if (error) { 1936 /* 1937 * Things are going to be so screwed we should just kill 1938 * the process. 1939 * how do we do that? 1940 */ 1941 PROC_LOCK(td->td_proc); 1942 psignal(td->td_proc, SIGSEGV); 1943 PROC_UNLOCK(td->td_proc); 1944 } else { 1945 /* 1946 * Optimisation: 1947 * Ensure that we have a spare thread available, 1948 * for when we re-enter the kernel. 1949 */ 1950 if (td->td_standin == NULL) 1951 thread_alloc_spare(td, NULL); 1952 } 1953 1954 ku->ku_mflags = 0; 1955 /* 1956 * Clear thread mailbox first, then clear system tick count. 1957 * The order is important because thread_statclock() use 1958 * mailbox pointer to see if it is an userland thread or 1959 * an UTS kernel thread. 1960 */ 1961 td->td_mailbox = NULL; 1962 td->td_usticks = 0; 1963 return (error); /* go sync */ 1964 } 1965 1966 /* 1967 * Enforce single-threading. 1968 * 1969 * Returns 1 if the caller must abort (another thread is waiting to 1970 * exit the process or similar). Process is locked! 1971 * Returns 0 when you are successfully the only thread running. 1972 * A process has successfully single threaded in the suspend mode when 1973 * There are no threads in user mode. Threads in the kernel must be 1974 * allowed to continue until they get to the user boundary. They may even 1975 * copy out their return values and data before suspending. They may however be 1976 * accellerated in reaching the user boundary as we will wake up 1977 * any sleeping threads that are interruptable. (PCATCH). 1978 */ 1979 int 1980 thread_single(int force_exit) 1981 { 1982 struct thread *td; 1983 struct thread *td2; 1984 struct proc *p; 1985 1986 td = curthread; 1987 p = td->td_proc; 1988 mtx_assert(&Giant, MA_NOTOWNED); 1989 PROC_LOCK_ASSERT(p, MA_OWNED); 1990 KASSERT((td != NULL), ("curthread is NULL")); 1991 1992 if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1) 1993 return (0); 1994 1995 /* Is someone already single threading? */ 1996 if (p->p_singlethread) 1997 return (1); 1998 1999 if (force_exit == SINGLE_EXIT) { 2000 p->p_flag |= P_SINGLE_EXIT; 2001 } else 2002 p->p_flag &= ~P_SINGLE_EXIT; 2003 p->p_flag |= P_STOPPED_SINGLE; 2004 mtx_lock_spin(&sched_lock); 2005 p->p_singlethread = td; 2006 while ((p->p_numthreads - p->p_suspcount) != 1) { 2007 FOREACH_THREAD_IN_PROC(p, td2) { 2008 if (td2 == td) 2009 continue; 2010 td2->td_flags |= TDF_ASTPENDING; 2011 if (TD_IS_INHIBITED(td2)) { 2012 if (force_exit == SINGLE_EXIT) { 2013 if (TD_IS_SUSPENDED(td2)) { 2014 thread_unsuspend_one(td2); 2015 } 2016 if (TD_ON_SLEEPQ(td2) && 2017 (td2->td_flags & TDF_SINTR)) { 2018 sleepq_abort(td2); 2019 } 2020 } else { 2021 if (TD_IS_SUSPENDED(td2)) 2022 continue; 2023 /* 2024 * maybe other inhibitted states too? 2025 * XXXKSE Is it totally safe to 2026 * suspend a non-interruptable thread? 2027 */ 2028 if (td2->td_inhibitors & 2029 (TDI_SLEEPING | TDI_SWAPPED)) 2030 thread_suspend_one(td2); 2031 } 2032 } 2033 } 2034 /* 2035 * Maybe we suspended some threads.. was it enough? 2036 */ 2037 if ((p->p_numthreads - p->p_suspcount) == 1) 2038 break; 2039 2040 /* 2041 * Wake us up when everyone else has suspended. 2042 * In the mean time we suspend as well. 2043 */ 2044 thread_suspend_one(td); 2045 PROC_UNLOCK(p); 2046 mi_switch(SW_VOL); 2047 mtx_unlock_spin(&sched_lock); 2048 PROC_LOCK(p); 2049 mtx_lock_spin(&sched_lock); 2050 } 2051 if (force_exit == SINGLE_EXIT) { 2052 if (td->td_upcall) 2053 upcall_remove(td); 2054 kse_purge(p, td); 2055 } 2056 mtx_unlock_spin(&sched_lock); 2057 return (0); 2058 } 2059 2060 /* 2061 * Called in from locations that can safely check to see 2062 * whether we have to suspend or at least throttle for a 2063 * single-thread event (e.g. fork). 2064 * 2065 * Such locations include userret(). 2066 * If the "return_instead" argument is non zero, the thread must be able to 2067 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 2068 * 2069 * The 'return_instead' argument tells the function if it may do a 2070 * thread_exit() or suspend, or whether the caller must abort and back 2071 * out instead. 2072 * 2073 * If the thread that set the single_threading request has set the 2074 * P_SINGLE_EXIT bit in the process flags then this call will never return 2075 * if 'return_instead' is false, but will exit. 2076 * 2077 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 2078 *---------------+--------------------+--------------------- 2079 * 0 | returns 0 | returns 0 or 1 2080 * | when ST ends | immediatly 2081 *---------------+--------------------+--------------------- 2082 * 1 | thread exits | returns 1 2083 * | | immediatly 2084 * 0 = thread_exit() or suspension ok, 2085 * other = return error instead of stopping the thread. 2086 * 2087 * While a full suspension is under effect, even a single threading 2088 * thread would be suspended if it made this call (but it shouldn't). 2089 * This call should only be made from places where 2090 * thread_exit() would be safe as that may be the outcome unless 2091 * return_instead is set. 2092 */ 2093 int 2094 thread_suspend_check(int return_instead) 2095 { 2096 struct thread *td; 2097 struct proc *p; 2098 2099 td = curthread; 2100 p = td->td_proc; 2101 mtx_assert(&Giant, MA_NOTOWNED); 2102 PROC_LOCK_ASSERT(p, MA_OWNED); 2103 while (P_SHOULDSTOP(p)) { 2104 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 2105 KASSERT(p->p_singlethread != NULL, 2106 ("singlethread not set")); 2107 /* 2108 * The only suspension in action is a 2109 * single-threading. Single threader need not stop. 2110 * XXX Should be safe to access unlocked 2111 * as it can only be set to be true by us. 2112 */ 2113 if (p->p_singlethread == td) 2114 return (0); /* Exempt from stopping. */ 2115 } 2116 if (return_instead) 2117 return (1); 2118 2119 mtx_lock_spin(&sched_lock); 2120 thread_stopped(p); 2121 /* 2122 * If the process is waiting for us to exit, 2123 * this thread should just suicide. 2124 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 2125 */ 2126 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 2127 if (p->p_flag & P_SA) 2128 thread_exit(); 2129 else 2130 thr_exit1(); 2131 } 2132 2133 /* 2134 * When a thread suspends, it just 2135 * moves to the processes's suspend queue 2136 * and stays there. 2137 */ 2138 thread_suspend_one(td); 2139 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 2140 if (p->p_numthreads == p->p_suspcount) { 2141 thread_unsuspend_one(p->p_singlethread); 2142 } 2143 } 2144 PROC_UNLOCK(p); 2145 mi_switch(SW_INVOL); 2146 mtx_unlock_spin(&sched_lock); 2147 PROC_LOCK(p); 2148 } 2149 return (0); 2150 } 2151 2152 void 2153 thread_suspend_one(struct thread *td) 2154 { 2155 struct proc *p = td->td_proc; 2156 2157 mtx_assert(&sched_lock, MA_OWNED); 2158 PROC_LOCK_ASSERT(p, MA_OWNED); 2159 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 2160 p->p_suspcount++; 2161 TD_SET_SUSPENDED(td); 2162 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 2163 /* 2164 * Hack: If we are suspending but are on the sleep queue 2165 * then we are in msleep or the cv equivalent. We 2166 * want to look like we have two Inhibitors. 2167 * May already be set.. doesn't matter. 2168 */ 2169 if (TD_ON_SLEEPQ(td)) 2170 TD_SET_SLEEPING(td); 2171 } 2172 2173 void 2174 thread_unsuspend_one(struct thread *td) 2175 { 2176 struct proc *p = td->td_proc; 2177 2178 mtx_assert(&sched_lock, MA_OWNED); 2179 PROC_LOCK_ASSERT(p, MA_OWNED); 2180 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 2181 TD_CLR_SUSPENDED(td); 2182 p->p_suspcount--; 2183 setrunnable(td); 2184 } 2185 2186 /* 2187 * Allow all threads blocked by single threading to continue running. 2188 */ 2189 void 2190 thread_unsuspend(struct proc *p) 2191 { 2192 struct thread *td; 2193 2194 mtx_assert(&sched_lock, MA_OWNED); 2195 PROC_LOCK_ASSERT(p, MA_OWNED); 2196 if (!P_SHOULDSTOP(p)) { 2197 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2198 thread_unsuspend_one(td); 2199 } 2200 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 2201 (p->p_numthreads == p->p_suspcount)) { 2202 /* 2203 * Stopping everything also did the job for the single 2204 * threading request. Now we've downgraded to single-threaded, 2205 * let it continue. 2206 */ 2207 thread_unsuspend_one(p->p_singlethread); 2208 } 2209 } 2210 2211 void 2212 thread_single_end(void) 2213 { 2214 struct thread *td; 2215 struct proc *p; 2216 2217 td = curthread; 2218 p = td->td_proc; 2219 PROC_LOCK_ASSERT(p, MA_OWNED); 2220 p->p_flag &= ~(P_STOPPED_SINGLE | P_SINGLE_EXIT); 2221 mtx_lock_spin(&sched_lock); 2222 p->p_singlethread = NULL; 2223 /* 2224 * If there are other threads they mey now run, 2225 * unless of course there is a blanket 'stop order' 2226 * on the process. The single threader must be allowed 2227 * to continue however as this is a bad place to stop. 2228 */ 2229 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2230 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2231 thread_unsuspend_one(td); 2232 } 2233 } 2234 mtx_unlock_spin(&sched_lock); 2235 } 2236 2237 int 2238 thread_upcall_check(struct thread *td) 2239 { 2240 PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); 2241 if (td->td_kflags & TDK_WAKEUP) 2242 return (1); 2243 else 2244 return (0); 2245 } 2246