1 /* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/mutex.h> 38 #include <sys/proc.h> 39 #include <sys/smp.h> 40 #include <sys/sysctl.h> 41 #include <sys/sysproto.h> 42 #include <sys/filedesc.h> 43 #include <sys/sched.h> 44 #include <sys/signalvar.h> 45 #include <sys/sx.h> 46 #include <sys/tty.h> 47 #include <sys/turnstile.h> 48 #include <sys/user.h> 49 #include <sys/jail.h> 50 #include <sys/kse.h> 51 #include <sys/ktr.h> 52 #include <sys/ucontext.h> 53 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_object.h> 57 #include <vm/pmap.h> 58 #include <vm/uma.h> 59 #include <vm/vm_map.h> 60 61 #include <machine/frame.h> 62 63 /* 64 * KSEGRP related storage. 65 */ 66 static uma_zone_t ksegrp_zone; 67 static uma_zone_t kse_zone; 68 static uma_zone_t thread_zone; 69 static uma_zone_t upcall_zone; 70 71 /* DEBUG ONLY */ 72 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 73 static int thread_debug = 0; 74 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 75 &thread_debug, 0, "thread debug"); 76 77 static int max_threads_per_proc = 150; 78 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 79 &max_threads_per_proc, 0, "Limit on threads per proc"); 80 81 static int max_groups_per_proc = 50; 82 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 83 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 84 85 static int max_threads_hits; 86 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 87 &max_threads_hits, 0, ""); 88 89 static int virtual_cpu; 90 91 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 92 93 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 94 TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 95 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 96 TAILQ_HEAD(, kse_upcall) zombie_upcalls = 97 TAILQ_HEAD_INITIALIZER(zombie_upcalls); 98 struct mtx kse_zombie_lock; 99 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 100 101 static void kse_purge(struct proc *p, struct thread *td); 102 static void kse_purge_group(struct thread *td); 103 static int thread_update_usr_ticks(struct thread *td, int user); 104 static void thread_alloc_spare(struct thread *td, struct thread *spare); 105 106 static int 107 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 108 { 109 int error, new_val; 110 int def_val; 111 112 #ifdef SMP 113 def_val = mp_ncpus; 114 #else 115 def_val = 1; 116 #endif 117 if (virtual_cpu == 0) 118 new_val = def_val; 119 else 120 new_val = virtual_cpu; 121 error = sysctl_handle_int(oidp, &new_val, 0, req); 122 if (error != 0 || req->newptr == NULL) 123 return (error); 124 if (new_val < 0) 125 return (EINVAL); 126 virtual_cpu = new_val; 127 return (0); 128 } 129 130 /* DEBUG ONLY */ 131 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 132 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 133 "debug virtual cpus"); 134 135 /* 136 * Prepare a thread for use. 137 */ 138 static void 139 thread_ctor(void *mem, int size, void *arg) 140 { 141 struct thread *td; 142 143 td = (struct thread *)mem; 144 td->td_state = TDS_INACTIVE; 145 td->td_oncpu = NOCPU; 146 td->td_critnest = 1; 147 } 148 149 /* 150 * Reclaim a thread after use. 151 */ 152 static void 153 thread_dtor(void *mem, int size, void *arg) 154 { 155 struct thread *td; 156 157 td = (struct thread *)mem; 158 159 #ifdef INVARIANTS 160 /* Verify that this thread is in a safe state to free. */ 161 switch (td->td_state) { 162 case TDS_INHIBITED: 163 case TDS_RUNNING: 164 case TDS_CAN_RUN: 165 case TDS_RUNQ: 166 /* 167 * We must never unlink a thread that is in one of 168 * these states, because it is currently active. 169 */ 170 panic("bad state for thread unlinking"); 171 /* NOTREACHED */ 172 case TDS_INACTIVE: 173 break; 174 default: 175 panic("bad thread state"); 176 /* NOTREACHED */ 177 } 178 #endif 179 } 180 181 /* 182 * Initialize type-stable parts of a thread (when newly created). 183 */ 184 static void 185 thread_init(void *mem, int size) 186 { 187 struct thread *td; 188 189 td = (struct thread *)mem; 190 mtx_lock(&Giant); 191 vm_thread_new(td, 0); 192 mtx_unlock(&Giant); 193 cpu_thread_setup(td); 194 td->td_turnstile = turnstile_alloc(); 195 td->td_sched = (struct td_sched *)&td[1]; 196 } 197 198 /* 199 * Tear down type-stable parts of a thread (just before being discarded). 200 */ 201 static void 202 thread_fini(void *mem, int size) 203 { 204 struct thread *td; 205 206 td = (struct thread *)mem; 207 turnstile_free(td->td_turnstile); 208 vm_thread_dispose(td); 209 } 210 211 /* 212 * Initialize type-stable parts of a kse (when newly created). 213 */ 214 static void 215 kse_init(void *mem, int size) 216 { 217 struct kse *ke; 218 219 ke = (struct kse *)mem; 220 ke->ke_sched = (struct ke_sched *)&ke[1]; 221 } 222 223 /* 224 * Initialize type-stable parts of a ksegrp (when newly created). 225 */ 226 static void 227 ksegrp_init(void *mem, int size) 228 { 229 struct ksegrp *kg; 230 231 kg = (struct ksegrp *)mem; 232 kg->kg_sched = (struct kg_sched *)&kg[1]; 233 } 234 235 /* 236 * KSE is linked into kse group. 237 */ 238 void 239 kse_link(struct kse *ke, struct ksegrp *kg) 240 { 241 struct proc *p = kg->kg_proc; 242 243 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 244 kg->kg_kses++; 245 ke->ke_state = KES_UNQUEUED; 246 ke->ke_proc = p; 247 ke->ke_ksegrp = kg; 248 ke->ke_thread = NULL; 249 ke->ke_oncpu = NOCPU; 250 ke->ke_flags = 0; 251 } 252 253 void 254 kse_unlink(struct kse *ke) 255 { 256 struct ksegrp *kg; 257 258 mtx_assert(&sched_lock, MA_OWNED); 259 kg = ke->ke_ksegrp; 260 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 261 if (ke->ke_state == KES_IDLE) { 262 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 263 kg->kg_idle_kses--; 264 } 265 --kg->kg_kses; 266 /* 267 * Aggregate stats from the KSE 268 */ 269 kse_stash(ke); 270 } 271 272 void 273 ksegrp_link(struct ksegrp *kg, struct proc *p) 274 { 275 276 TAILQ_INIT(&kg->kg_threads); 277 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 278 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 279 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 280 TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 281 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 282 kg->kg_proc = p; 283 /* 284 * the following counters are in the -zero- section 285 * and may not need clearing 286 */ 287 kg->kg_numthreads = 0; 288 kg->kg_runnable = 0; 289 kg->kg_kses = 0; 290 kg->kg_runq_kses = 0; /* XXXKSE change name */ 291 kg->kg_idle_kses = 0; 292 kg->kg_numupcalls = 0; 293 /* link it in now that it's consistent */ 294 p->p_numksegrps++; 295 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 296 } 297 298 void 299 ksegrp_unlink(struct ksegrp *kg) 300 { 301 struct proc *p; 302 303 mtx_assert(&sched_lock, MA_OWNED); 304 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 305 KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 306 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 307 308 p = kg->kg_proc; 309 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 310 p->p_numksegrps--; 311 /* 312 * Aggregate stats from the KSE 313 */ 314 ksegrp_stash(kg); 315 } 316 317 struct kse_upcall * 318 upcall_alloc(void) 319 { 320 struct kse_upcall *ku; 321 322 ku = uma_zalloc(upcall_zone, M_WAITOK); 323 bzero(ku, sizeof(*ku)); 324 return (ku); 325 } 326 327 void 328 upcall_free(struct kse_upcall *ku) 329 { 330 331 uma_zfree(upcall_zone, ku); 332 } 333 334 void 335 upcall_link(struct kse_upcall *ku, struct ksegrp *kg) 336 { 337 338 mtx_assert(&sched_lock, MA_OWNED); 339 TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 340 ku->ku_ksegrp = kg; 341 kg->kg_numupcalls++; 342 } 343 344 void 345 upcall_unlink(struct kse_upcall *ku) 346 { 347 struct ksegrp *kg = ku->ku_ksegrp; 348 349 mtx_assert(&sched_lock, MA_OWNED); 350 KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 351 TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 352 kg->kg_numupcalls--; 353 upcall_stash(ku); 354 } 355 356 void 357 upcall_remove(struct thread *td) 358 { 359 360 if (td->td_upcall) { 361 td->td_upcall->ku_owner = NULL; 362 upcall_unlink(td->td_upcall); 363 td->td_upcall = 0; 364 } 365 } 366 367 /* 368 * For a newly created process, 369 * link up all the structures and its initial threads etc. 370 */ 371 void 372 proc_linkup(struct proc *p, struct ksegrp *kg, 373 struct kse *ke, struct thread *td) 374 { 375 376 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 377 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 378 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 379 p->p_numksegrps = 0; 380 p->p_numthreads = 0; 381 382 ksegrp_link(kg, p); 383 kse_link(ke, kg); 384 thread_link(td, kg); 385 } 386 387 /* 388 struct kse_thr_interrupt_args { 389 struct kse_thr_mailbox * tmbx; 390 int cmd; 391 long data; 392 }; 393 */ 394 int 395 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 396 { 397 struct proc *p; 398 struct thread *td2; 399 400 p = td->td_proc; 401 402 if (!(p->p_flag & P_SA)) 403 return (EINVAL); 404 405 switch (uap->cmd) { 406 case KSE_INTR_SENDSIG: 407 if (uap->data < 0 || uap->data > _SIG_MAXSIG) 408 return (EINVAL); 409 case KSE_INTR_INTERRUPT: 410 case KSE_INTR_RESTART: 411 PROC_LOCK(p); 412 mtx_lock_spin(&sched_lock); 413 FOREACH_THREAD_IN_PROC(p, td2) { 414 if (td2->td_mailbox == uap->tmbx) 415 break; 416 } 417 if (td2 == NULL) { 418 mtx_unlock_spin(&sched_lock); 419 PROC_UNLOCK(p); 420 return (ESRCH); 421 } 422 if (uap->cmd == KSE_INTR_SENDSIG) { 423 if (uap->data > 0) { 424 td2->td_flags &= ~TDF_INTERRUPT; 425 mtx_unlock_spin(&sched_lock); 426 tdsignal(td2, (int)uap->data, SIGTARGET_TD); 427 } else { 428 mtx_unlock_spin(&sched_lock); 429 } 430 } else { 431 td2->td_flags |= TDF_INTERRUPT | TDF_ASTPENDING; 432 if (TD_CAN_UNBIND(td2)) 433 td2->td_upcall->ku_flags |= KUF_DOUPCALL; 434 if (uap->cmd == KSE_INTR_INTERRUPT) 435 td2->td_intrval = EINTR; 436 else 437 td2->td_intrval = ERESTART; 438 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 439 if (td2->td_flags & TDF_CVWAITQ) 440 cv_abort(td2); 441 else 442 abortsleep(td2); 443 } 444 mtx_unlock_spin(&sched_lock); 445 } 446 PROC_UNLOCK(p); 447 break; 448 case KSE_INTR_SIGEXIT: 449 if (uap->data < 1 || uap->data > _SIG_MAXSIG) 450 return (EINVAL); 451 PROC_LOCK(p); 452 sigexit(td, (int)uap->data); 453 break; 454 default: 455 return (EINVAL); 456 } 457 return (0); 458 } 459 460 /* 461 struct kse_exit_args { 462 register_t dummy; 463 }; 464 */ 465 int 466 kse_exit(struct thread *td, struct kse_exit_args *uap) 467 { 468 struct proc *p; 469 struct ksegrp *kg; 470 struct kse *ke; 471 struct kse_upcall *ku, *ku2; 472 int error, count; 473 474 p = td->td_proc; 475 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 476 return (EINVAL); 477 kg = td->td_ksegrp; 478 count = 0; 479 PROC_LOCK(p); 480 mtx_lock_spin(&sched_lock); 481 FOREACH_UPCALL_IN_GROUP(kg, ku2) { 482 if (ku2->ku_flags & KUF_EXITING) 483 count++; 484 } 485 if ((kg->kg_numupcalls - count) == 1 && 486 (kg->kg_numthreads > 1)) { 487 mtx_unlock_spin(&sched_lock); 488 PROC_UNLOCK(p); 489 return (EDEADLK); 490 } 491 ku->ku_flags |= KUF_EXITING; 492 mtx_unlock_spin(&sched_lock); 493 PROC_UNLOCK(p); 494 error = suword(&ku->ku_mailbox->km_flags, ku->ku_mflags|KMF_DONE); 495 PROC_LOCK(p); 496 if (error) 497 psignal(p, SIGSEGV); 498 mtx_lock_spin(&sched_lock); 499 upcall_remove(td); 500 ke = td->td_kse; 501 if (p->p_numthreads == 1) { 502 kse_purge(p, td); 503 p->p_flag &= ~P_SA; 504 mtx_unlock_spin(&sched_lock); 505 PROC_UNLOCK(p); 506 } else { 507 if (kg->kg_numthreads == 1) { /* Shutdown a group */ 508 kse_purge_group(td); 509 ke->ke_flags |= KEF_EXIT; 510 } 511 thread_stopped(p); 512 thread_exit(); 513 /* NOTREACHED */ 514 } 515 return (0); 516 } 517 518 /* 519 * Either becomes an upcall or waits for an awakening event and 520 * then becomes an upcall. Only error cases return. 521 */ 522 /* 523 struct kse_release_args { 524 struct timespec *timeout; 525 }; 526 */ 527 int 528 kse_release(struct thread *td, struct kse_release_args *uap) 529 { 530 struct proc *p; 531 struct ksegrp *kg; 532 struct kse_upcall *ku; 533 struct timespec timeout; 534 struct timeval tv; 535 sigset_t sigset; 536 int error; 537 538 p = td->td_proc; 539 kg = td->td_ksegrp; 540 if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) 541 return (EINVAL); 542 if (uap->timeout != NULL) { 543 if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 544 return (error); 545 TIMESPEC_TO_TIMEVAL(&tv, &timeout); 546 } 547 if (td->td_flags & TDF_SA) 548 td->td_pflags |= TDP_UPCALLING; 549 else { 550 ku->ku_mflags = fuword(&ku->ku_mailbox->km_flags); 551 if (ku->ku_mflags == -1) { 552 PROC_LOCK(p); 553 sigexit(td, SIGSEGV); 554 } 555 } 556 PROC_LOCK(p); 557 if (ku->ku_mflags & KMF_WAITSIGEVENT) { 558 /* UTS wants to wait for signal event */ 559 if (!(p->p_flag & P_SIGEVENT) && !(ku->ku_flags & KUF_DOUPCALL)) 560 error = msleep(&p->p_siglist, &p->p_mtx, PPAUSE|PCATCH, 561 "ksesigwait", (uap->timeout ? tvtohz(&tv) : 0)); 562 p->p_flag &= ~P_SIGEVENT; 563 sigset = p->p_siglist; 564 PROC_UNLOCK(p); 565 error = copyout(&sigset, &ku->ku_mailbox->km_sigscaught, 566 sizeof(sigset)); 567 } else { 568 if (! kg->kg_completed && !(ku->ku_flags & KUF_DOUPCALL)) { 569 kg->kg_upsleeps++; 570 error = msleep(&kg->kg_completed, &p->p_mtx, 571 PPAUSE|PCATCH, "kserel", 572 (uap->timeout ? tvtohz(&tv) : 0)); 573 kg->kg_upsleeps--; 574 } 575 PROC_UNLOCK(p); 576 } 577 if (ku->ku_flags & KUF_DOUPCALL) { 578 mtx_lock_spin(&sched_lock); 579 ku->ku_flags &= ~KUF_DOUPCALL; 580 mtx_unlock_spin(&sched_lock); 581 } 582 return (0); 583 } 584 585 /* struct kse_wakeup_args { 586 struct kse_mailbox *mbx; 587 }; */ 588 int 589 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 590 { 591 struct proc *p; 592 struct ksegrp *kg; 593 struct kse_upcall *ku; 594 struct thread *td2; 595 596 p = td->td_proc; 597 td2 = NULL; 598 ku = NULL; 599 /* KSE-enabled processes only, please. */ 600 if (!(p->p_flag & P_SA)) 601 return (EINVAL); 602 PROC_LOCK(p); 603 mtx_lock_spin(&sched_lock); 604 if (uap->mbx) { 605 FOREACH_KSEGRP_IN_PROC(p, kg) { 606 FOREACH_UPCALL_IN_GROUP(kg, ku) { 607 if (ku->ku_mailbox == uap->mbx) 608 break; 609 } 610 if (ku) 611 break; 612 } 613 } else { 614 kg = td->td_ksegrp; 615 if (kg->kg_upsleeps) { 616 wakeup_one(&kg->kg_completed); 617 mtx_unlock_spin(&sched_lock); 618 PROC_UNLOCK(p); 619 return (0); 620 } 621 ku = TAILQ_FIRST(&kg->kg_upcalls); 622 } 623 if (ku) { 624 if ((td2 = ku->ku_owner) == NULL) { 625 panic("%s: no owner", __func__); 626 } else if (TD_ON_SLEEPQ(td2) && 627 ((td2->td_wchan == &kg->kg_completed) || 628 (td2->td_wchan == &p->p_siglist && 629 (ku->ku_mflags & KMF_WAITSIGEVENT)))) { 630 abortsleep(td2); 631 } else { 632 ku->ku_flags |= KUF_DOUPCALL; 633 } 634 mtx_unlock_spin(&sched_lock); 635 PROC_UNLOCK(p); 636 return (0); 637 } 638 mtx_unlock_spin(&sched_lock); 639 PROC_UNLOCK(p); 640 return (ESRCH); 641 } 642 643 /* 644 * No new KSEG: first call: use current KSE, don't schedule an upcall 645 * All other situations, do allocate max new KSEs and schedule an upcall. 646 */ 647 /* struct kse_create_args { 648 struct kse_mailbox *mbx; 649 int newgroup; 650 }; */ 651 int 652 kse_create(struct thread *td, struct kse_create_args *uap) 653 { 654 struct kse *newke; 655 struct ksegrp *newkg; 656 struct ksegrp *kg; 657 struct proc *p; 658 struct kse_mailbox mbx; 659 struct kse_upcall *newku; 660 int err, ncpus, sa = 0, first = 0; 661 struct thread *newtd; 662 663 p = td->td_proc; 664 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 665 return (err); 666 667 /* Too bad, why hasn't kernel always a cpu counter !? */ 668 #ifdef SMP 669 ncpus = mp_ncpus; 670 #else 671 ncpus = 1; 672 #endif 673 if (virtual_cpu != 0) 674 ncpus = virtual_cpu; 675 if (!(mbx.km_flags & KMF_BOUND)) 676 sa = TDF_SA; 677 else 678 ncpus = 1; 679 PROC_LOCK(p); 680 if (!(p->p_flag & P_SA)) { 681 first = 1; 682 p->p_flag |= P_SA; 683 } 684 PROC_UNLOCK(p); 685 if (!sa && !uap->newgroup && !first) 686 return (EINVAL); 687 kg = td->td_ksegrp; 688 if (uap->newgroup) { 689 /* Have race condition but it is cheap */ 690 if (p->p_numksegrps >= max_groups_per_proc) 691 return (EPROCLIM); 692 /* 693 * If we want a new KSEGRP it doesn't matter whether 694 * we have already fired up KSE mode before or not. 695 * We put the process in KSE mode and create a new KSEGRP. 696 */ 697 newkg = ksegrp_alloc(); 698 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 699 kg_startzero, kg_endzero)); 700 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 701 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 702 PROC_LOCK(p); 703 mtx_lock_spin(&sched_lock); 704 if (p->p_numksegrps >= max_groups_per_proc) { 705 mtx_unlock_spin(&sched_lock); 706 PROC_UNLOCK(p); 707 ksegrp_free(newkg); 708 return (EPROCLIM); 709 } 710 ksegrp_link(newkg, p); 711 sched_fork_ksegrp(kg, newkg); 712 mtx_unlock_spin(&sched_lock); 713 PROC_UNLOCK(p); 714 } else { 715 if (!first && ((td->td_flags & TDF_SA) ^ sa) != 0) 716 return (EINVAL); 717 newkg = kg; 718 } 719 720 /* 721 * Creating upcalls more than number of physical cpu does 722 * not help performance. 723 */ 724 if (newkg->kg_numupcalls >= ncpus) 725 return (EPROCLIM); 726 727 if (newkg->kg_numupcalls == 0) { 728 /* 729 * Initialize KSE group 730 * 731 * For multiplxed group, create KSEs as many as physical 732 * cpus. This increases concurrent even if userland 733 * is not MP safe and can only run on single CPU. 734 * In ideal world, every physical cpu should execute a thread. 735 * If there is enough KSEs, threads in kernel can be 736 * executed parallel on different cpus with full speed, 737 * Concurrent in kernel shouldn't be restricted by number of 738 * upcalls userland provides. Adding more upcall structures 739 * only increases concurrent in userland. 740 * 741 * For bound thread group, because there is only thread in the 742 * group, we only create one KSE for the group. Thread in this 743 * kind of group will never schedule an upcall when blocked, 744 * this intends to simulate pthread system scope thread. 745 */ 746 while (newkg->kg_kses < ncpus) { 747 newke = kse_alloc(); 748 bzero(&newke->ke_startzero, RANGEOF(struct kse, 749 ke_startzero, ke_endzero)); 750 #if 0 751 mtx_lock_spin(&sched_lock); 752 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 753 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 754 mtx_unlock_spin(&sched_lock); 755 #endif 756 mtx_lock_spin(&sched_lock); 757 kse_link(newke, newkg); 758 sched_fork_kse(td->td_kse, newke); 759 /* Add engine */ 760 kse_reassign(newke); 761 mtx_unlock_spin(&sched_lock); 762 } 763 } 764 newku = upcall_alloc(); 765 newku->ku_mailbox = uap->mbx; 766 newku->ku_func = mbx.km_func; 767 bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 768 769 /* For the first call this may not have been set */ 770 if (td->td_standin == NULL) 771 thread_alloc_spare(td, NULL); 772 773 PROC_LOCK(p); 774 if (newkg->kg_numupcalls >= ncpus) { 775 PROC_UNLOCK(p); 776 upcall_free(newku); 777 return (EPROCLIM); 778 } 779 if (first && sa) { 780 SIGSETOR(p->p_siglist, td->td_siglist); 781 SIGEMPTYSET(td->td_siglist); 782 SIGFILLSET(td->td_sigmask); 783 SIG_CANTMASK(td->td_sigmask); 784 } 785 mtx_lock_spin(&sched_lock); 786 PROC_UNLOCK(p); 787 upcall_link(newku, newkg); 788 if (mbx.km_quantum) 789 newkg->kg_upquantum = max(1, mbx.km_quantum/tick); 790 791 /* 792 * Each upcall structure has an owner thread, find which 793 * one owns it. 794 */ 795 if (uap->newgroup) { 796 /* 797 * Because new ksegrp hasn't thread, 798 * create an initial upcall thread to own it. 799 */ 800 newtd = thread_schedule_upcall(td, newku); 801 } else { 802 /* 803 * If current thread hasn't an upcall structure, 804 * just assign the upcall to it. 805 */ 806 if (td->td_upcall == NULL) { 807 newku->ku_owner = td; 808 td->td_upcall = newku; 809 newtd = td; 810 } else { 811 /* 812 * Create a new upcall thread to own it. 813 */ 814 newtd = thread_schedule_upcall(td, newku); 815 } 816 } 817 if (!sa) { 818 newtd->td_mailbox = mbx.km_curthread; 819 newtd->td_flags &= ~TDF_SA; 820 if (newtd != td) { 821 mtx_unlock_spin(&sched_lock); 822 cpu_set_upcall_kse(newtd, newku); 823 mtx_lock_spin(&sched_lock); 824 } 825 } else { 826 newtd->td_flags |= TDF_SA; 827 } 828 if (newtd != td) 829 setrunqueue(newtd); 830 mtx_unlock_spin(&sched_lock); 831 return (0); 832 } 833 834 /* 835 * Initialize global thread allocation resources. 836 */ 837 void 838 threadinit(void) 839 { 840 841 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 842 thread_ctor, thread_dtor, thread_init, thread_fini, 843 UMA_ALIGN_CACHE, 0); 844 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 845 NULL, NULL, ksegrp_init, NULL, 846 UMA_ALIGN_CACHE, 0); 847 kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 848 NULL, NULL, kse_init, NULL, 849 UMA_ALIGN_CACHE, 0); 850 upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 851 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 852 } 853 854 /* 855 * Stash an embarasingly extra thread into the zombie thread queue. 856 */ 857 void 858 thread_stash(struct thread *td) 859 { 860 mtx_lock_spin(&kse_zombie_lock); 861 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 862 mtx_unlock_spin(&kse_zombie_lock); 863 } 864 865 /* 866 * Stash an embarasingly extra kse into the zombie kse queue. 867 */ 868 void 869 kse_stash(struct kse *ke) 870 { 871 mtx_lock_spin(&kse_zombie_lock); 872 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 873 mtx_unlock_spin(&kse_zombie_lock); 874 } 875 876 /* 877 * Stash an embarasingly extra upcall into the zombie upcall queue. 878 */ 879 880 void 881 upcall_stash(struct kse_upcall *ku) 882 { 883 mtx_lock_spin(&kse_zombie_lock); 884 TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 885 mtx_unlock_spin(&kse_zombie_lock); 886 } 887 888 /* 889 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 890 */ 891 void 892 ksegrp_stash(struct ksegrp *kg) 893 { 894 mtx_lock_spin(&kse_zombie_lock); 895 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 896 mtx_unlock_spin(&kse_zombie_lock); 897 } 898 899 /* 900 * Reap zombie kse resource. 901 */ 902 void 903 thread_reap(void) 904 { 905 struct thread *td_first, *td_next; 906 struct kse *ke_first, *ke_next; 907 struct ksegrp *kg_first, * kg_next; 908 struct kse_upcall *ku_first, *ku_next; 909 910 /* 911 * Don't even bother to lock if none at this instant, 912 * we really don't care about the next instant.. 913 */ 914 if ((!TAILQ_EMPTY(&zombie_threads)) 915 || (!TAILQ_EMPTY(&zombie_kses)) 916 || (!TAILQ_EMPTY(&zombie_ksegrps)) 917 || (!TAILQ_EMPTY(&zombie_upcalls))) { 918 mtx_lock_spin(&kse_zombie_lock); 919 td_first = TAILQ_FIRST(&zombie_threads); 920 ke_first = TAILQ_FIRST(&zombie_kses); 921 kg_first = TAILQ_FIRST(&zombie_ksegrps); 922 ku_first = TAILQ_FIRST(&zombie_upcalls); 923 if (td_first) 924 TAILQ_INIT(&zombie_threads); 925 if (ke_first) 926 TAILQ_INIT(&zombie_kses); 927 if (kg_first) 928 TAILQ_INIT(&zombie_ksegrps); 929 if (ku_first) 930 TAILQ_INIT(&zombie_upcalls); 931 mtx_unlock_spin(&kse_zombie_lock); 932 while (td_first) { 933 td_next = TAILQ_NEXT(td_first, td_runq); 934 if (td_first->td_ucred) 935 crfree(td_first->td_ucred); 936 thread_free(td_first); 937 td_first = td_next; 938 } 939 while (ke_first) { 940 ke_next = TAILQ_NEXT(ke_first, ke_procq); 941 kse_free(ke_first); 942 ke_first = ke_next; 943 } 944 while (kg_first) { 945 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 946 ksegrp_free(kg_first); 947 kg_first = kg_next; 948 } 949 while (ku_first) { 950 ku_next = TAILQ_NEXT(ku_first, ku_link); 951 upcall_free(ku_first); 952 ku_first = ku_next; 953 } 954 } 955 } 956 957 /* 958 * Allocate a ksegrp. 959 */ 960 struct ksegrp * 961 ksegrp_alloc(void) 962 { 963 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 964 } 965 966 /* 967 * Allocate a kse. 968 */ 969 struct kse * 970 kse_alloc(void) 971 { 972 return (uma_zalloc(kse_zone, M_WAITOK)); 973 } 974 975 /* 976 * Allocate a thread. 977 */ 978 struct thread * 979 thread_alloc(void) 980 { 981 thread_reap(); /* check if any zombies to get */ 982 return (uma_zalloc(thread_zone, M_WAITOK)); 983 } 984 985 /* 986 * Deallocate a ksegrp. 987 */ 988 void 989 ksegrp_free(struct ksegrp *td) 990 { 991 uma_zfree(ksegrp_zone, td); 992 } 993 994 /* 995 * Deallocate a kse. 996 */ 997 void 998 kse_free(struct kse *td) 999 { 1000 uma_zfree(kse_zone, td); 1001 } 1002 1003 /* 1004 * Deallocate a thread. 1005 */ 1006 void 1007 thread_free(struct thread *td) 1008 { 1009 1010 cpu_thread_clean(td); 1011 uma_zfree(thread_zone, td); 1012 } 1013 1014 /* 1015 * Store the thread context in the UTS's mailbox. 1016 * then add the mailbox at the head of a list we are building in user space. 1017 * The list is anchored in the ksegrp structure. 1018 */ 1019 int 1020 thread_export_context(struct thread *td, int willexit) 1021 { 1022 struct proc *p; 1023 struct ksegrp *kg; 1024 uintptr_t mbx; 1025 void *addr; 1026 int error = 0, temp, sig; 1027 mcontext_t mc; 1028 1029 p = td->td_proc; 1030 kg = td->td_ksegrp; 1031 1032 /* Export the user/machine context. */ 1033 get_mcontext(td, &mc, 0); 1034 addr = (void *)(&td->td_mailbox->tm_context.uc_mcontext); 1035 error = copyout(&mc, addr, sizeof(mcontext_t)); 1036 if (error) 1037 goto bad; 1038 1039 /* Exports clock ticks in kernel mode */ 1040 addr = (caddr_t)(&td->td_mailbox->tm_sticks); 1041 temp = fuword32(addr) + td->td_usticks; 1042 if (suword32(addr, temp)) { 1043 error = EFAULT; 1044 goto bad; 1045 } 1046 1047 /* 1048 * Post sync signal, or process SIGKILL and SIGSTOP. 1049 * For sync signal, it is only possible when the signal is not 1050 * caught by userland or process is being debugged. 1051 */ 1052 PROC_LOCK(p); 1053 if (td->td_flags & TDF_NEEDSIGCHK) { 1054 mtx_lock_spin(&sched_lock); 1055 td->td_flags &= ~TDF_NEEDSIGCHK; 1056 mtx_unlock_spin(&sched_lock); 1057 mtx_lock(&p->p_sigacts->ps_mtx); 1058 while ((sig = cursig(td)) != 0) 1059 postsig(sig); 1060 mtx_unlock(&p->p_sigacts->ps_mtx); 1061 } 1062 if (willexit) 1063 SIGFILLSET(td->td_sigmask); 1064 PROC_UNLOCK(p); 1065 1066 /* Get address in latest mbox of list pointer */ 1067 addr = (void *)(&td->td_mailbox->tm_next); 1068 /* 1069 * Put the saved address of the previous first 1070 * entry into this one 1071 */ 1072 for (;;) { 1073 mbx = (uintptr_t)kg->kg_completed; 1074 if (suword(addr, mbx)) { 1075 error = EFAULT; 1076 goto bad; 1077 } 1078 PROC_LOCK(p); 1079 if (mbx == (uintptr_t)kg->kg_completed) { 1080 kg->kg_completed = td->td_mailbox; 1081 /* 1082 * The thread context may be taken away by 1083 * other upcall threads when we unlock 1084 * process lock. it's no longer valid to 1085 * use it again in any other places. 1086 */ 1087 td->td_mailbox = NULL; 1088 PROC_UNLOCK(p); 1089 break; 1090 } 1091 PROC_UNLOCK(p); 1092 } 1093 td->td_usticks = 0; 1094 return (0); 1095 1096 bad: 1097 PROC_LOCK(p); 1098 sigexit(td, SIGILL); 1099 return (error); 1100 } 1101 1102 /* 1103 * Take the list of completed mailboxes for this KSEGRP and put them on this 1104 * upcall's mailbox as it's the next one going up. 1105 */ 1106 static int 1107 thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1108 { 1109 struct proc *p = kg->kg_proc; 1110 void *addr; 1111 uintptr_t mbx; 1112 1113 addr = (void *)(&ku->ku_mailbox->km_completed); 1114 for (;;) { 1115 mbx = (uintptr_t)kg->kg_completed; 1116 if (suword(addr, mbx)) { 1117 PROC_LOCK(p); 1118 psignal(p, SIGSEGV); 1119 PROC_UNLOCK(p); 1120 return (EFAULT); 1121 } 1122 PROC_LOCK(p); 1123 if (mbx == (uintptr_t)kg->kg_completed) { 1124 kg->kg_completed = NULL; 1125 PROC_UNLOCK(p); 1126 break; 1127 } 1128 PROC_UNLOCK(p); 1129 } 1130 return (0); 1131 } 1132 1133 /* 1134 * This function should be called at statclock interrupt time 1135 */ 1136 int 1137 thread_statclock(int user) 1138 { 1139 struct thread *td = curthread; 1140 struct ksegrp *kg = td->td_ksegrp; 1141 1142 if (kg->kg_numupcalls == 0 || !(td->td_flags & TDF_SA)) 1143 return (0); 1144 if (user) { 1145 /* Current always do via ast() */ 1146 mtx_lock_spin(&sched_lock); 1147 td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1148 mtx_unlock_spin(&sched_lock); 1149 td->td_uuticks++; 1150 } else { 1151 if (td->td_mailbox != NULL) 1152 td->td_usticks++; 1153 else { 1154 /* XXXKSE 1155 * We will call thread_user_enter() for every 1156 * kernel entry in future, so if the thread mailbox 1157 * is NULL, it must be a UTS kernel, don't account 1158 * clock ticks for it. 1159 */ 1160 } 1161 } 1162 return (0); 1163 } 1164 1165 /* 1166 * Export state clock ticks for userland 1167 */ 1168 static int 1169 thread_update_usr_ticks(struct thread *td, int user) 1170 { 1171 struct proc *p = td->td_proc; 1172 struct kse_thr_mailbox *tmbx; 1173 struct kse_upcall *ku; 1174 struct ksegrp *kg; 1175 caddr_t addr; 1176 u_int uticks; 1177 1178 if ((ku = td->td_upcall) == NULL) 1179 return (-1); 1180 1181 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1182 if ((tmbx == NULL) || (tmbx == (void *)-1)) 1183 return (-1); 1184 if (user) { 1185 uticks = td->td_uuticks; 1186 td->td_uuticks = 0; 1187 addr = (caddr_t)&tmbx->tm_uticks; 1188 } else { 1189 uticks = td->td_usticks; 1190 td->td_usticks = 0; 1191 addr = (caddr_t)&tmbx->tm_sticks; 1192 } 1193 if (uticks) { 1194 if (suword32(addr, uticks+fuword32(addr))) { 1195 PROC_LOCK(p); 1196 psignal(p, SIGSEGV); 1197 PROC_UNLOCK(p); 1198 return (-2); 1199 } 1200 } 1201 kg = td->td_ksegrp; 1202 if (kg->kg_upquantum && ticks >= kg->kg_nextupcall) { 1203 mtx_lock_spin(&sched_lock); 1204 td->td_upcall->ku_flags |= KUF_DOUPCALL; 1205 mtx_unlock_spin(&sched_lock); 1206 } 1207 return (0); 1208 } 1209 1210 /* 1211 * Discard the current thread and exit from its context. 1212 * 1213 * Because we can't free a thread while we're operating under its context, 1214 * push the current thread into our CPU's deadthread holder. This means 1215 * we needn't worry about someone else grabbing our context before we 1216 * do a cpu_throw(). 1217 */ 1218 void 1219 thread_exit(void) 1220 { 1221 struct thread *td; 1222 struct kse *ke; 1223 struct proc *p; 1224 struct ksegrp *kg; 1225 1226 td = curthread; 1227 kg = td->td_ksegrp; 1228 p = td->td_proc; 1229 ke = td->td_kse; 1230 1231 mtx_assert(&sched_lock, MA_OWNED); 1232 KASSERT(p != NULL, ("thread exiting without a process")); 1233 KASSERT(ke != NULL, ("thread exiting without a kse")); 1234 KASSERT(kg != NULL, ("thread exiting without a kse group")); 1235 PROC_LOCK_ASSERT(p, MA_OWNED); 1236 CTR1(KTR_PROC, "thread_exit: thread %p", td); 1237 KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 1238 1239 if (td->td_standin != NULL) { 1240 thread_stash(td->td_standin); 1241 td->td_standin = NULL; 1242 } 1243 1244 cpu_thread_exit(td); /* XXXSMP */ 1245 1246 /* 1247 * The last thread is left attached to the process 1248 * So that the whole bundle gets recycled. Skip 1249 * all this stuff. 1250 */ 1251 if (p->p_numthreads > 1) { 1252 thread_unlink(td); 1253 if (p->p_maxthrwaits) 1254 wakeup(&p->p_numthreads); 1255 /* 1256 * The test below is NOT true if we are the 1257 * sole exiting thread. P_STOPPED_SNGL is unset 1258 * in exit1() after it is the only survivor. 1259 */ 1260 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1261 if (p->p_numthreads == p->p_suspcount) { 1262 thread_unsuspend_one(p->p_singlethread); 1263 } 1264 } 1265 1266 /* 1267 * Because each upcall structure has an owner thread, 1268 * owner thread exits only when process is in exiting 1269 * state, so upcall to userland is no longer needed, 1270 * deleting upcall structure is safe here. 1271 * So when all threads in a group is exited, all upcalls 1272 * in the group should be automatically freed. 1273 */ 1274 if (td->td_upcall) 1275 upcall_remove(td); 1276 1277 sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); 1278 sched_exit_kse(FIRST_KSE_IN_PROC(p), ke); 1279 ke->ke_state = KES_UNQUEUED; 1280 ke->ke_thread = NULL; 1281 /* 1282 * Decide what to do with the KSE attached to this thread. 1283 */ 1284 if (ke->ke_flags & KEF_EXIT) { 1285 kse_unlink(ke); 1286 if (kg->kg_kses == 0) { 1287 sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), kg); 1288 ksegrp_unlink(kg); 1289 } 1290 } 1291 else 1292 kse_reassign(ke); 1293 PROC_UNLOCK(p); 1294 td->td_kse = NULL; 1295 td->td_state = TDS_INACTIVE; 1296 #if 0 1297 td->td_proc = NULL; 1298 #endif 1299 td->td_ksegrp = NULL; 1300 td->td_last_kse = NULL; 1301 PCPU_SET(deadthread, td); 1302 } else { 1303 PROC_UNLOCK(p); 1304 } 1305 /* XXX Shouldn't cpu_throw() here. */ 1306 mtx_assert(&sched_lock, MA_OWNED); 1307 cpu_throw(td, choosethread()); 1308 panic("I'm a teapot!"); 1309 /* NOTREACHED */ 1310 } 1311 1312 /* 1313 * Do any thread specific cleanups that may be needed in wait() 1314 * called with Giant held, proc and schedlock not held. 1315 */ 1316 void 1317 thread_wait(struct proc *p) 1318 { 1319 struct thread *td; 1320 1321 KASSERT((p->p_numthreads == 1), ("Muliple threads in wait1()")); 1322 KASSERT((p->p_numksegrps == 1), ("Muliple ksegrps in wait1()")); 1323 FOREACH_THREAD_IN_PROC(p, td) { 1324 if (td->td_standin != NULL) { 1325 thread_free(td->td_standin); 1326 td->td_standin = NULL; 1327 } 1328 cpu_thread_clean(td); 1329 } 1330 thread_reap(); /* check for zombie threads etc. */ 1331 } 1332 1333 /* 1334 * Link a thread to a process. 1335 * set up anything that needs to be initialized for it to 1336 * be used by the process. 1337 * 1338 * Note that we do not link to the proc's ucred here. 1339 * The thread is linked as if running but no KSE assigned. 1340 */ 1341 void 1342 thread_link(struct thread *td, struct ksegrp *kg) 1343 { 1344 struct proc *p; 1345 1346 p = kg->kg_proc; 1347 td->td_state = TDS_INACTIVE; 1348 td->td_proc = p; 1349 td->td_ksegrp = kg; 1350 td->td_last_kse = NULL; 1351 td->td_flags = 0; 1352 td->td_kse = NULL; 1353 1354 LIST_INIT(&td->td_contested); 1355 callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); 1356 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 1357 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 1358 p->p_numthreads++; 1359 kg->kg_numthreads++; 1360 } 1361 1362 void 1363 thread_unlink(struct thread *td) 1364 { 1365 struct proc *p = td->td_proc; 1366 struct ksegrp *kg = td->td_ksegrp; 1367 1368 mtx_assert(&sched_lock, MA_OWNED); 1369 TAILQ_REMOVE(&p->p_threads, td, td_plist); 1370 p->p_numthreads--; 1371 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1372 kg->kg_numthreads--; 1373 /* could clear a few other things here */ 1374 } 1375 1376 /* 1377 * Purge a ksegrp resource. When a ksegrp is preparing to 1378 * exit, it calls this function. 1379 */ 1380 static void 1381 kse_purge_group(struct thread *td) 1382 { 1383 struct ksegrp *kg; 1384 struct kse *ke; 1385 1386 kg = td->td_ksegrp; 1387 KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1388 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1389 KASSERT(ke->ke_state == KES_IDLE, 1390 ("%s: wrong idle KSE state", __func__)); 1391 kse_unlink(ke); 1392 } 1393 KASSERT((kg->kg_kses == 1), 1394 ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1395 KASSERT((kg->kg_numupcalls == 0), 1396 ("%s: ksegrp still has %d upcall datas", 1397 __func__, kg->kg_numupcalls)); 1398 } 1399 1400 /* 1401 * Purge a process's KSE resource. When a process is preparing to 1402 * exit, it calls kse_purge to release any extra KSE resources in 1403 * the process. 1404 */ 1405 static void 1406 kse_purge(struct proc *p, struct thread *td) 1407 { 1408 struct ksegrp *kg; 1409 struct kse *ke; 1410 1411 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1412 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1413 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1414 p->p_numksegrps--; 1415 /* 1416 * There is no ownership for KSE, after all threads 1417 * in the group exited, it is possible that some KSEs 1418 * were left in idle queue, gc them now. 1419 */ 1420 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1421 KASSERT(ke->ke_state == KES_IDLE, 1422 ("%s: wrong idle KSE state", __func__)); 1423 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1424 kg->kg_idle_kses--; 1425 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1426 kg->kg_kses--; 1427 kse_stash(ke); 1428 } 1429 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1430 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1431 ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1432 KASSERT((kg->kg_numupcalls == 0), 1433 ("%s: ksegrp still has %d upcall datas", 1434 __func__, kg->kg_numupcalls)); 1435 1436 if (kg != td->td_ksegrp) 1437 ksegrp_stash(kg); 1438 } 1439 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1440 p->p_numksegrps++; 1441 } 1442 1443 /* 1444 * This function is intended to be used to initialize a spare thread 1445 * for upcall. Initialize thread's large data area outside sched_lock 1446 * for thread_schedule_upcall(). 1447 */ 1448 void 1449 thread_alloc_spare(struct thread *td, struct thread *spare) 1450 { 1451 if (td->td_standin) 1452 return; 1453 if (spare == NULL) 1454 spare = thread_alloc(); 1455 td->td_standin = spare; 1456 bzero(&spare->td_startzero, 1457 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1458 spare->td_proc = td->td_proc; 1459 spare->td_ucred = crhold(td->td_ucred); 1460 } 1461 1462 /* 1463 * Create a thread and schedule it for upcall on the KSE given. 1464 * Use our thread's standin so that we don't have to allocate one. 1465 */ 1466 struct thread * 1467 thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 1468 { 1469 struct thread *td2; 1470 1471 mtx_assert(&sched_lock, MA_OWNED); 1472 1473 /* 1474 * Schedule an upcall thread on specified kse_upcall, 1475 * the kse_upcall must be free. 1476 * td must have a spare thread. 1477 */ 1478 KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1479 if ((td2 = td->td_standin) != NULL) { 1480 td->td_standin = NULL; 1481 } else { 1482 panic("no reserve thread when scheduling an upcall"); 1483 return (NULL); 1484 } 1485 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1486 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1487 bcopy(&td->td_startcopy, &td2->td_startcopy, 1488 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1489 thread_link(td2, ku->ku_ksegrp); 1490 /* inherit blocked thread's context */ 1491 cpu_set_upcall(td2, td); 1492 /* Let the new thread become owner of the upcall */ 1493 ku->ku_owner = td2; 1494 td2->td_upcall = ku; 1495 td2->td_flags = TDF_SA; 1496 td2->td_pflags = TDP_UPCALLING; 1497 td2->td_kse = NULL; 1498 td2->td_state = TDS_CAN_RUN; 1499 td2->td_inhibitors = 0; 1500 SIGFILLSET(td2->td_sigmask); 1501 SIG_CANTMASK(td2->td_sigmask); 1502 sched_fork_thread(td, td2); 1503 return (td2); /* bogus.. should be a void function */ 1504 } 1505 1506 /* 1507 * It is only used when thread generated a trap and process is being 1508 * debugged. 1509 */ 1510 void 1511 thread_signal_add(struct thread *td, int sig) 1512 { 1513 struct proc *p; 1514 siginfo_t siginfo; 1515 struct sigacts *ps; 1516 int error; 1517 1518 p = td->td_proc; 1519 PROC_LOCK_ASSERT(p, MA_OWNED); 1520 ps = p->p_sigacts; 1521 mtx_assert(&ps->ps_mtx, MA_OWNED); 1522 1523 cpu_thread_siginfo(sig, 0, &siginfo); 1524 mtx_unlock(&ps->ps_mtx); 1525 PROC_UNLOCK(p); 1526 error = copyout(&siginfo, &td->td_mailbox->tm_syncsig, sizeof(siginfo)); 1527 if (error) { 1528 PROC_LOCK(p); 1529 sigexit(td, SIGILL); 1530 } 1531 PROC_LOCK(p); 1532 SIGADDSET(td->td_sigmask, sig); 1533 mtx_lock(&ps->ps_mtx); 1534 } 1535 1536 void 1537 thread_switchout(struct thread *td) 1538 { 1539 struct kse_upcall *ku; 1540 struct thread *td2; 1541 1542 mtx_assert(&sched_lock, MA_OWNED); 1543 1544 /* 1545 * If the outgoing thread is in threaded group and has never 1546 * scheduled an upcall, decide whether this is a short 1547 * or long term event and thus whether or not to schedule 1548 * an upcall. 1549 * If it is a short term event, just suspend it in 1550 * a way that takes its KSE with it. 1551 * Select the events for which we want to schedule upcalls. 1552 * For now it's just sleep. 1553 * XXXKSE eventually almost any inhibition could do. 1554 */ 1555 if (TD_CAN_UNBIND(td) && (td->td_standin) && TD_ON_SLEEPQ(td)) { 1556 /* 1557 * Release ownership of upcall, and schedule an upcall 1558 * thread, this new upcall thread becomes the owner of 1559 * the upcall structure. 1560 */ 1561 ku = td->td_upcall; 1562 ku->ku_owner = NULL; 1563 td->td_upcall = NULL; 1564 td->td_flags &= ~TDF_CAN_UNBIND; 1565 td2 = thread_schedule_upcall(td, ku); 1566 setrunqueue(td2); 1567 } 1568 } 1569 1570 /* 1571 * Setup done on the thread when it enters the kernel. 1572 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1573 */ 1574 void 1575 thread_user_enter(struct proc *p, struct thread *td) 1576 { 1577 struct ksegrp *kg; 1578 struct kse_upcall *ku; 1579 struct kse_thr_mailbox *tmbx; 1580 uint32_t tflags; 1581 1582 kg = td->td_ksegrp; 1583 1584 /* 1585 * First check that we shouldn't just abort. 1586 * But check if we are the single thread first! 1587 */ 1588 if (p->p_flag & P_SINGLE_EXIT) { 1589 PROC_LOCK(p); 1590 mtx_lock_spin(&sched_lock); 1591 thread_stopped(p); 1592 thread_exit(); 1593 /* NOTREACHED */ 1594 } 1595 1596 /* 1597 * If we are doing a syscall in a KSE environment, 1598 * note where our mailbox is. There is always the 1599 * possibility that we could do this lazily (in kse_reassign()), 1600 * but for now do it every time. 1601 */ 1602 kg = td->td_ksegrp; 1603 if (td->td_flags & TDF_SA) { 1604 ku = td->td_upcall; 1605 KASSERT(ku, ("%s: no upcall owned", __func__)); 1606 KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1607 KASSERT(!TD_CAN_UNBIND(td), ("%s: can unbind", __func__)); 1608 ku->ku_mflags = fuword32((void *)&ku->ku_mailbox->km_flags); 1609 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1610 if ((tmbx == NULL) || (tmbx == (void *)-1L) || 1611 (ku->ku_mflags & KMF_NOUPCALL)) { 1612 td->td_mailbox = NULL; 1613 } else { 1614 if (td->td_standin == NULL) 1615 thread_alloc_spare(td, NULL); 1616 tflags = fuword32(&tmbx->tm_flags); 1617 /* 1618 * On some architectures, TP register points to thread 1619 * mailbox but not points to kse mailbox, and userland 1620 * can not atomically clear km_curthread, but can 1621 * use TP register, and set TMF_NOUPCALL in thread 1622 * flag to indicate a critical region. 1623 */ 1624 if (tflags & TMF_NOUPCALL) { 1625 td->td_mailbox = NULL; 1626 } else { 1627 td->td_mailbox = tmbx; 1628 mtx_lock_spin(&sched_lock); 1629 td->td_flags |= TDF_CAN_UNBIND; 1630 mtx_unlock_spin(&sched_lock); 1631 } 1632 } 1633 } 1634 } 1635 1636 /* 1637 * The extra work we go through if we are a threaded process when we 1638 * return to userland. 1639 * 1640 * If we are a KSE process and returning to user mode, check for 1641 * extra work to do before we return (e.g. for more syscalls 1642 * to complete first). If we were in a critical section, we should 1643 * just return to let it finish. Same if we were in the UTS (in 1644 * which case the mailbox's context's busy indicator will be set). 1645 * The only traps we suport will have set the mailbox. 1646 * We will clear it here. 1647 */ 1648 int 1649 thread_userret(struct thread *td, struct trapframe *frame) 1650 { 1651 int error = 0, upcalls, uts_crit; 1652 struct kse_upcall *ku; 1653 struct ksegrp *kg, *kg2; 1654 struct proc *p; 1655 struct timespec ts; 1656 1657 p = td->td_proc; 1658 kg = td->td_ksegrp; 1659 ku = td->td_upcall; 1660 1661 /* Nothing to do with bound thread */ 1662 if (!(td->td_flags & TDF_SA)) 1663 return (0); 1664 1665 /* 1666 * Stat clock interrupt hit in userland, it 1667 * is returning from interrupt, charge thread's 1668 * userland time for UTS. 1669 */ 1670 if (td->td_flags & TDF_USTATCLOCK) { 1671 thread_update_usr_ticks(td, 1); 1672 mtx_lock_spin(&sched_lock); 1673 td->td_flags &= ~TDF_USTATCLOCK; 1674 mtx_unlock_spin(&sched_lock); 1675 if (kg->kg_completed || 1676 (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1677 thread_user_enter(p, td); 1678 } 1679 1680 uts_crit = (td->td_mailbox == NULL); 1681 /* 1682 * Optimisation: 1683 * This thread has not started any upcall. 1684 * If there is no work to report other than ourself, 1685 * then it can return direct to userland. 1686 */ 1687 if (TD_CAN_UNBIND(td)) { 1688 mtx_lock_spin(&sched_lock); 1689 td->td_flags &= ~TDF_CAN_UNBIND; 1690 if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && 1691 (kg->kg_completed == NULL) && 1692 (ku->ku_flags & KUF_DOUPCALL) == 0 && 1693 (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { 1694 mtx_unlock_spin(&sched_lock); 1695 thread_update_usr_ticks(td, 0); 1696 nanotime(&ts); 1697 error = copyout(&ts, 1698 (caddr_t)&ku->ku_mailbox->km_timeofday, 1699 sizeof(ts)); 1700 td->td_mailbox = 0; 1701 ku->ku_mflags = 0; 1702 if (error) 1703 goto out; 1704 return (0); 1705 } 1706 mtx_unlock_spin(&sched_lock); 1707 thread_export_context(td, 0); 1708 /* 1709 * There is something to report, and we own an upcall 1710 * strucuture, we can go to userland. 1711 * Turn ourself into an upcall thread. 1712 */ 1713 td->td_pflags |= TDP_UPCALLING; 1714 } else if (td->td_mailbox && (ku == NULL)) { 1715 thread_export_context(td, 1); 1716 PROC_LOCK(p); 1717 /* 1718 * There are upcall threads waiting for 1719 * work to do, wake one of them up. 1720 * XXXKSE Maybe wake all of them up. 1721 */ 1722 if (kg->kg_upsleeps) 1723 wakeup_one(&kg->kg_completed); 1724 mtx_lock_spin(&sched_lock); 1725 thread_stopped(p); 1726 thread_exit(); 1727 /* NOTREACHED */ 1728 } 1729 1730 KASSERT(ku != NULL, ("upcall is NULL\n")); 1731 KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1732 1733 if (p->p_numthreads > max_threads_per_proc) { 1734 max_threads_hits++; 1735 PROC_LOCK(p); 1736 mtx_lock_spin(&sched_lock); 1737 p->p_maxthrwaits++; 1738 while (p->p_numthreads > max_threads_per_proc) { 1739 upcalls = 0; 1740 FOREACH_KSEGRP_IN_PROC(p, kg2) { 1741 if (kg2->kg_numupcalls == 0) 1742 upcalls++; 1743 else 1744 upcalls += kg2->kg_numupcalls; 1745 } 1746 if (upcalls >= max_threads_per_proc) 1747 break; 1748 mtx_unlock_spin(&sched_lock); 1749 if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1750 "maxthreads", NULL)) { 1751 mtx_lock_spin(&sched_lock); 1752 break; 1753 } else { 1754 mtx_lock_spin(&sched_lock); 1755 } 1756 } 1757 p->p_maxthrwaits--; 1758 mtx_unlock_spin(&sched_lock); 1759 PROC_UNLOCK(p); 1760 } 1761 1762 if (td->td_pflags & TDP_UPCALLING) { 1763 uts_crit = 0; 1764 kg->kg_nextupcall = ticks+kg->kg_upquantum; 1765 /* 1766 * There is no more work to do and we are going to ride 1767 * this thread up to userland as an upcall. 1768 * Do the last parts of the setup needed for the upcall. 1769 */ 1770 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1771 td, td->td_proc->p_pid, td->td_proc->p_comm); 1772 1773 td->td_pflags &= ~TDP_UPCALLING; 1774 if (ku->ku_flags & KUF_DOUPCALL) { 1775 mtx_lock_spin(&sched_lock); 1776 ku->ku_flags &= ~KUF_DOUPCALL; 1777 mtx_unlock_spin(&sched_lock); 1778 } 1779 /* 1780 * Set user context to the UTS 1781 */ 1782 if (!(ku->ku_mflags & KMF_NOUPCALL)) { 1783 cpu_set_upcall_kse(td, ku); 1784 error = suword(&ku->ku_mailbox->km_curthread, 0); 1785 if (error) 1786 goto out; 1787 } 1788 1789 /* 1790 * Unhook the list of completed threads. 1791 * anything that completes after this gets to 1792 * come in next time. 1793 * Put the list of completed thread mailboxes on 1794 * this KSE's mailbox. 1795 */ 1796 if (!(ku->ku_mflags & KMF_NOCOMPLETED) && 1797 (error = thread_link_mboxes(kg, ku)) != 0) 1798 goto out; 1799 } 1800 if (!uts_crit) { 1801 nanotime(&ts); 1802 error = copyout(&ts, &ku->ku_mailbox->km_timeofday, sizeof(ts)); 1803 } 1804 1805 out: 1806 if (error) { 1807 /* 1808 * Things are going to be so screwed we should just kill 1809 * the process. 1810 * how do we do that? 1811 */ 1812 PROC_LOCK(td->td_proc); 1813 psignal(td->td_proc, SIGSEGV); 1814 PROC_UNLOCK(td->td_proc); 1815 } else { 1816 /* 1817 * Optimisation: 1818 * Ensure that we have a spare thread available, 1819 * for when we re-enter the kernel. 1820 */ 1821 if (td->td_standin == NULL) 1822 thread_alloc_spare(td, NULL); 1823 } 1824 1825 ku->ku_mflags = 0; 1826 /* 1827 * Clear thread mailbox first, then clear system tick count. 1828 * The order is important because thread_statclock() use 1829 * mailbox pointer to see if it is an userland thread or 1830 * an UTS kernel thread. 1831 */ 1832 td->td_mailbox = NULL; 1833 td->td_usticks = 0; 1834 return (error); /* go sync */ 1835 } 1836 1837 /* 1838 * Enforce single-threading. 1839 * 1840 * Returns 1 if the caller must abort (another thread is waiting to 1841 * exit the process or similar). Process is locked! 1842 * Returns 0 when you are successfully the only thread running. 1843 * A process has successfully single threaded in the suspend mode when 1844 * There are no threads in user mode. Threads in the kernel must be 1845 * allowed to continue until they get to the user boundary. They may even 1846 * copy out their return values and data before suspending. They may however be 1847 * accellerated in reaching the user boundary as we will wake up 1848 * any sleeping threads that are interruptable. (PCATCH). 1849 */ 1850 int 1851 thread_single(int force_exit) 1852 { 1853 struct thread *td; 1854 struct thread *td2; 1855 struct proc *p; 1856 1857 td = curthread; 1858 p = td->td_proc; 1859 mtx_assert(&Giant, MA_OWNED); 1860 PROC_LOCK_ASSERT(p, MA_OWNED); 1861 KASSERT((td != NULL), ("curthread is NULL")); 1862 1863 if ((p->p_flag & P_SA) == 0 && p->p_numthreads == 1) 1864 return (0); 1865 1866 /* Is someone already single threading? */ 1867 if (p->p_singlethread) 1868 return (1); 1869 1870 if (force_exit == SINGLE_EXIT) { 1871 p->p_flag |= P_SINGLE_EXIT; 1872 } else 1873 p->p_flag &= ~P_SINGLE_EXIT; 1874 p->p_flag |= P_STOPPED_SINGLE; 1875 mtx_lock_spin(&sched_lock); 1876 p->p_singlethread = td; 1877 while ((p->p_numthreads - p->p_suspcount) != 1) { 1878 FOREACH_THREAD_IN_PROC(p, td2) { 1879 if (td2 == td) 1880 continue; 1881 td2->td_flags |= TDF_ASTPENDING; 1882 if (TD_IS_INHIBITED(td2)) { 1883 if (force_exit == SINGLE_EXIT) { 1884 if (TD_IS_SUSPENDED(td2)) { 1885 thread_unsuspend_one(td2); 1886 } 1887 if (TD_ON_SLEEPQ(td2) && 1888 (td2->td_flags & TDF_SINTR)) { 1889 if (td2->td_flags & TDF_CVWAITQ) 1890 cv_abort(td2); 1891 else 1892 abortsleep(td2); 1893 } 1894 } else { 1895 if (TD_IS_SUSPENDED(td2)) 1896 continue; 1897 /* 1898 * maybe other inhibitted states too? 1899 * XXXKSE Is it totally safe to 1900 * suspend a non-interruptable thread? 1901 */ 1902 if (td2->td_inhibitors & 1903 (TDI_SLEEPING | TDI_SWAPPED)) 1904 thread_suspend_one(td2); 1905 } 1906 } 1907 } 1908 /* 1909 * Maybe we suspended some threads.. was it enough? 1910 */ 1911 if ((p->p_numthreads - p->p_suspcount) == 1) 1912 break; 1913 1914 /* 1915 * Wake us up when everyone else has suspended. 1916 * In the mean time we suspend as well. 1917 */ 1918 thread_suspend_one(td); 1919 DROP_GIANT(); 1920 PROC_UNLOCK(p); 1921 p->p_stats->p_ru.ru_nvcsw++; 1922 mi_switch(); 1923 mtx_unlock_spin(&sched_lock); 1924 PICKUP_GIANT(); 1925 PROC_LOCK(p); 1926 mtx_lock_spin(&sched_lock); 1927 } 1928 if (force_exit == SINGLE_EXIT) { 1929 if (td->td_upcall) 1930 upcall_remove(td); 1931 kse_purge(p, td); 1932 } 1933 mtx_unlock_spin(&sched_lock); 1934 return (0); 1935 } 1936 1937 /* 1938 * Called in from locations that can safely check to see 1939 * whether we have to suspend or at least throttle for a 1940 * single-thread event (e.g. fork). 1941 * 1942 * Such locations include userret(). 1943 * If the "return_instead" argument is non zero, the thread must be able to 1944 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 1945 * 1946 * The 'return_instead' argument tells the function if it may do a 1947 * thread_exit() or suspend, or whether the caller must abort and back 1948 * out instead. 1949 * 1950 * If the thread that set the single_threading request has set the 1951 * P_SINGLE_EXIT bit in the process flags then this call will never return 1952 * if 'return_instead' is false, but will exit. 1953 * 1954 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 1955 *---------------+--------------------+--------------------- 1956 * 0 | returns 0 | returns 0 or 1 1957 * | when ST ends | immediatly 1958 *---------------+--------------------+--------------------- 1959 * 1 | thread exits | returns 1 1960 * | | immediatly 1961 * 0 = thread_exit() or suspension ok, 1962 * other = return error instead of stopping the thread. 1963 * 1964 * While a full suspension is under effect, even a single threading 1965 * thread would be suspended if it made this call (but it shouldn't). 1966 * This call should only be made from places where 1967 * thread_exit() would be safe as that may be the outcome unless 1968 * return_instead is set. 1969 */ 1970 int 1971 thread_suspend_check(int return_instead) 1972 { 1973 struct thread *td; 1974 struct proc *p; 1975 1976 td = curthread; 1977 p = td->td_proc; 1978 PROC_LOCK_ASSERT(p, MA_OWNED); 1979 while (P_SHOULDSTOP(p)) { 1980 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1981 KASSERT(p->p_singlethread != NULL, 1982 ("singlethread not set")); 1983 /* 1984 * The only suspension in action is a 1985 * single-threading. Single threader need not stop. 1986 * XXX Should be safe to access unlocked 1987 * as it can only be set to be true by us. 1988 */ 1989 if (p->p_singlethread == td) 1990 return (0); /* Exempt from stopping. */ 1991 } 1992 if (return_instead) 1993 return (1); 1994 1995 mtx_lock_spin(&sched_lock); 1996 thread_stopped(p); 1997 /* 1998 * If the process is waiting for us to exit, 1999 * this thread should just suicide. 2000 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 2001 */ 2002 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 2003 while (mtx_owned(&Giant)) 2004 mtx_unlock(&Giant); 2005 if (p->p_flag & P_SA) 2006 thread_exit(); 2007 else 2008 thr_exit1(); 2009 } 2010 2011 /* 2012 * When a thread suspends, it just 2013 * moves to the processes's suspend queue 2014 * and stays there. 2015 */ 2016 thread_suspend_one(td); 2017 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 2018 if (p->p_numthreads == p->p_suspcount) { 2019 thread_unsuspend_one(p->p_singlethread); 2020 } 2021 } 2022 DROP_GIANT(); 2023 PROC_UNLOCK(p); 2024 p->p_stats->p_ru.ru_nivcsw++; 2025 mi_switch(); 2026 mtx_unlock_spin(&sched_lock); 2027 PICKUP_GIANT(); 2028 PROC_LOCK(p); 2029 } 2030 return (0); 2031 } 2032 2033 void 2034 thread_suspend_one(struct thread *td) 2035 { 2036 struct proc *p = td->td_proc; 2037 2038 mtx_assert(&sched_lock, MA_OWNED); 2039 PROC_LOCK_ASSERT(p, MA_OWNED); 2040 KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); 2041 p->p_suspcount++; 2042 TD_SET_SUSPENDED(td); 2043 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 2044 /* 2045 * Hack: If we are suspending but are on the sleep queue 2046 * then we are in msleep or the cv equivalent. We 2047 * want to look like we have two Inhibitors. 2048 * May already be set.. doesn't matter. 2049 */ 2050 if (TD_ON_SLEEPQ(td)) 2051 TD_SET_SLEEPING(td); 2052 } 2053 2054 void 2055 thread_unsuspend_one(struct thread *td) 2056 { 2057 struct proc *p = td->td_proc; 2058 2059 mtx_assert(&sched_lock, MA_OWNED); 2060 PROC_LOCK_ASSERT(p, MA_OWNED); 2061 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 2062 TD_CLR_SUSPENDED(td); 2063 p->p_suspcount--; 2064 setrunnable(td); 2065 } 2066 2067 /* 2068 * Allow all threads blocked by single threading to continue running. 2069 */ 2070 void 2071 thread_unsuspend(struct proc *p) 2072 { 2073 struct thread *td; 2074 2075 mtx_assert(&sched_lock, MA_OWNED); 2076 PROC_LOCK_ASSERT(p, MA_OWNED); 2077 if (!P_SHOULDSTOP(p)) { 2078 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2079 thread_unsuspend_one(td); 2080 } 2081 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 2082 (p->p_numthreads == p->p_suspcount)) { 2083 /* 2084 * Stopping everything also did the job for the single 2085 * threading request. Now we've downgraded to single-threaded, 2086 * let it continue. 2087 */ 2088 thread_unsuspend_one(p->p_singlethread); 2089 } 2090 } 2091 2092 void 2093 thread_single_end(void) 2094 { 2095 struct thread *td; 2096 struct proc *p; 2097 2098 td = curthread; 2099 p = td->td_proc; 2100 PROC_LOCK_ASSERT(p, MA_OWNED); 2101 p->p_flag &= ~P_STOPPED_SINGLE; 2102 mtx_lock_spin(&sched_lock); 2103 p->p_singlethread = NULL; 2104 /* 2105 * If there are other threads they mey now run, 2106 * unless of course there is a blanket 'stop order' 2107 * on the process. The single threader must be allowed 2108 * to continue however as this is a bad place to stop. 2109 */ 2110 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2111 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2112 thread_unsuspend_one(td); 2113 } 2114 } 2115 mtx_unlock_spin(&sched_lock); 2116 } 2117 2118 2119