1 /* 2 * Copyright (C) 2001 Julian Elischer <julian@freebsd.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice(s), this list of conditions and the following disclaimer as 10 * the first lines of this file unmodified other than the possible 11 * addition of one or more copyright notices. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice(s), this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 20 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 26 * DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/lock.h> 35 #include <sys/malloc.h> 36 #include <sys/mutex.h> 37 #include <sys/proc.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/sysproto.h> 41 #include <sys/filedesc.h> 42 #include <sys/sched.h> 43 #include <sys/signalvar.h> 44 #include <sys/sx.h> 45 #include <sys/tty.h> 46 #include <sys/user.h> 47 #include <sys/jail.h> 48 #include <sys/kse.h> 49 #include <sys/ktr.h> 50 #include <sys/ucontext.h> 51 52 #include <vm/vm.h> 53 #include <vm/vm_object.h> 54 #include <vm/pmap.h> 55 #include <vm/uma.h> 56 #include <vm/vm_map.h> 57 58 #include <machine/frame.h> 59 60 /* 61 * KSEGRP related storage. 62 */ 63 static uma_zone_t ksegrp_zone; 64 static uma_zone_t kse_zone; 65 static uma_zone_t thread_zone; 66 static uma_zone_t upcall_zone; 67 68 /* DEBUG ONLY */ 69 SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW, 0, "thread allocation"); 70 static int thread_debug = 0; 71 SYSCTL_INT(_kern_threads, OID_AUTO, debug, CTLFLAG_RW, 72 &thread_debug, 0, "thread debug"); 73 74 static int max_threads_per_proc = 30; 75 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW, 76 &max_threads_per_proc, 0, "Limit on threads per proc"); 77 78 static int max_groups_per_proc = 5; 79 SYSCTL_INT(_kern_threads, OID_AUTO, max_groups_per_proc, CTLFLAG_RW, 80 &max_groups_per_proc, 0, "Limit on thread groups per proc"); 81 82 static int max_threads_hits; 83 SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD, 84 &max_threads_hits, 0, ""); 85 86 static int virtual_cpu; 87 88 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start)) 89 90 TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); 91 TAILQ_HEAD(, kse) zombie_kses = TAILQ_HEAD_INITIALIZER(zombie_kses); 92 TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); 93 TAILQ_HEAD(, kse_upcall) zombie_upcalls = 94 TAILQ_HEAD_INITIALIZER(zombie_upcalls); 95 struct mtx kse_zombie_lock; 96 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); 97 98 static void kse_purge(struct proc *p, struct thread *td); 99 static void kse_purge_group(struct thread *td); 100 static int thread_update_usr_ticks(struct thread *td, int user); 101 static void thread_alloc_spare(struct thread *td, struct thread *spare); 102 103 static int 104 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS) 105 { 106 int error, new_val; 107 int def_val; 108 109 #ifdef SMP 110 def_val = mp_ncpus; 111 #else 112 def_val = 1; 113 #endif 114 if (virtual_cpu == 0) 115 new_val = def_val; 116 else 117 new_val = virtual_cpu; 118 error = sysctl_handle_int(oidp, &new_val, 0, req); 119 if (error != 0 || req->newptr == NULL) 120 return (error); 121 if (new_val < 0) 122 return (EINVAL); 123 virtual_cpu = new_val; 124 return (0); 125 } 126 127 /* DEBUG ONLY */ 128 SYSCTL_PROC(_kern_threads, OID_AUTO, virtual_cpu, CTLTYPE_INT|CTLFLAG_RW, 129 0, sizeof(virtual_cpu), sysctl_kse_virtual_cpu, "I", 130 "debug virtual cpus"); 131 132 /* 133 * Prepare a thread for use. 134 */ 135 static void 136 thread_ctor(void *mem, int size, void *arg) 137 { 138 struct thread *td; 139 140 td = (struct thread *)mem; 141 td->td_state = TDS_INACTIVE; 142 } 143 144 /* 145 * Reclaim a thread after use. 146 */ 147 static void 148 thread_dtor(void *mem, int size, void *arg) 149 { 150 struct thread *td; 151 152 td = (struct thread *)mem; 153 154 #ifdef INVARIANTS 155 /* Verify that this thread is in a safe state to free. */ 156 switch (td->td_state) { 157 case TDS_INHIBITED: 158 case TDS_RUNNING: 159 case TDS_CAN_RUN: 160 case TDS_RUNQ: 161 /* 162 * We must never unlink a thread that is in one of 163 * these states, because it is currently active. 164 */ 165 panic("bad state for thread unlinking"); 166 /* NOTREACHED */ 167 case TDS_INACTIVE: 168 break; 169 default: 170 panic("bad thread state"); 171 /* NOTREACHED */ 172 } 173 #endif 174 } 175 176 /* 177 * Initialize type-stable parts of a thread (when newly created). 178 */ 179 static void 180 thread_init(void *mem, int size) 181 { 182 struct thread *td; 183 184 td = (struct thread *)mem; 185 mtx_lock(&Giant); 186 pmap_new_thread(td, 0); 187 mtx_unlock(&Giant); 188 cpu_thread_setup(td); 189 td->td_sched = (struct td_sched *)&td[1]; 190 } 191 192 /* 193 * Tear down type-stable parts of a thread (just before being discarded). 194 */ 195 static void 196 thread_fini(void *mem, int size) 197 { 198 struct thread *td; 199 200 td = (struct thread *)mem; 201 pmap_dispose_thread(td); 202 } 203 204 /* 205 * Initialize type-stable parts of a kse (when newly created). 206 */ 207 static void 208 kse_init(void *mem, int size) 209 { 210 struct kse *ke; 211 212 ke = (struct kse *)mem; 213 ke->ke_sched = (struct ke_sched *)&ke[1]; 214 } 215 216 /* 217 * Initialize type-stable parts of a ksegrp (when newly created). 218 */ 219 static void 220 ksegrp_init(void *mem, int size) 221 { 222 struct ksegrp *kg; 223 224 kg = (struct ksegrp *)mem; 225 kg->kg_sched = (struct kg_sched *)&kg[1]; 226 } 227 228 /* 229 * KSE is linked into kse group. 230 */ 231 void 232 kse_link(struct kse *ke, struct ksegrp *kg) 233 { 234 struct proc *p = kg->kg_proc; 235 236 TAILQ_INSERT_HEAD(&kg->kg_kseq, ke, ke_kglist); 237 kg->kg_kses++; 238 ke->ke_state = KES_UNQUEUED; 239 ke->ke_proc = p; 240 ke->ke_ksegrp = kg; 241 ke->ke_thread = NULL; 242 ke->ke_oncpu = NOCPU; 243 ke->ke_flags = 0; 244 } 245 246 void 247 kse_unlink(struct kse *ke) 248 { 249 struct ksegrp *kg; 250 251 mtx_assert(&sched_lock, MA_OWNED); 252 kg = ke->ke_ksegrp; 253 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 254 if (ke->ke_state == KES_IDLE) { 255 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 256 kg->kg_idle_kses--; 257 } 258 if (--kg->kg_kses == 0) 259 ksegrp_unlink(kg); 260 /* 261 * Aggregate stats from the KSE 262 */ 263 kse_stash(ke); 264 } 265 266 void 267 ksegrp_link(struct ksegrp *kg, struct proc *p) 268 { 269 270 TAILQ_INIT(&kg->kg_threads); 271 TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ 272 TAILQ_INIT(&kg->kg_slpq); /* links with td_runq */ 273 TAILQ_INIT(&kg->kg_kseq); /* all kses in ksegrp */ 274 TAILQ_INIT(&kg->kg_iq); /* all idle kses in ksegrp */ 275 TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ 276 kg->kg_proc = p; 277 /* 278 * the following counters are in the -zero- section 279 * and may not need clearing 280 */ 281 kg->kg_numthreads = 0; 282 kg->kg_runnable = 0; 283 kg->kg_kses = 0; 284 kg->kg_runq_kses = 0; /* XXXKSE change name */ 285 kg->kg_idle_kses = 0; 286 kg->kg_numupcalls = 0; 287 /* link it in now that it's consistent */ 288 p->p_numksegrps++; 289 TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); 290 } 291 292 void 293 ksegrp_unlink(struct ksegrp *kg) 294 { 295 struct proc *p; 296 297 mtx_assert(&sched_lock, MA_OWNED); 298 KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); 299 KASSERT((kg->kg_kses == 0), ("ksegrp_unlink: residual kses")); 300 KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); 301 302 p = kg->kg_proc; 303 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 304 p->p_numksegrps--; 305 /* 306 * Aggregate stats from the KSE 307 */ 308 ksegrp_stash(kg); 309 } 310 311 struct kse_upcall * 312 upcall_alloc(void) 313 { 314 struct kse_upcall *ku; 315 316 ku = uma_zalloc(upcall_zone, M_WAITOK); 317 bzero(ku, sizeof(*ku)); 318 return (ku); 319 } 320 321 void 322 upcall_free(struct kse_upcall *ku) 323 { 324 325 uma_zfree(upcall_zone, ku); 326 } 327 328 void 329 upcall_link(struct kse_upcall *ku, struct ksegrp *kg) 330 { 331 332 mtx_assert(&sched_lock, MA_OWNED); 333 TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); 334 ku->ku_ksegrp = kg; 335 kg->kg_numupcalls++; 336 } 337 338 void 339 upcall_unlink(struct kse_upcall *ku) 340 { 341 struct ksegrp *kg = ku->ku_ksegrp; 342 343 mtx_assert(&sched_lock, MA_OWNED); 344 KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); 345 TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); 346 kg->kg_numupcalls--; 347 upcall_stash(ku); 348 } 349 350 void 351 upcall_remove(struct thread *td) 352 { 353 354 if (td->td_upcall) { 355 td->td_upcall->ku_owner = NULL; 356 upcall_unlink(td->td_upcall); 357 td->td_upcall = 0; 358 } 359 } 360 361 /* 362 * For a newly created process, 363 * link up all the structures and its initial threads etc. 364 */ 365 void 366 proc_linkup(struct proc *p, struct ksegrp *kg, 367 struct kse *ke, struct thread *td) 368 { 369 370 TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ 371 TAILQ_INIT(&p->p_threads); /* all threads in proc */ 372 TAILQ_INIT(&p->p_suspended); /* Threads suspended */ 373 p->p_numksegrps = 0; 374 p->p_numthreads = 0; 375 376 ksegrp_link(kg, p); 377 kse_link(ke, kg); 378 thread_link(td, kg); 379 } 380 381 /* 382 struct kse_thr_interrupt_args { 383 struct kse_thr_mailbox * tmbx; 384 }; 385 */ 386 int 387 kse_thr_interrupt(struct thread *td, struct kse_thr_interrupt_args *uap) 388 { 389 struct proc *p; 390 struct thread *td2; 391 392 p = td->td_proc; 393 if (!(p->p_flag & P_THREADED) || (uap->tmbx == NULL)) 394 return (EINVAL); 395 mtx_lock_spin(&sched_lock); 396 FOREACH_THREAD_IN_PROC(p, td2) { 397 if (td2->td_mailbox == uap->tmbx) { 398 td2->td_flags |= TDF_INTERRUPT; 399 if (TD_ON_SLEEPQ(td2) && (td2->td_flags & TDF_SINTR)) { 400 if (td2->td_flags & TDF_CVWAITQ) 401 cv_abort(td2); 402 else 403 abortsleep(td2); 404 } 405 mtx_unlock_spin(&sched_lock); 406 return (0); 407 } 408 } 409 mtx_unlock_spin(&sched_lock); 410 return (ESRCH); 411 } 412 413 /* 414 struct kse_exit_args { 415 register_t dummy; 416 }; 417 */ 418 int 419 kse_exit(struct thread *td, struct kse_exit_args *uap) 420 { 421 struct proc *p; 422 struct ksegrp *kg; 423 struct kse *ke; 424 425 p = td->td_proc; 426 /* 427 * Only UTS can call the syscall and current group 428 * should be a threaded group. 429 */ 430 if ((td->td_mailbox != NULL) || (td->td_ksegrp->kg_numupcalls == 0)) 431 return (EINVAL); 432 KASSERT((td->td_upcall != NULL), ("%s: not own an upcall", __func__)); 433 434 kg = td->td_ksegrp; 435 /* Serialize removing upcall */ 436 PROC_LOCK(p); 437 mtx_lock_spin(&sched_lock); 438 if ((kg->kg_numupcalls == 1) && (kg->kg_numthreads > 1)) { 439 mtx_unlock_spin(&sched_lock); 440 PROC_UNLOCK(p); 441 return (EDEADLK); 442 } 443 ke = td->td_kse; 444 upcall_remove(td); 445 if (p->p_numthreads == 1) { 446 kse_purge(p, td); 447 p->p_flag &= ~P_THREADED; 448 mtx_unlock_spin(&sched_lock); 449 PROC_UNLOCK(p); 450 } else { 451 if (kg->kg_numthreads == 1) { /* Shutdown a group */ 452 kse_purge_group(td); 453 ke->ke_flags |= KEF_EXIT; 454 } 455 thread_exit(); 456 /* NOTREACHED */ 457 } 458 return (0); 459 } 460 461 /* 462 * Either becomes an upcall or waits for an awakening event and 463 * then becomes an upcall. Only error cases return. 464 */ 465 /* 466 struct kse_release_args { 467 struct timespec *timeout; 468 }; 469 */ 470 int 471 kse_release(struct thread *td, struct kse_release_args *uap) 472 { 473 struct proc *p; 474 struct ksegrp *kg; 475 struct timespec ts, ts2, ts3, timeout; 476 struct timeval tv; 477 int error; 478 479 p = td->td_proc; 480 kg = td->td_ksegrp; 481 /* 482 * Only UTS can call the syscall and current group 483 * should be a threaded group. 484 */ 485 if ((td->td_mailbox != NULL) || (td->td_ksegrp->kg_numupcalls == 0)) 486 return (EINVAL); 487 KASSERT((td->td_upcall != NULL), ("%s: not own an upcall", __func__)); 488 if (uap->timeout != NULL) { 489 if ((error = copyin(uap->timeout, &timeout, sizeof(timeout)))) 490 return (error); 491 getnanouptime(&ts); 492 timespecadd(&ts, &timeout); 493 TIMESPEC_TO_TIMEVAL(&tv, &timeout); 494 } 495 mtx_lock_spin(&sched_lock); 496 /* Change OURSELF to become an upcall. */ 497 td->td_flags = TDF_UPCALLING; 498 if (p->p_sflag & PS_NEEDSIGCHK) 499 td->td_flags |= TDF_ASTPENDING; 500 mtx_unlock_spin(&sched_lock); 501 PROC_LOCK(p); 502 while ((td->td_upcall->ku_flags & KUF_DOUPCALL) == 0 && 503 (kg->kg_completed == NULL)) { 504 kg->kg_upsleeps++; 505 error = msleep(&kg->kg_completed, &p->p_mtx, PPAUSE|PCATCH, 506 "kse_rel", (uap->timeout ? tvtohz(&tv) : 0)); 507 kg->kg_upsleeps--; 508 PROC_UNLOCK(p); 509 if (uap->timeout == NULL || error != EWOULDBLOCK) 510 return (0); 511 getnanouptime(&ts2); 512 if (timespeccmp(&ts2, &ts, >=)) 513 return (0); 514 ts3 = ts; 515 timespecsub(&ts3, &ts2); 516 TIMESPEC_TO_TIMEVAL(&tv, &ts3); 517 PROC_LOCK(p); 518 } 519 PROC_UNLOCK(p); 520 return (0); 521 } 522 523 /* struct kse_wakeup_args { 524 struct kse_mailbox *mbx; 525 }; */ 526 int 527 kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) 528 { 529 struct proc *p; 530 struct ksegrp *kg; 531 struct kse_upcall *ku; 532 struct thread *td2; 533 534 p = td->td_proc; 535 td2 = NULL; 536 ku = NULL; 537 /* KSE-enabled processes only, please. */ 538 if (!(p->p_flag & P_THREADED)) 539 return (EINVAL); 540 PROC_LOCK(p); 541 mtx_lock_spin(&sched_lock); 542 if (uap->mbx) { 543 FOREACH_KSEGRP_IN_PROC(p, kg) { 544 FOREACH_UPCALL_IN_GROUP(kg, ku) { 545 if (ku->ku_mailbox == uap->mbx) 546 break; 547 } 548 if (ku) 549 break; 550 } 551 } else { 552 kg = td->td_ksegrp; 553 if (kg->kg_upsleeps) { 554 wakeup_one(&kg->kg_completed); 555 mtx_unlock_spin(&sched_lock); 556 PROC_UNLOCK(p); 557 return (0); 558 } 559 ku = TAILQ_FIRST(&kg->kg_upcalls); 560 } 561 if (ku) { 562 if ((td2 = ku->ku_owner) == NULL) { 563 panic("%s: no owner", __func__); 564 } else if (TD_ON_SLEEPQ(td2) && 565 (td2->td_wchan == &kg->kg_completed)) { 566 abortsleep(td2); 567 } else { 568 ku->ku_flags |= KUF_DOUPCALL; 569 } 570 mtx_unlock_spin(&sched_lock); 571 PROC_UNLOCK(p); 572 return (0); 573 } 574 mtx_unlock_spin(&sched_lock); 575 PROC_UNLOCK(p); 576 return (ESRCH); 577 } 578 579 /* 580 * No new KSEG: first call: use current KSE, don't schedule an upcall 581 * All other situations, do allocate max new KSEs and schedule an upcall. 582 */ 583 /* struct kse_create_args { 584 struct kse_mailbox *mbx; 585 int newgroup; 586 }; */ 587 int 588 kse_create(struct thread *td, struct kse_create_args *uap) 589 { 590 struct kse *newke; 591 struct ksegrp *newkg; 592 struct ksegrp *kg; 593 struct proc *p; 594 struct kse_mailbox mbx; 595 struct kse_upcall *newku; 596 int err, ncpus; 597 598 p = td->td_proc; 599 if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) 600 return (err); 601 602 /* Too bad, why hasn't kernel always a cpu counter !? */ 603 #ifdef SMP 604 ncpus = mp_ncpus; 605 #else 606 ncpus = 1; 607 #endif 608 if (thread_debug && virtual_cpu != 0) 609 ncpus = virtual_cpu; 610 611 /* Easier to just set it than to test and set */ 612 p->p_flag |= P_THREADED; 613 kg = td->td_ksegrp; 614 if (uap->newgroup) { 615 /* Have race condition but it is cheap */ 616 if (p->p_numksegrps >= max_groups_per_proc) 617 return (EPROCLIM); 618 /* 619 * If we want a new KSEGRP it doesn't matter whether 620 * we have already fired up KSE mode before or not. 621 * We put the process in KSE mode and create a new KSEGRP. 622 */ 623 newkg = ksegrp_alloc(); 624 bzero(&newkg->kg_startzero, RANGEOF(struct ksegrp, 625 kg_startzero, kg_endzero)); 626 bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, 627 RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy)); 628 mtx_lock_spin(&sched_lock); 629 if (p->p_numksegrps >= max_groups_per_proc) { 630 mtx_unlock_spin(&sched_lock); 631 ksegrp_free(newkg); 632 return (EPROCLIM); 633 } 634 ksegrp_link(newkg, p); 635 mtx_unlock_spin(&sched_lock); 636 } else { 637 newkg = kg; 638 } 639 640 /* 641 * Creating upcalls more than number of physical cpu does 642 * not help performance. 643 */ 644 if (newkg->kg_numupcalls >= ncpus) 645 return (EPROCLIM); 646 647 if (newkg->kg_numupcalls == 0) { 648 /* 649 * Initialize KSE group, optimized for MP. 650 * Create KSEs as many as physical cpus, this increases 651 * concurrent even if userland is not MP safe and can only run 652 * on single CPU (for early version of libpthread, it is true). 653 * In ideal world, every physical cpu should execute a thread. 654 * If there is enough KSEs, threads in kernel can be 655 * executed parallel on different cpus with full speed, 656 * Concurrent in kernel shouldn't be restricted by number of 657 * upcalls userland provides. 658 * Adding more upcall structures only increases concurrent 659 * in userland. 660 * Highest performance configuration is: 661 * N kses = N upcalls = N phyiscal cpus 662 */ 663 while (newkg->kg_kses < ncpus) { 664 newke = kse_alloc(); 665 bzero(&newke->ke_startzero, RANGEOF(struct kse, 666 ke_startzero, ke_endzero)); 667 #if 0 668 mtx_lock_spin(&sched_lock); 669 bcopy(&ke->ke_startcopy, &newke->ke_startcopy, 670 RANGEOF(struct kse, ke_startcopy, ke_endcopy)); 671 mtx_unlock_spin(&sched_lock); 672 #endif 673 mtx_lock_spin(&sched_lock); 674 kse_link(newke, newkg); 675 /* Add engine */ 676 kse_reassign(newke); 677 mtx_unlock_spin(&sched_lock); 678 } 679 } 680 newku = upcall_alloc(); 681 newku->ku_mailbox = uap->mbx; 682 newku->ku_func = mbx.km_func; 683 bcopy(&mbx.km_stack, &newku->ku_stack, sizeof(stack_t)); 684 685 /* For the first call this may not have been set */ 686 if (td->td_standin == NULL) 687 thread_alloc_spare(td, NULL); 688 689 mtx_lock_spin(&sched_lock); 690 if (newkg->kg_numupcalls >= ncpus) { 691 mtx_unlock_spin(&sched_lock); 692 upcall_free(newku); 693 return (EPROCLIM); 694 } 695 upcall_link(newku, newkg); 696 697 /* 698 * Each upcall structure has an owner thread, find which 699 * one owns it. 700 */ 701 if (uap->newgroup) { 702 /* 703 * Because new ksegrp hasn't thread, 704 * create an initial upcall thread to own it. 705 */ 706 thread_schedule_upcall(td, newku); 707 } else { 708 /* 709 * If current thread hasn't an upcall structure, 710 * just assign the upcall to it. 711 */ 712 if (td->td_upcall == NULL) { 713 newku->ku_owner = td; 714 td->td_upcall = newku; 715 } else { 716 /* 717 * Create a new upcall thread to own it. 718 */ 719 thread_schedule_upcall(td, newku); 720 } 721 } 722 mtx_unlock_spin(&sched_lock); 723 return (0); 724 } 725 726 /* 727 * Fill a ucontext_t with a thread's context information. 728 * 729 * This is an analogue to getcontext(3). 730 */ 731 void 732 thread_getcontext(struct thread *td, ucontext_t *uc) 733 { 734 735 /* 736 * XXX this is declared in a MD include file, i386/include/ucontext.h but 737 * is used in MI code. 738 */ 739 #ifdef __i386__ 740 get_mcontext(td, &uc->uc_mcontext); 741 #endif 742 uc->uc_sigmask = td->td_proc->p_sigmask; 743 } 744 745 /* 746 * Set a thread's context from a ucontext_t. 747 * 748 * This is an analogue to setcontext(3). 749 */ 750 int 751 thread_setcontext(struct thread *td, ucontext_t *uc) 752 { 753 int ret; 754 755 /* 756 * XXX this is declared in a MD include file, i386/include/ucontext.h but 757 * is used in MI code. 758 */ 759 #ifdef __i386__ 760 ret = set_mcontext(td, &uc->uc_mcontext); 761 #else 762 ret = ENOSYS; 763 #endif 764 if (ret == 0) { 765 SIG_CANTMASK(uc->uc_sigmask); 766 PROC_LOCK(td->td_proc); 767 td->td_proc->p_sigmask = uc->uc_sigmask; 768 PROC_UNLOCK(td->td_proc); 769 } 770 return (ret); 771 } 772 773 /* 774 * Initialize global thread allocation resources. 775 */ 776 void 777 threadinit(void) 778 { 779 780 #ifndef __ia64__ 781 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 782 thread_ctor, thread_dtor, thread_init, thread_fini, 783 UMA_ALIGN_CACHE, 0); 784 #else 785 /* 786 * XXX the ia64 kstack allocator is really lame and is at the mercy 787 * of contigmallloc(). This hackery is to pre-construct a whole 788 * pile of thread structures with associated kernel stacks early 789 * in the system startup while contigmalloc() still works. Once we 790 * have them, keep them. Sigh. 791 */ 792 thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(), 793 thread_ctor, thread_dtor, thread_init, thread_fini, 794 UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); 795 uma_prealloc(thread_zone, 512); /* XXX arbitary */ 796 #endif 797 ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), 798 NULL, NULL, ksegrp_init, NULL, 799 UMA_ALIGN_CACHE, 0); 800 kse_zone = uma_zcreate("KSE", sched_sizeof_kse(), 801 NULL, NULL, kse_init, NULL, 802 UMA_ALIGN_CACHE, 0); 803 upcall_zone = uma_zcreate("UPCALL", sizeof(struct kse_upcall), 804 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 805 } 806 807 /* 808 * Stash an embarasingly extra thread into the zombie thread queue. 809 */ 810 void 811 thread_stash(struct thread *td) 812 { 813 mtx_lock_spin(&kse_zombie_lock); 814 TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); 815 mtx_unlock_spin(&kse_zombie_lock); 816 } 817 818 /* 819 * Stash an embarasingly extra kse into the zombie kse queue. 820 */ 821 void 822 kse_stash(struct kse *ke) 823 { 824 mtx_lock_spin(&kse_zombie_lock); 825 TAILQ_INSERT_HEAD(&zombie_kses, ke, ke_procq); 826 mtx_unlock_spin(&kse_zombie_lock); 827 } 828 829 /* 830 * Stash an embarasingly extra upcall into the zombie upcall queue. 831 */ 832 833 void 834 upcall_stash(struct kse_upcall *ku) 835 { 836 mtx_lock_spin(&kse_zombie_lock); 837 TAILQ_INSERT_HEAD(&zombie_upcalls, ku, ku_link); 838 mtx_unlock_spin(&kse_zombie_lock); 839 } 840 841 /* 842 * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. 843 */ 844 void 845 ksegrp_stash(struct ksegrp *kg) 846 { 847 mtx_lock_spin(&kse_zombie_lock); 848 TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); 849 mtx_unlock_spin(&kse_zombie_lock); 850 } 851 852 /* 853 * Reap zombie kse resource. 854 */ 855 void 856 thread_reap(void) 857 { 858 struct thread *td_first, *td_next; 859 struct kse *ke_first, *ke_next; 860 struct ksegrp *kg_first, * kg_next; 861 struct kse_upcall *ku_first, *ku_next; 862 863 /* 864 * Don't even bother to lock if none at this instant, 865 * we really don't care about the next instant.. 866 */ 867 if ((!TAILQ_EMPTY(&zombie_threads)) 868 || (!TAILQ_EMPTY(&zombie_kses)) 869 || (!TAILQ_EMPTY(&zombie_ksegrps)) 870 || (!TAILQ_EMPTY(&zombie_upcalls))) { 871 mtx_lock_spin(&kse_zombie_lock); 872 td_first = TAILQ_FIRST(&zombie_threads); 873 ke_first = TAILQ_FIRST(&zombie_kses); 874 kg_first = TAILQ_FIRST(&zombie_ksegrps); 875 ku_first = TAILQ_FIRST(&zombie_upcalls); 876 if (td_first) 877 TAILQ_INIT(&zombie_threads); 878 if (ke_first) 879 TAILQ_INIT(&zombie_kses); 880 if (kg_first) 881 TAILQ_INIT(&zombie_ksegrps); 882 if (ku_first) 883 TAILQ_INIT(&zombie_upcalls); 884 mtx_unlock_spin(&kse_zombie_lock); 885 while (td_first) { 886 td_next = TAILQ_NEXT(td_first, td_runq); 887 if (td_first->td_ucred) 888 crfree(td_first->td_ucred); 889 thread_free(td_first); 890 td_first = td_next; 891 } 892 while (ke_first) { 893 ke_next = TAILQ_NEXT(ke_first, ke_procq); 894 kse_free(ke_first); 895 ke_first = ke_next; 896 } 897 while (kg_first) { 898 kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); 899 ksegrp_free(kg_first); 900 kg_first = kg_next; 901 } 902 while (ku_first) { 903 ku_next = TAILQ_NEXT(ku_first, ku_link); 904 upcall_free(ku_first); 905 ku_first = ku_next; 906 } 907 } 908 } 909 910 /* 911 * Allocate a ksegrp. 912 */ 913 struct ksegrp * 914 ksegrp_alloc(void) 915 { 916 return (uma_zalloc(ksegrp_zone, M_WAITOK)); 917 } 918 919 /* 920 * Allocate a kse. 921 */ 922 struct kse * 923 kse_alloc(void) 924 { 925 return (uma_zalloc(kse_zone, M_WAITOK)); 926 } 927 928 /* 929 * Allocate a thread. 930 */ 931 struct thread * 932 thread_alloc(void) 933 { 934 thread_reap(); /* check if any zombies to get */ 935 return (uma_zalloc(thread_zone, M_WAITOK)); 936 } 937 938 /* 939 * Deallocate a ksegrp. 940 */ 941 void 942 ksegrp_free(struct ksegrp *td) 943 { 944 uma_zfree(ksegrp_zone, td); 945 } 946 947 /* 948 * Deallocate a kse. 949 */ 950 void 951 kse_free(struct kse *td) 952 { 953 uma_zfree(kse_zone, td); 954 } 955 956 /* 957 * Deallocate a thread. 958 */ 959 void 960 thread_free(struct thread *td) 961 { 962 963 cpu_thread_clean(td); 964 uma_zfree(thread_zone, td); 965 } 966 967 /* 968 * Store the thread context in the UTS's mailbox. 969 * then add the mailbox at the head of a list we are building in user space. 970 * The list is anchored in the ksegrp structure. 971 */ 972 int 973 thread_export_context(struct thread *td) 974 { 975 struct proc *p; 976 struct ksegrp *kg; 977 uintptr_t mbx; 978 void *addr; 979 int error,temp; 980 ucontext_t uc; 981 982 p = td->td_proc; 983 kg = td->td_ksegrp; 984 985 /* Export the user/machine context. */ 986 addr = (void *)(&td->td_mailbox->tm_context); 987 error = copyin(addr, &uc, sizeof(ucontext_t)); 988 if (error) 989 goto bad; 990 991 thread_getcontext(td, &uc); 992 error = copyout(&uc, addr, sizeof(ucontext_t)); 993 if (error) 994 goto bad; 995 996 /* Exports clock ticks in kernel mode */ 997 addr = (caddr_t)(&td->td_mailbox->tm_sticks); 998 temp = fuword(addr) + td->td_usticks; 999 if (suword(addr, temp)) 1000 goto bad; 1001 1002 addr = (caddr_t)(&td->td_mailbox->tm_slices); 1003 temp = fuword(addr) - td->td_usticks; 1004 if (suword(addr, temp)) 1005 goto bad; 1006 1007 /* Get address in latest mbox of list pointer */ 1008 addr = (void *)(&td->td_mailbox->tm_next); 1009 /* 1010 * Put the saved address of the previous first 1011 * entry into this one 1012 */ 1013 for (;;) { 1014 mbx = (uintptr_t)kg->kg_completed; 1015 if (suword(addr, mbx)) { 1016 error = EFAULT; 1017 goto bad; 1018 } 1019 PROC_LOCK(p); 1020 if (mbx == (uintptr_t)kg->kg_completed) { 1021 kg->kg_completed = td->td_mailbox; 1022 /* 1023 * The thread context may be taken away by 1024 * other upcall threads when we unlock 1025 * process lock. it's no longer valid to 1026 * use it again in any other places. 1027 */ 1028 td->td_mailbox = NULL; 1029 PROC_UNLOCK(p); 1030 break; 1031 } 1032 PROC_UNLOCK(p); 1033 } 1034 td->td_usticks = 0; 1035 return (0); 1036 1037 bad: 1038 PROC_LOCK(p); 1039 psignal(p, SIGSEGV); 1040 PROC_UNLOCK(p); 1041 /* The mailbox is bad, don't use it */ 1042 td->td_mailbox = NULL; 1043 td->td_usticks = 0; 1044 return (error); 1045 } 1046 1047 /* 1048 * Take the list of completed mailboxes for this KSEGRP and put them on this 1049 * upcall's mailbox as it's the next one going up. 1050 */ 1051 static int 1052 thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) 1053 { 1054 struct proc *p = kg->kg_proc; 1055 void *addr; 1056 uintptr_t mbx; 1057 1058 addr = (void *)(&ku->ku_mailbox->km_completed); 1059 for (;;) { 1060 mbx = (uintptr_t)kg->kg_completed; 1061 if (suword(addr, mbx)) { 1062 PROC_LOCK(p); 1063 psignal(p, SIGSEGV); 1064 PROC_UNLOCK(p); 1065 return (EFAULT); 1066 } 1067 PROC_LOCK(p); 1068 if (mbx == (uintptr_t)kg->kg_completed) { 1069 kg->kg_completed = NULL; 1070 PROC_UNLOCK(p); 1071 break; 1072 } 1073 PROC_UNLOCK(p); 1074 } 1075 return (0); 1076 } 1077 1078 /* 1079 * This function should be called at statclock interrupt time 1080 */ 1081 int 1082 thread_statclock(int user) 1083 { 1084 struct thread *td = curthread; 1085 1086 if (td->td_ksegrp->kg_numupcalls == 0) 1087 return (-1); 1088 if (user) { 1089 /* Current always do via ast() */ 1090 mtx_lock_spin(&sched_lock); 1091 td->td_flags |= (TDF_USTATCLOCK|TDF_ASTPENDING); 1092 mtx_unlock_spin(&sched_lock); 1093 td->td_uuticks++; 1094 } else { 1095 if (td->td_mailbox != NULL) 1096 td->td_usticks++; 1097 else { 1098 /* XXXKSE 1099 * We will call thread_user_enter() for every 1100 * kernel entry in future, so if the thread mailbox 1101 * is NULL, it must be a UTS kernel, don't account 1102 * clock ticks for it. 1103 */ 1104 } 1105 } 1106 return (0); 1107 } 1108 1109 /* 1110 * Export state clock ticks for userland 1111 */ 1112 static int 1113 thread_update_usr_ticks(struct thread *td, int user) 1114 { 1115 struct proc *p = td->td_proc; 1116 struct kse_thr_mailbox *tmbx; 1117 struct kse_upcall *ku; 1118 caddr_t addr; 1119 uint uticks; 1120 int slices; 1121 1122 if ((ku = td->td_upcall) == NULL) 1123 return (-1); 1124 1125 tmbx = (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1126 if ((tmbx == NULL) || (tmbx == (void *)-1)) 1127 return (-1); 1128 if (user) { 1129 uticks = td->td_uuticks; 1130 td->td_uuticks = 0; 1131 addr = (caddr_t)&tmbx->tm_uticks; 1132 } else { 1133 uticks = td->td_usticks; 1134 td->td_usticks = 0; 1135 addr = (caddr_t)&tmbx->tm_sticks; 1136 } 1137 if (uticks) { 1138 if (suword(addr, uticks+fuword(addr))) { 1139 PROC_LOCK(p); 1140 psignal(p, SIGSEGV); 1141 PROC_UNLOCK(p); 1142 return (-2); 1143 } 1144 addr = (caddr_t)&tmbx->tm_slices; 1145 slices = (int)fuword(addr); 1146 if (slices > 0) { 1147 slices -= (int)uticks; 1148 if (suword(addr, slices)) { 1149 PROC_LOCK(p); 1150 psignal(p, SIGSEGV); 1151 PROC_UNLOCK(p); 1152 return (-2); 1153 } 1154 if (slices <= 0) { 1155 mtx_lock_spin(&sched_lock); 1156 td->td_upcall->ku_flags |= KUF_DOUPCALL; 1157 mtx_unlock_spin(&sched_lock); 1158 } 1159 } 1160 } 1161 return (0); 1162 } 1163 1164 /* 1165 * Discard the current thread and exit from its context. 1166 * 1167 * Because we can't free a thread while we're operating under its context, 1168 * push the current thread into our CPU's deadthread holder. This means 1169 * we needn't worry about someone else grabbing our context before we 1170 * do a cpu_throw(). 1171 */ 1172 void 1173 thread_exit(void) 1174 { 1175 struct thread *td; 1176 struct kse *ke; 1177 struct proc *p; 1178 struct ksegrp *kg; 1179 1180 td = curthread; 1181 kg = td->td_ksegrp; 1182 p = td->td_proc; 1183 ke = td->td_kse; 1184 1185 mtx_assert(&sched_lock, MA_OWNED); 1186 KASSERT(p != NULL, ("thread exiting without a process")); 1187 KASSERT(ke != NULL, ("thread exiting without a kse")); 1188 KASSERT(kg != NULL, ("thread exiting without a kse group")); 1189 PROC_LOCK_ASSERT(p, MA_OWNED); 1190 CTR1(KTR_PROC, "thread_exit: thread %p", td); 1191 KASSERT(!mtx_owned(&Giant), ("dying thread owns giant")); 1192 1193 if (td->td_standin != NULL) { 1194 thread_stash(td->td_standin); 1195 td->td_standin = NULL; 1196 } 1197 1198 cpu_thread_exit(td); /* XXXSMP */ 1199 1200 /* 1201 * The last thread is left attached to the process 1202 * So that the whole bundle gets recycled. Skip 1203 * all this stuff. 1204 */ 1205 if (p->p_numthreads > 1) { 1206 /* 1207 * Unlink this thread from its proc and the kseg. 1208 * In keeping with the other structs we probably should 1209 * have a thread_unlink() that does some of this but it 1210 * would only be called from here (I think) so it would 1211 * be a waste. (might be useful for proc_fini() as well.) 1212 */ 1213 TAILQ_REMOVE(&p->p_threads, td, td_plist); 1214 p->p_numthreads--; 1215 TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); 1216 kg->kg_numthreads--; 1217 if (p->p_maxthrwaits) 1218 wakeup(&p->p_numthreads); 1219 /* 1220 * The test below is NOT true if we are the 1221 * sole exiting thread. P_STOPPED_SNGL is unset 1222 * in exit1() after it is the only survivor. 1223 */ 1224 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1225 if (p->p_numthreads == p->p_suspcount) { 1226 thread_unsuspend_one(p->p_singlethread); 1227 } 1228 } 1229 1230 /* 1231 * Because each upcall structure has an owner thread, 1232 * owner thread exits only when process is in exiting 1233 * state, so upcall to userland is no longer needed, 1234 * deleting upcall structure is safe here. 1235 * So when all threads in a group is exited, all upcalls 1236 * in the group should be automatically freed. 1237 */ 1238 if (td->td_upcall) 1239 upcall_remove(td); 1240 1241 ke->ke_state = KES_UNQUEUED; 1242 ke->ke_thread = NULL; 1243 /* 1244 * Decide what to do with the KSE attached to this thread. 1245 */ 1246 if (ke->ke_flags & KEF_EXIT) 1247 kse_unlink(ke); 1248 else 1249 kse_reassign(ke); 1250 PROC_UNLOCK(p); 1251 td->td_kse = NULL; 1252 td->td_state = TDS_INACTIVE; 1253 td->td_proc = NULL; 1254 td->td_ksegrp = NULL; 1255 td->td_last_kse = NULL; 1256 PCPU_SET(deadthread, td); 1257 } else { 1258 PROC_UNLOCK(p); 1259 } 1260 cpu_throw(); 1261 /* NOTREACHED */ 1262 } 1263 1264 /* 1265 * Do any thread specific cleanups that may be needed in wait() 1266 * called with Giant held, proc and schedlock not held. 1267 */ 1268 void 1269 thread_wait(struct proc *p) 1270 { 1271 struct thread *td; 1272 1273 KASSERT((p->p_numthreads == 1), ("Muliple threads in wait1()")); 1274 KASSERT((p->p_numksegrps == 1), ("Muliple ksegrps in wait1()")); 1275 FOREACH_THREAD_IN_PROC(p, td) { 1276 if (td->td_standin != NULL) { 1277 thread_free(td->td_standin); 1278 td->td_standin = NULL; 1279 } 1280 cpu_thread_clean(td); 1281 } 1282 thread_reap(); /* check for zombie threads etc. */ 1283 } 1284 1285 /* 1286 * Link a thread to a process. 1287 * set up anything that needs to be initialized for it to 1288 * be used by the process. 1289 * 1290 * Note that we do not link to the proc's ucred here. 1291 * The thread is linked as if running but no KSE assigned. 1292 */ 1293 void 1294 thread_link(struct thread *td, struct ksegrp *kg) 1295 { 1296 struct proc *p; 1297 1298 p = kg->kg_proc; 1299 td->td_state = TDS_INACTIVE; 1300 td->td_proc = p; 1301 td->td_ksegrp = kg; 1302 td->td_last_kse = NULL; 1303 td->td_flags = 0; 1304 td->td_kse = NULL; 1305 1306 LIST_INIT(&td->td_contested); 1307 callout_init(&td->td_slpcallout, 1); 1308 TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); 1309 TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); 1310 p->p_numthreads++; 1311 kg->kg_numthreads++; 1312 } 1313 1314 /* 1315 * Purge a ksegrp resource. When a ksegrp is preparing to 1316 * exit, it calls this function. 1317 */ 1318 void 1319 kse_purge_group(struct thread *td) 1320 { 1321 struct ksegrp *kg; 1322 struct kse *ke; 1323 1324 kg = td->td_ksegrp; 1325 KASSERT(kg->kg_numthreads == 1, ("%s: bad thread number", __func__)); 1326 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1327 KASSERT(ke->ke_state == KES_IDLE, 1328 ("%s: wrong idle KSE state", __func__)); 1329 kse_unlink(ke); 1330 } 1331 KASSERT((kg->kg_kses == 1), 1332 ("%s: ksegrp still has %d KSEs", __func__, kg->kg_kses)); 1333 KASSERT((kg->kg_numupcalls == 0), 1334 ("%s: ksegrp still has %d upcall datas", 1335 __func__, kg->kg_numupcalls)); 1336 } 1337 1338 /* 1339 * Purge a process's KSE resource. When a process is preparing to 1340 * exit, it calls kse_purge to release any extra KSE resources in 1341 * the process. 1342 */ 1343 void 1344 kse_purge(struct proc *p, struct thread *td) 1345 { 1346 struct ksegrp *kg; 1347 struct kse *ke; 1348 1349 KASSERT(p->p_numthreads == 1, ("bad thread number")); 1350 mtx_lock_spin(&sched_lock); 1351 while ((kg = TAILQ_FIRST(&p->p_ksegrps)) != NULL) { 1352 TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); 1353 p->p_numksegrps--; 1354 /* 1355 * There is no ownership for KSE, after all threads 1356 * in the group exited, it is possible that some KSEs 1357 * were left in idle queue, gc them now. 1358 */ 1359 while ((ke = TAILQ_FIRST(&kg->kg_iq)) != NULL) { 1360 KASSERT(ke->ke_state == KES_IDLE, 1361 ("%s: wrong idle KSE state", __func__)); 1362 TAILQ_REMOVE(&kg->kg_iq, ke, ke_kgrlist); 1363 kg->kg_idle_kses--; 1364 TAILQ_REMOVE(&kg->kg_kseq, ke, ke_kglist); 1365 kg->kg_kses--; 1366 kse_stash(ke); 1367 } 1368 KASSERT(((kg->kg_kses == 0) && (kg != td->td_ksegrp)) || 1369 ((kg->kg_kses == 1) && (kg == td->td_ksegrp)), 1370 ("ksegrp has wrong kg_kses: %d", kg->kg_kses)); 1371 KASSERT((kg->kg_numupcalls == 0), 1372 ("%s: ksegrp still has %d upcall datas", 1373 __func__, kg->kg_numupcalls)); 1374 1375 if (kg != td->td_ksegrp) 1376 ksegrp_stash(kg); 1377 } 1378 TAILQ_INSERT_HEAD(&p->p_ksegrps, td->td_ksegrp, kg_ksegrp); 1379 p->p_numksegrps++; 1380 mtx_unlock_spin(&sched_lock); 1381 } 1382 1383 /* 1384 * This function is intended to be used to initialize a spare thread 1385 * for upcall. Initialize thread's large data area outside sched_lock 1386 * for thread_schedule_upcall(). 1387 */ 1388 void 1389 thread_alloc_spare(struct thread *td, struct thread *spare) 1390 { 1391 if (td->td_standin) 1392 return; 1393 if (spare == NULL) 1394 spare = thread_alloc(); 1395 td->td_standin = spare; 1396 bzero(&spare->td_startzero, 1397 (unsigned)RANGEOF(struct thread, td_startzero, td_endzero)); 1398 spare->td_proc = td->td_proc; 1399 /* Setup PCB and fork address */ 1400 cpu_set_upcall(spare, td->td_pcb); 1401 /* 1402 * XXXKSE do we really need this? (default values for the 1403 * frame). 1404 */ 1405 bcopy(td->td_frame, spare->td_frame, sizeof(struct trapframe)); 1406 spare->td_ucred = crhold(td->td_ucred); 1407 } 1408 1409 /* 1410 * Create a thread and schedule it for upcall on the KSE given. 1411 * Use our thread's standin so that we don't have to allocate one. 1412 */ 1413 struct thread * 1414 thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) 1415 { 1416 struct thread *td2; 1417 1418 mtx_assert(&sched_lock, MA_OWNED); 1419 1420 /* 1421 * Schedule an upcall thread on specified kse_upcall, 1422 * the kse_upcall must be free. 1423 * td must have a spare thread. 1424 */ 1425 KASSERT(ku->ku_owner == NULL, ("%s: upcall has owner", __func__)); 1426 if ((td2 = td->td_standin) != NULL) { 1427 td->td_standin = NULL; 1428 } else { 1429 panic("no reserve thread when scheduling an upcall"); 1430 return (NULL); 1431 } 1432 CTR3(KTR_PROC, "thread_schedule_upcall: thread %p (pid %d, %s)", 1433 td2, td->td_proc->p_pid, td->td_proc->p_comm); 1434 bcopy(&td->td_startcopy, &td2->td_startcopy, 1435 (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy)); 1436 thread_link(td2, ku->ku_ksegrp); 1437 /* Let the new thread become owner of the upcall */ 1438 ku->ku_owner = td2; 1439 td2->td_upcall = ku; 1440 td2->td_flags = TDF_UPCALLING; 1441 if (td->td_proc->p_sflag & PS_NEEDSIGCHK) 1442 td2->td_flags |= TDF_ASTPENDING; 1443 td2->td_kse = NULL; 1444 td2->td_state = TDS_CAN_RUN; 1445 td2->td_inhibitors = 0; 1446 setrunqueue(td2); 1447 return (td2); /* bogus.. should be a void function */ 1448 } 1449 1450 void 1451 thread_signal_add(struct thread *td, int sig) 1452 { 1453 struct kse_upcall *ku; 1454 struct proc *p; 1455 sigset_t ss; 1456 int error; 1457 1458 PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); 1459 td = curthread; 1460 ku = td->td_upcall; 1461 p = td->td_proc; 1462 1463 PROC_UNLOCK(p); 1464 error = copyin(&ku->ku_mailbox->km_sigscaught, &ss, sizeof(sigset_t)); 1465 if (error) 1466 goto error; 1467 1468 SIGADDSET(ss, sig); 1469 1470 error = copyout(&ss, &ku->ku_mailbox->km_sigscaught, sizeof(sigset_t)); 1471 if (error) 1472 goto error; 1473 1474 PROC_LOCK(p); 1475 return; 1476 error: 1477 PROC_LOCK(p); 1478 sigexit(td, SIGILL); 1479 } 1480 1481 1482 /* 1483 * Schedule an upcall to notify a KSE process recieved signals. 1484 * 1485 */ 1486 void 1487 thread_signal_upcall(struct thread *td) 1488 { 1489 mtx_lock_spin(&sched_lock); 1490 td->td_flags |= TDF_UPCALLING; 1491 mtx_unlock_spin(&sched_lock); 1492 1493 return; 1494 } 1495 1496 /* 1497 * Setup done on the thread when it enters the kernel. 1498 * XXXKSE Presently only for syscalls but eventually all kernel entries. 1499 */ 1500 void 1501 thread_user_enter(struct proc *p, struct thread *td) 1502 { 1503 struct ksegrp *kg; 1504 struct kse_upcall *ku; 1505 1506 kg = td->td_ksegrp; 1507 /* 1508 * First check that we shouldn't just abort. 1509 * But check if we are the single thread first! 1510 * XXX p_singlethread not locked, but should be safe. 1511 */ 1512 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1513 PROC_LOCK(p); 1514 mtx_lock_spin(&sched_lock); 1515 thread_exit(); 1516 /* NOTREACHED */ 1517 } 1518 1519 /* 1520 * If we are doing a syscall in a KSE environment, 1521 * note where our mailbox is. There is always the 1522 * possibility that we could do this lazily (in kse_reassign()), 1523 * but for now do it every time. 1524 */ 1525 kg = td->td_ksegrp; 1526 if (kg->kg_numupcalls) { 1527 ku = td->td_upcall; 1528 KASSERT(ku, ("%s: no upcall owned", __func__)); 1529 KASSERT((ku->ku_owner == td), ("%s: wrong owner", __func__)); 1530 td->td_mailbox = 1531 (void *)fuword((void *)&ku->ku_mailbox->km_curthread); 1532 if ((td->td_mailbox == NULL) || 1533 (td->td_mailbox == (void *)-1)) { 1534 /* Don't schedule upcall when blocked */ 1535 td->td_mailbox = NULL; 1536 mtx_lock_spin(&sched_lock); 1537 td->td_flags &= ~TDF_CAN_UNBIND; 1538 mtx_unlock_spin(&sched_lock); 1539 } else { 1540 if (td->td_standin == NULL) 1541 thread_alloc_spare(td, NULL); 1542 mtx_lock_spin(&sched_lock); 1543 td->td_flags |= TDF_CAN_UNBIND; 1544 mtx_unlock_spin(&sched_lock); 1545 } 1546 } 1547 } 1548 1549 /* 1550 * The extra work we go through if we are a threaded process when we 1551 * return to userland. 1552 * 1553 * If we are a KSE process and returning to user mode, check for 1554 * extra work to do before we return (e.g. for more syscalls 1555 * to complete first). If we were in a critical section, we should 1556 * just return to let it finish. Same if we were in the UTS (in 1557 * which case the mailbox's context's busy indicator will be set). 1558 * The only traps we suport will have set the mailbox. 1559 * We will clear it here. 1560 */ 1561 int 1562 thread_userret(struct thread *td, struct trapframe *frame) 1563 { 1564 int error = 0, upcalls; 1565 struct kse_upcall *ku; 1566 struct ksegrp *kg, *kg2; 1567 struct proc *p; 1568 struct timespec ts; 1569 1570 p = td->td_proc; 1571 kg = td->td_ksegrp; 1572 1573 /* Nothing to do with non-threaded group/process */ 1574 if (td->td_ksegrp->kg_numupcalls == 0) 1575 return (0); 1576 1577 /* 1578 * Stat clock interrupt hit in userland, it 1579 * is returning from interrupt, charge thread's 1580 * userland time for UTS. 1581 */ 1582 if (td->td_flags & TDF_USTATCLOCK) { 1583 thread_update_usr_ticks(td, 1); 1584 mtx_lock_spin(&sched_lock); 1585 td->td_flags &= ~TDF_USTATCLOCK; 1586 mtx_unlock_spin(&sched_lock); 1587 if (kg->kg_completed || 1588 (td->td_upcall->ku_flags & KUF_DOUPCALL)) 1589 thread_user_enter(p, td); 1590 } 1591 1592 /* 1593 * Optimisation: 1594 * This thread has not started any upcall. 1595 * If there is no work to report other than ourself, 1596 * then it can return direct to userland. 1597 */ 1598 if (TD_CAN_UNBIND(td)) { 1599 mtx_lock_spin(&sched_lock); 1600 td->td_flags &= ~TDF_CAN_UNBIND; 1601 mtx_unlock_spin(&sched_lock); 1602 if ((kg->kg_completed == NULL) && 1603 (td->td_upcall->ku_flags & KUF_DOUPCALL) == 0) { 1604 thread_update_usr_ticks(td, 0); 1605 if (!(kg->kg_completed || 1606 (td->td_upcall->ku_flags & KUF_DOUPCALL))) { 1607 td->td_mailbox = NULL; 1608 return (0); 1609 } 1610 } 1611 error = thread_export_context(td); 1612 if (error) { 1613 /* 1614 * Failing to do the KSE operation just defaults 1615 * back to synchonous operation, so just return from 1616 * the syscall. 1617 */ 1618 return (0); 1619 } 1620 /* 1621 * There is something to report, and we own an upcall 1622 * strucuture, we can go to userland. 1623 * Turn ourself into an upcall thread. 1624 */ 1625 mtx_lock_spin(&sched_lock); 1626 td->td_flags |= TDF_UPCALLING; 1627 mtx_unlock_spin(&sched_lock); 1628 } else if (td->td_mailbox) { 1629 error = thread_export_context(td); 1630 if (error) { 1631 PROC_LOCK(td->td_proc); 1632 mtx_lock_spin(&sched_lock); 1633 /* possibly upcall with error? */ 1634 } else { 1635 PROC_LOCK(td->td_proc); 1636 mtx_lock_spin(&sched_lock); 1637 /* 1638 * There are upcall threads waiting for 1639 * work to do, wake one of them up. 1640 * XXXKSE Maybe wake all of them up. 1641 */ 1642 if (kg->kg_upsleeps) 1643 wakeup_one(&kg->kg_completed); 1644 } 1645 thread_exit(); 1646 /* NOTREACHED */ 1647 } 1648 1649 KASSERT(TD_CAN_UNBIND(td) == 0, ("can unbind")); 1650 1651 if (p->p_numthreads > max_threads_per_proc) { 1652 max_threads_hits++; 1653 PROC_LOCK(p); 1654 while (p->p_numthreads > max_threads_per_proc) { 1655 if (P_SHOULDSTOP(p)) 1656 break; 1657 upcalls = 0; 1658 mtx_lock_spin(&sched_lock); 1659 FOREACH_KSEGRP_IN_PROC(p, kg2) { 1660 if (kg2->kg_numupcalls == 0) 1661 upcalls++; 1662 else 1663 upcalls += kg2->kg_numupcalls; 1664 } 1665 mtx_unlock_spin(&sched_lock); 1666 if (upcalls >= max_threads_per_proc) 1667 break; 1668 p->p_maxthrwaits++; 1669 msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, 1670 "maxthreads", NULL); 1671 p->p_maxthrwaits--; 1672 } 1673 PROC_UNLOCK(p); 1674 } 1675 1676 if (td->td_flags & TDF_UPCALLING) { 1677 ku = td->td_upcall; 1678 /* 1679 * There is no more work to do and we are going to ride 1680 * this thread up to userland as an upcall. 1681 * Do the last parts of the setup needed for the upcall. 1682 */ 1683 CTR3(KTR_PROC, "userret: upcall thread %p (pid %d, %s)", 1684 td, td->td_proc->p_pid, td->td_proc->p_comm); 1685 1686 /* 1687 * Set user context to the UTS. 1688 * Will use Giant in cpu_thread_clean() because it uses 1689 * kmem_free(kernel_map, ...) 1690 */ 1691 cpu_set_upcall_kse(td, ku); 1692 mtx_lock_spin(&sched_lock); 1693 td->td_flags &= ~TDF_UPCALLING; 1694 if (ku->ku_flags & KUF_DOUPCALL) 1695 ku->ku_flags &= ~KUF_DOUPCALL; 1696 mtx_unlock_spin(&sched_lock); 1697 1698 /* 1699 * Unhook the list of completed threads. 1700 * anything that completes after this gets to 1701 * come in next time. 1702 * Put the list of completed thread mailboxes on 1703 * this KSE's mailbox. 1704 */ 1705 error = thread_link_mboxes(kg, ku); 1706 if (error) 1707 goto out; 1708 1709 /* 1710 * Set state and clear the thread mailbox pointer. 1711 * From now on we are just a bound outgoing process. 1712 * **Problem** userret is often called several times. 1713 * it would be nice if this all happenned only on the first 1714 * time through. (the scan for extra work etc.) 1715 */ 1716 error = suword((caddr_t)&ku->ku_mailbox->km_curthread, 0); 1717 if (error) 1718 goto out; 1719 1720 /* Export current system time */ 1721 nanotime(&ts); 1722 error = copyout(&ts, (caddr_t)&ku->ku_mailbox->km_timeofday, 1723 sizeof(ts)); 1724 } 1725 1726 out: 1727 if (error) { 1728 /* 1729 * Things are going to be so screwed we should just kill 1730 * the process. 1731 * how do we do that? 1732 */ 1733 PROC_LOCK(td->td_proc); 1734 psignal(td->td_proc, SIGSEGV); 1735 PROC_UNLOCK(td->td_proc); 1736 } else { 1737 /* 1738 * Optimisation: 1739 * Ensure that we have a spare thread available, 1740 * for when we re-enter the kernel. 1741 */ 1742 if (td->td_standin == NULL) 1743 thread_alloc_spare(td, NULL); 1744 } 1745 1746 /* 1747 * Clear thread mailbox first, then clear system tick count. 1748 * The order is important because thread_statclock() use 1749 * mailbox pointer to see if it is an userland thread or 1750 * an UTS kernel thread. 1751 */ 1752 td->td_mailbox = NULL; 1753 td->td_usticks = 0; 1754 return (error); /* go sync */ 1755 } 1756 1757 /* 1758 * Enforce single-threading. 1759 * 1760 * Returns 1 if the caller must abort (another thread is waiting to 1761 * exit the process or similar). Process is locked! 1762 * Returns 0 when you are successfully the only thread running. 1763 * A process has successfully single threaded in the suspend mode when 1764 * There are no threads in user mode. Threads in the kernel must be 1765 * allowed to continue until they get to the user boundary. They may even 1766 * copy out their return values and data before suspending. They may however be 1767 * accellerated in reaching the user boundary as we will wake up 1768 * any sleeping threads that are interruptable. (PCATCH). 1769 */ 1770 int 1771 thread_single(int force_exit) 1772 { 1773 struct thread *td; 1774 struct thread *td2; 1775 struct proc *p; 1776 1777 td = curthread; 1778 p = td->td_proc; 1779 mtx_assert(&Giant, MA_OWNED); 1780 PROC_LOCK_ASSERT(p, MA_OWNED); 1781 KASSERT((td != NULL), ("curthread is NULL")); 1782 1783 if ((p->p_flag & P_THREADED) == 0) 1784 return (0); 1785 1786 /* Is someone already single threading? */ 1787 if (p->p_singlethread) 1788 return (1); 1789 1790 if (force_exit == SINGLE_EXIT) { 1791 p->p_flag |= P_SINGLE_EXIT; 1792 } else 1793 p->p_flag &= ~P_SINGLE_EXIT; 1794 p->p_flag |= P_STOPPED_SINGLE; 1795 p->p_singlethread = td; 1796 /* XXXKSE Which lock protects the below values? */ 1797 while ((p->p_numthreads - p->p_suspcount) != 1) { 1798 mtx_lock_spin(&sched_lock); 1799 FOREACH_THREAD_IN_PROC(p, td2) { 1800 if (td2 == td) 1801 continue; 1802 td->td_flags |= TDF_ASTPENDING; 1803 if (TD_IS_INHIBITED(td2)) { 1804 if (force_exit == SINGLE_EXIT) { 1805 if (TD_IS_SUSPENDED(td2)) { 1806 thread_unsuspend_one(td2); 1807 } 1808 if (TD_ON_SLEEPQ(td2) && 1809 (td2->td_flags & TDF_SINTR)) { 1810 if (td2->td_flags & TDF_CVWAITQ) 1811 cv_abort(td2); 1812 else 1813 abortsleep(td2); 1814 } 1815 } else { 1816 if (TD_IS_SUSPENDED(td2)) 1817 continue; 1818 /* 1819 * maybe other inhibitted states too? 1820 * XXXKSE Is it totally safe to 1821 * suspend a non-interruptable thread? 1822 */ 1823 if (td2->td_inhibitors & 1824 (TDI_SLEEPING | TDI_SWAPPED)) 1825 thread_suspend_one(td2); 1826 } 1827 } 1828 } 1829 /* 1830 * Maybe we suspended some threads.. was it enough? 1831 */ 1832 if ((p->p_numthreads - p->p_suspcount) == 1) { 1833 mtx_unlock_spin(&sched_lock); 1834 break; 1835 } 1836 1837 /* 1838 * Wake us up when everyone else has suspended. 1839 * In the mean time we suspend as well. 1840 */ 1841 thread_suspend_one(td); 1842 mtx_unlock(&Giant); 1843 PROC_UNLOCK(p); 1844 p->p_stats->p_ru.ru_nvcsw++; 1845 mi_switch(); 1846 mtx_unlock_spin(&sched_lock); 1847 mtx_lock(&Giant); 1848 PROC_LOCK(p); 1849 } 1850 if (force_exit == SINGLE_EXIT) { 1851 if (td->td_upcall) { 1852 mtx_lock_spin(&sched_lock); 1853 upcall_remove(td); 1854 mtx_unlock_spin(&sched_lock); 1855 } 1856 kse_purge(p, td); 1857 } 1858 return (0); 1859 } 1860 1861 /* 1862 * Called in from locations that can safely check to see 1863 * whether we have to suspend or at least throttle for a 1864 * single-thread event (e.g. fork). 1865 * 1866 * Such locations include userret(). 1867 * If the "return_instead" argument is non zero, the thread must be able to 1868 * accept 0 (caller may continue), or 1 (caller must abort) as a result. 1869 * 1870 * The 'return_instead' argument tells the function if it may do a 1871 * thread_exit() or suspend, or whether the caller must abort and back 1872 * out instead. 1873 * 1874 * If the thread that set the single_threading request has set the 1875 * P_SINGLE_EXIT bit in the process flags then this call will never return 1876 * if 'return_instead' is false, but will exit. 1877 * 1878 * P_SINGLE_EXIT | return_instead == 0| return_instead != 0 1879 *---------------+--------------------+--------------------- 1880 * 0 | returns 0 | returns 0 or 1 1881 * | when ST ends | immediatly 1882 *---------------+--------------------+--------------------- 1883 * 1 | thread exits | returns 1 1884 * | | immediatly 1885 * 0 = thread_exit() or suspension ok, 1886 * other = return error instead of stopping the thread. 1887 * 1888 * While a full suspension is under effect, even a single threading 1889 * thread would be suspended if it made this call (but it shouldn't). 1890 * This call should only be made from places where 1891 * thread_exit() would be safe as that may be the outcome unless 1892 * return_instead is set. 1893 */ 1894 int 1895 thread_suspend_check(int return_instead) 1896 { 1897 struct thread *td; 1898 struct proc *p; 1899 struct ksegrp *kg; 1900 1901 td = curthread; 1902 p = td->td_proc; 1903 kg = td->td_ksegrp; 1904 PROC_LOCK_ASSERT(p, MA_OWNED); 1905 while (P_SHOULDSTOP(p)) { 1906 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1907 KASSERT(p->p_singlethread != NULL, 1908 ("singlethread not set")); 1909 /* 1910 * The only suspension in action is a 1911 * single-threading. Single threader need not stop. 1912 * XXX Should be safe to access unlocked 1913 * as it can only be set to be true by us. 1914 */ 1915 if (p->p_singlethread == td) 1916 return (0); /* Exempt from stopping. */ 1917 } 1918 if (return_instead) 1919 return (1); 1920 1921 /* 1922 * If the process is waiting for us to exit, 1923 * this thread should just suicide. 1924 * Assumes that P_SINGLE_EXIT implies P_STOPPED_SINGLE. 1925 */ 1926 if ((p->p_flag & P_SINGLE_EXIT) && (p->p_singlethread != td)) { 1927 mtx_lock_spin(&sched_lock); 1928 while (mtx_owned(&Giant)) 1929 mtx_unlock(&Giant); 1930 thread_exit(); 1931 } 1932 1933 /* 1934 * When a thread suspends, it just 1935 * moves to the processes's suspend queue 1936 * and stays there. 1937 */ 1938 mtx_lock_spin(&sched_lock); 1939 if ((p->p_flag & P_STOPPED_SIG) && 1940 (p->p_suspcount+1 == p->p_numthreads)) { 1941 mtx_unlock_spin(&sched_lock); 1942 PROC_LOCK(p->p_pptr); 1943 if ((p->p_pptr->p_procsig->ps_flag & 1944 PS_NOCLDSTOP) == 0) { 1945 psignal(p->p_pptr, SIGCHLD); 1946 } 1947 PROC_UNLOCK(p->p_pptr); 1948 mtx_lock_spin(&sched_lock); 1949 } 1950 mtx_assert(&Giant, MA_NOTOWNED); 1951 thread_suspend_one(td); 1952 PROC_UNLOCK(p); 1953 if (P_SHOULDSTOP(p) == P_STOPPED_SINGLE) { 1954 if (p->p_numthreads == p->p_suspcount) { 1955 thread_unsuspend_one(p->p_singlethread); 1956 } 1957 } 1958 p->p_stats->p_ru.ru_nivcsw++; 1959 mi_switch(); 1960 mtx_unlock_spin(&sched_lock); 1961 PROC_LOCK(p); 1962 } 1963 return (0); 1964 } 1965 1966 void 1967 thread_suspend_one(struct thread *td) 1968 { 1969 struct proc *p = td->td_proc; 1970 1971 mtx_assert(&sched_lock, MA_OWNED); 1972 p->p_suspcount++; 1973 TD_SET_SUSPENDED(td); 1974 TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); 1975 /* 1976 * Hack: If we are suspending but are on the sleep queue 1977 * then we are in msleep or the cv equivalent. We 1978 * want to look like we have two Inhibitors. 1979 * May already be set.. doesn't matter. 1980 */ 1981 if (TD_ON_SLEEPQ(td)) 1982 TD_SET_SLEEPING(td); 1983 } 1984 1985 void 1986 thread_unsuspend_one(struct thread *td) 1987 { 1988 struct proc *p = td->td_proc; 1989 1990 mtx_assert(&sched_lock, MA_OWNED); 1991 TAILQ_REMOVE(&p->p_suspended, td, td_runq); 1992 TD_CLR_SUSPENDED(td); 1993 p->p_suspcount--; 1994 setrunnable(td); 1995 } 1996 1997 /* 1998 * Allow all threads blocked by single threading to continue running. 1999 */ 2000 void 2001 thread_unsuspend(struct proc *p) 2002 { 2003 struct thread *td; 2004 2005 mtx_assert(&sched_lock, MA_OWNED); 2006 PROC_LOCK_ASSERT(p, MA_OWNED); 2007 if (!P_SHOULDSTOP(p)) { 2008 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2009 thread_unsuspend_one(td); 2010 } 2011 } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && 2012 (p->p_numthreads == p->p_suspcount)) { 2013 /* 2014 * Stopping everything also did the job for the single 2015 * threading request. Now we've downgraded to single-threaded, 2016 * let it continue. 2017 */ 2018 thread_unsuspend_one(p->p_singlethread); 2019 } 2020 } 2021 2022 void 2023 thread_single_end(void) 2024 { 2025 struct thread *td; 2026 struct proc *p; 2027 2028 td = curthread; 2029 p = td->td_proc; 2030 PROC_LOCK_ASSERT(p, MA_OWNED); 2031 p->p_flag &= ~P_STOPPED_SINGLE; 2032 p->p_singlethread = NULL; 2033 /* 2034 * If there are other threads they mey now run, 2035 * unless of course there is a blanket 'stop order' 2036 * on the process. The single threader must be allowed 2037 * to continue however as this is a bad place to stop. 2038 */ 2039 if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { 2040 mtx_lock_spin(&sched_lock); 2041 while (( td = TAILQ_FIRST(&p->p_suspended))) { 2042 thread_unsuspend_one(td); 2043 } 2044 mtx_unlock_spin(&sched_lock); 2045 } 2046 } 2047 2048 2049