1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * Process switching routines. 30 */ 31 32#if defined(__lint) 33#include <sys/thread.h> 34#include <sys/systm.h> 35#include <sys/time.h> 36#else /* __lint */ 37#include "assym.h" 38#endif /* __lint */ 39 40#include <sys/asm_linkage.h> 41#include <sys/asm_misc.h> 42#include <sys/regset.h> 43#include <sys/privregs.h> 44#include <sys/stack.h> 45#include <sys/segments.h> 46 47/* 48 * resume(thread_id_t t); 49 * 50 * a thread can only run on one processor at a time. there 51 * exists a window on MPs where the current thread on one 52 * processor is capable of being dispatched by another processor. 53 * some overlap between outgoing and incoming threads can happen 54 * when they are the same thread. in this case where the threads 55 * are the same, resume() on one processor will spin on the incoming 56 * thread until resume() on the other processor has finished with 57 * the outgoing thread. 58 * 59 * The MMU context changes when the resuming thread resides in a different 60 * process. Kernel threads are known by resume to reside in process 0. 61 * The MMU context, therefore, only changes when resuming a thread in 62 * a process different from curproc. 63 * 64 * resume_from_intr() is called when the thread being resumed was not 65 * passivated by resume (e.g. was interrupted). This means that the 66 * resume lock is already held and that a restore context is not needed. 67 * Also, the MMU context is not changed on the resume in this case. 68 * 69 * resume_from_zombie() is the same as resume except the calling thread 70 * is a zombie and must be put on the deathrow list after the CPU is 71 * off the stack. 72 */ 73 74#if !defined(__lint) 75 76#if LWP_PCB_FPU != 0 77#error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work 78#endif /* LWP_PCB_FPU != 0 */ 79 80#endif /* !__lint */ 81 82#if defined(__amd64) 83 84/* 85 * Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15) 86 * 87 * The stack frame must be created before the save of %rsp so that tracebacks 88 * of swtch()ed-out processes show the process as having last called swtch(). 89 */ 90#define SAVE_REGS(thread_t, retaddr) \ 91 movq %rbp, T_RBP(thread_t); \ 92 movq %rbx, T_RBX(thread_t); \ 93 movq %r12, T_R12(thread_t); \ 94 movq %r13, T_R13(thread_t); \ 95 movq %r14, T_R14(thread_t); \ 96 movq %r15, T_R15(thread_t); \ 97 pushq %rbp; \ 98 movq %rsp, %rbp; \ 99 movq %rsp, T_SP(thread_t); \ 100 movq retaddr, T_PC(thread_t); \ 101 movq %rdi, %r12; \ 102 call __dtrace_probe___sched_off__cpu 103 104/* 105 * Restore non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15) 106 * 107 * We load up %rsp from the label_t as part of the context switch, so 108 * we don't repeat that here. 109 * 110 * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t 111 * already has the effect of putting the stack back the way it was when 112 * we came in. 113 */ 114#define RESTORE_REGS(scratch_reg) \ 115 movq %gs:CPU_THREAD, scratch_reg; \ 116 movq T_RBP(scratch_reg), %rbp; \ 117 movq T_RBX(scratch_reg), %rbx; \ 118 movq T_R12(scratch_reg), %r12; \ 119 movq T_R13(scratch_reg), %r13; \ 120 movq T_R14(scratch_reg), %r14; \ 121 movq T_R15(scratch_reg), %r15 122 123/* 124 * Get pointer to a thread's hat structure 125 */ 126#define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \ 127 movq T_PROCP(thread_t), hatp; \ 128 movq P_AS(hatp), scratch_reg; \ 129 movq A_HAT(scratch_reg), hatp 130 131#define TSC_READ() \ 132 call tsc_read; \ 133 movq %rax, %r14; 134 135/* 136 * If we are resuming an interrupt thread, store a timestamp in the thread 137 * structure. If an interrupt occurs between tsc_read() and its subsequent 138 * store, the timestamp will be stale by the time it is stored. We can detect 139 * this by doing a compare-and-swap on the thread's timestamp, since any 140 * interrupt occurring in this window will put a new timestamp in the thread's 141 * t_intr_start field. 142 */ 143#define STORE_INTR_START(thread_t) \ 144 testw $T_INTR_THREAD, T_FLAGS(thread_t); \ 145 jz 1f; \ 1460: \ 147 TSC_READ(); \ 148 movq T_INTR_START(thread_t), %rax; \ 149 cmpxchgq %r14, T_INTR_START(thread_t); \ 150 jnz 0b; \ 1511: 152 153#elif defined (__i386) 154 155/* 156 * Save non-volatile registers (%ebp, %esi, %edi and %ebx) 157 * 158 * The stack frame must be created before the save of %esp so that tracebacks 159 * of swtch()ed-out processes show the process as having last called swtch(). 160 */ 161#define SAVE_REGS(thread_t, retaddr) \ 162 movl %ebp, T_EBP(thread_t); \ 163 movl %ebx, T_EBX(thread_t); \ 164 movl %esi, T_ESI(thread_t); \ 165 movl %edi, T_EDI(thread_t); \ 166 pushl %ebp; \ 167 movl %esp, %ebp; \ 168 movl %esp, T_SP(thread_t); \ 169 movl retaddr, T_PC(thread_t); \ 170 movl 8(%ebp), %edi; \ 171 pushl %edi; \ 172 call __dtrace_probe___sched_off__cpu; \ 173 addl $CLONGSIZE, %esp 174 175/* 176 * Restore non-volatile registers (%ebp, %esi, %edi and %ebx) 177 * 178 * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t 179 * already has the effect of putting the stack back the way it was when 180 * we came in. 181 */ 182#define RESTORE_REGS(scratch_reg) \ 183 movl %gs:CPU_THREAD, scratch_reg; \ 184 movl T_EBP(scratch_reg), %ebp; \ 185 movl T_EBX(scratch_reg), %ebx; \ 186 movl T_ESI(scratch_reg), %esi; \ 187 movl T_EDI(scratch_reg), %edi 188 189/* 190 * Get pointer to a thread's hat structure 191 */ 192#define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \ 193 movl T_PROCP(thread_t), hatp; \ 194 movl P_AS(hatp), scratch_reg; \ 195 movl A_HAT(scratch_reg), hatp 196 197/* 198 * If we are resuming an interrupt thread, store a timestamp in the thread 199 * structure. If an interrupt occurs between tsc_read() and its subsequent 200 * store, the timestamp will be stale by the time it is stored. We can detect 201 * this by doing a compare-and-swap on the thread's timestamp, since any 202 * interrupt occurring in this window will put a new timestamp in the thread's 203 * t_intr_start field. 204 */ 205#define STORE_INTR_START(thread_t) \ 206 testw $T_INTR_THREAD, T_FLAGS(thread_t); \ 207 jz 1f; \ 208 pushl %ecx; \ 2090: \ 210 pushl T_INTR_START(thread_t); \ 211 pushl T_INTR_START+4(thread_t); \ 212 call tsc_read; \ 213 movl %eax, %ebx; \ 214 movl %edx, %ecx; \ 215 popl %edx; \ 216 popl %eax; \ 217 cmpxchg8b T_INTR_START(thread_t); \ 218 jnz 0b; \ 219 popl %ecx; \ 2201: 221 222#endif /* __amd64 */ 223 224#if defined(__lint) 225 226/* ARGSUSED */ 227void 228resume(kthread_t *t) 229{} 230 231#else /* __lint */ 232 233#if defined(__amd64) 234 235 ENTRY(resume) 236 movq %gs:CPU_THREAD, %rax 237 leaq resume_return(%rip), %r11 238 239 /* 240 * Save non-volatile registers, and set return address for current 241 * thread to resume_return. 242 * 243 * %r12 = t (new thread) when done 244 */ 245 SAVE_REGS(%rax, %r11) 246 247 LOADCPU(%r15) /* %r15 = CPU */ 248 movq CPU_THREAD(%r15), %r13 /* %r13 = curthread */ 249 250 /* 251 * Call savectx if thread has installed context ops. 252 * 253 * Note that if we have floating point context, the save op 254 * (either fpsave_begin or fpxsave_begin) will issue the 255 * async save instruction (fnsave or fxsave respectively) 256 * that we fwait for below. 257 */ 258 cmpq $0, T_CTX(%r13) /* should current thread savectx? */ 259 je .nosavectx /* skip call when zero */ 260 261 movq %r13, %rdi /* arg = thread pointer */ 262 call savectx /* call ctx ops */ 263.nosavectx: 264 265 /* 266 * Call savepctx if process has installed context ops. 267 */ 268 movq T_PROCP(%r13), %r14 /* %r14 = proc */ 269 cmpq $0, P_PCTX(%r14) /* should current thread savectx? */ 270 je .nosavepctx /* skip call when zero */ 271 272 movq %r14, %rdi /* arg = proc pointer */ 273 call savepctx /* call ctx ops */ 274.nosavepctx: 275 276 /* 277 * Temporarily switch to the idle thread's stack 278 */ 279 movq CPU_IDLE_THREAD(%r15), %rax /* idle thread pointer */ 280 281 /* 282 * Set the idle thread as the current thread 283 */ 284 movq T_SP(%rax), %rsp /* It is safe to set rsp */ 285 movq %rax, CPU_THREAD(%r15) 286 287 /* 288 * Switch in the hat context for the new thread 289 * 290 */ 291 GET_THREAD_HATP(%rdi, %r12, %r11) 292 call hat_switch 293 294 /* 295 * Clear and unlock previous thread's t_lock 296 * to allow it to be dispatched by another processor. 297 */ 298 movb $0, T_LOCK(%r13) 299 300 /* 301 * IMPORTANT: Registers at this point must be: 302 * %r12 = new thread 303 * 304 * Here we are in the idle thread, have dropped the old thread. 305 */ 306 ALTENTRY(_resume_from_idle) 307 /* 308 * spin until dispatched thread's mutex has 309 * been unlocked. this mutex is unlocked when 310 * it becomes safe for the thread to run. 311 */ 312.lock_thread_mutex: 313 lock 314 btsl $0, T_LOCK(%r12) /* attempt to lock new thread's mutex */ 315 jnc .thread_mutex_locked /* got it */ 316 317.spin_thread_mutex: 318 pause 319 cmpb $0, T_LOCK(%r12) /* check mutex status */ 320 jz .lock_thread_mutex /* clear, retry lock */ 321 jmp .spin_thread_mutex /* still locked, spin... */ 322 323.thread_mutex_locked: 324 /* 325 * Fix CPU structure to indicate new running thread. 326 * Set pointer in new thread to the CPU structure. 327 */ 328 LOADCPU(%r13) /* load current CPU pointer */ 329 cmpq %r13, T_CPU(%r12) 330 je .setup_cpu 331 332 /* cp->cpu_stats.sys.cpumigrate++ */ 333 incq CPU_STATS_SYS_CPUMIGRATE(%r13) 334 movq %r13, T_CPU(%r12) /* set new thread's CPU pointer */ 335 336.setup_cpu: 337 /* 338 * Setup rsp0 (kernel stack) in TSS to curthread's stack. 339 * (Note: Since we don't have saved 'regs' structure for all 340 * the threads we can't easily determine if we need to 341 * change rsp0. So, we simply change the rsp0 to bottom 342 * of the thread stack and it will work for all cases.) 343 * 344 * XX64 - Is this correct? 345 */ 346 movq CPU_TSS(%r13), %r14 347 movq T_STACK(%r12), %rax 348 addq $REGSIZE+MINFRAME, %rax /* to the bottom of thread stack */ 349#if !defined(__xpv) 350 movq %rax, TSS_RSP0(%r14) 351#else 352 movl $KDS_SEL, %edi 353 movq %rax, %rsi 354 call HYPERVISOR_stack_switch 355#endif /* __xpv */ 356 357 movq %r12, CPU_THREAD(%r13) /* set CPU's thread pointer */ 358 xorl %ebp, %ebp /* make $<threadlist behave better */ 359 movq T_LWP(%r12), %rax /* set associated lwp to */ 360 movq %rax, CPU_LWP(%r13) /* CPU's lwp ptr */ 361 362 movq T_SP(%r12), %rsp /* switch to outgoing thread's stack */ 363 movq T_PC(%r12), %r13 /* saved return addr */ 364 365 /* 366 * Call restorectx if context ops have been installed. 367 */ 368 cmpq $0, T_CTX(%r12) /* should resumed thread restorectx? */ 369 jz .norestorectx /* skip call when zero */ 370 movq %r12, %rdi /* arg = thread pointer */ 371 call restorectx /* call ctx ops */ 372.norestorectx: 373 374 /* 375 * Call restorepctx if context ops have been installed for the proc. 376 */ 377 movq T_PROCP(%r12), %rcx 378 cmpq $0, P_PCTX(%rcx) 379 jz .norestorepctx 380 movq %rcx, %rdi 381 call restorepctx 382.norestorepctx: 383 384 STORE_INTR_START(%r12) 385 386 /* 387 * Restore non-volatile registers, then have spl0 return to the 388 * resuming thread's PC after first setting the priority as low as 389 * possible and blocking all interrupt threads that may be active. 390 */ 391 movq %r13, %rax /* save return address */ 392 RESTORE_REGS(%r11) 393 pushq %rax /* push return address for spl0() */ 394 call __dtrace_probe___sched_on__cpu 395 jmp spl0 396 397resume_return: 398 /* 399 * Remove stack frame created in SAVE_REGS() 400 */ 401 addq $CLONGSIZE, %rsp 402 ret 403 SET_SIZE(_resume_from_idle) 404 SET_SIZE(resume) 405 406#elif defined (__i386) 407 408 ENTRY(resume) 409 movl %gs:CPU_THREAD, %eax 410 movl $resume_return, %ecx 411 412 /* 413 * Save non-volatile registers, and set return address for current 414 * thread to resume_return. 415 * 416 * %edi = t (new thread) when done. 417 */ 418 SAVE_REGS(%eax, %ecx) 419 420 LOADCPU(%ebx) /* %ebx = CPU */ 421 movl CPU_THREAD(%ebx), %esi /* %esi = curthread */ 422 423#ifdef DEBUG 424 call assert_ints_enabled /* panics if we are cli'd */ 425#endif 426 /* 427 * Call savectx if thread has installed context ops. 428 * 429 * Note that if we have floating point context, the save op 430 * (either fpsave_begin or fpxsave_begin) will issue the 431 * async save instruction (fnsave or fxsave respectively) 432 * that we fwait for below. 433 */ 434 movl T_CTX(%esi), %eax /* should current thread savectx? */ 435 testl %eax, %eax 436 jz .nosavectx /* skip call when zero */ 437 pushl %esi /* arg = thread pointer */ 438 call savectx /* call ctx ops */ 439 addl $4, %esp /* restore stack pointer */ 440.nosavectx: 441 442 /* 443 * Call savepctx if process has installed context ops. 444 */ 445 movl T_PROCP(%esi), %eax /* %eax = proc */ 446 cmpl $0, P_PCTX(%eax) /* should current thread savectx? */ 447 je .nosavepctx /* skip call when zero */ 448 pushl %eax /* arg = proc pointer */ 449 call savepctx /* call ctx ops */ 450 addl $4, %esp 451.nosavepctx: 452 453 /* 454 * Temporarily switch to the idle thread's stack 455 */ 456 movl CPU_IDLE_THREAD(%ebx), %eax /* idle thread pointer */ 457 458 /* 459 * Set the idle thread as the current thread 460 */ 461 movl T_SP(%eax), %esp /* It is safe to set esp */ 462 movl %eax, CPU_THREAD(%ebx) 463 464 /* switch in the hat context for the new thread */ 465 GET_THREAD_HATP(%ecx, %edi, %ecx) 466 pushl %ecx 467 call hat_switch 468 addl $4, %esp 469 470 /* 471 * Clear and unlock previous thread's t_lock 472 * to allow it to be dispatched by another processor. 473 */ 474 movb $0, T_LOCK(%esi) 475 476 /* 477 * IMPORTANT: Registers at this point must be: 478 * %edi = new thread 479 * 480 * Here we are in the idle thread, have dropped the old thread. 481 */ 482 ALTENTRY(_resume_from_idle) 483 /* 484 * spin until dispatched thread's mutex has 485 * been unlocked. this mutex is unlocked when 486 * it becomes safe for the thread to run. 487 */ 488.L4: 489 lock 490 btsl $0, T_LOCK(%edi) /* lock new thread's mutex */ 491 jc .L4_2 /* lock did not succeed */ 492 493 /* 494 * Fix CPU structure to indicate new running thread. 495 * Set pointer in new thread to the CPU structure. 496 */ 497 LOADCPU(%esi) /* load current CPU pointer */ 498 movl T_STACK(%edi), %eax /* here to use v pipeline of */ 499 /* Pentium. Used few lines below */ 500 cmpl %esi, T_CPU(%edi) 501 jne .L5_2 502.L5_1: 503 /* 504 * Setup esp0 (kernel stack) in TSS to curthread's stack. 505 * (Note: Since we don't have saved 'regs' structure for all 506 * the threads we can't easily determine if we need to 507 * change esp0. So, we simply change the esp0 to bottom 508 * of the thread stack and it will work for all cases.) 509 */ 510 movl CPU_TSS(%esi), %ecx 511 addl $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */ 512#if !defined(__xpv) 513 movl %eax, TSS_ESP0(%ecx) 514#else 515 pushl %eax 516 pushl $KDS_SEL 517 call HYPERVISOR_stack_switch 518 addl $8, %esp 519#endif /* __xpv */ 520 521 movl %edi, CPU_THREAD(%esi) /* set CPU's thread pointer */ 522 xorl %ebp, %ebp /* make $<threadlist behave better */ 523 movl T_LWP(%edi), %eax /* set associated lwp to */ 524 movl %eax, CPU_LWP(%esi) /* CPU's lwp ptr */ 525 526 movl T_SP(%edi), %esp /* switch to outgoing thread's stack */ 527 movl T_PC(%edi), %esi /* saved return addr */ 528 529 /* 530 * Call restorectx if context ops have been installed. 531 */ 532 movl T_CTX(%edi), %eax /* should resumed thread restorectx? */ 533 testl %eax, %eax 534 jz .norestorectx /* skip call when zero */ 535 pushl %edi /* arg = thread pointer */ 536 call restorectx /* call ctx ops */ 537 addl $4, %esp /* restore stack pointer */ 538.norestorectx: 539 540 /* 541 * Call restorepctx if context ops have been installed for the proc. 542 */ 543 movl T_PROCP(%edi), %eax 544 cmpl $0, P_PCTX(%eax) 545 je .norestorepctx 546 pushl %eax /* arg = proc pointer */ 547 call restorepctx 548 addl $4, %esp /* restore stack pointer */ 549.norestorepctx: 550 551 STORE_INTR_START(%edi) 552 553 /* 554 * Restore non-volatile registers, then have spl0 return to the 555 * resuming thread's PC after first setting the priority as low as 556 * possible and blocking all interrupt threads that may be active. 557 */ 558 movl %esi, %eax /* save return address */ 559 RESTORE_REGS(%ecx) 560 pushl %eax /* push return address for spl0() */ 561 call __dtrace_probe___sched_on__cpu 562 jmp spl0 563 564resume_return: 565 /* 566 * Remove stack frame created in SAVE_REGS() 567 */ 568 addl $CLONGSIZE, %esp 569 ret 570 571.L4_2: 572 pause 573 cmpb $0, T_LOCK(%edi) 574 je .L4 575 jmp .L4_2 576 577.L5_2: 578 /* cp->cpu_stats.sys.cpumigrate++ */ 579 addl $1, CPU_STATS_SYS_CPUMIGRATE(%esi) 580 adcl $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi) 581 movl %esi, T_CPU(%edi) /* set new thread's CPU pointer */ 582 jmp .L5_1 583 584 SET_SIZE(_resume_from_idle) 585 SET_SIZE(resume) 586 587#endif /* __amd64 */ 588#endif /* __lint */ 589 590#if defined(__lint) 591 592/* ARGSUSED */ 593void 594resume_from_zombie(kthread_t *t) 595{} 596 597#else /* __lint */ 598 599#if defined(__amd64) 600 601 ENTRY(resume_from_zombie) 602 movq %gs:CPU_THREAD, %rax 603 leaq resume_from_zombie_return(%rip), %r11 604 605 /* 606 * Save non-volatile registers, and set return address for current 607 * thread to resume_from_zombie_return. 608 * 609 * %r12 = t (new thread) when done 610 */ 611 SAVE_REGS(%rax, %r11) 612 613 movq %gs:CPU_THREAD, %r13 /* %r13 = curthread */ 614 615 /* clean up the fp unit. It might be left enabled */ 616 617#if defined(__xpv) /* XXPV XXtclayton */ 618 /* 619 * Remove this after bringup. 620 * (Too many #gp's for an instrumented hypervisor.) 621 */ 622 STTS(%rax) 623#else 624 movq %cr0, %rax 625 testq $CR0_TS, %rax 626 jnz .zfpu_disabled /* if TS already set, nothing to do */ 627 fninit /* init fpu & discard pending error */ 628 orq $CR0_TS, %rax 629 movq %rax, %cr0 630.zfpu_disabled: 631 632#endif /* __xpv */ 633 634 /* 635 * Temporarily switch to the idle thread's stack so that the zombie 636 * thread's stack can be reclaimed by the reaper. 637 */ 638 movq %gs:CPU_IDLE_THREAD, %rax /* idle thread pointer */ 639 movq T_SP(%rax), %rsp /* get onto idle thread stack */ 640 641 /* 642 * Sigh. If the idle thread has never run thread_start() 643 * then t_sp is mis-aligned by thread_load(). 644 */ 645 andq $_BITNOT(STACK_ALIGN-1), %rsp 646 647 /* 648 * Set the idle thread as the current thread. 649 */ 650 movq %rax, %gs:CPU_THREAD 651 652 /* switch in the hat context for the new thread */ 653 GET_THREAD_HATP(%rdi, %r12, %r11) 654 call hat_switch 655 656 /* 657 * Put the zombie on death-row. 658 */ 659 movq %r13, %rdi 660 call reapq_add 661 662 jmp _resume_from_idle /* finish job of resume */ 663 664resume_from_zombie_return: 665 RESTORE_REGS(%r11) /* restore non-volatile registers */ 666 call __dtrace_probe___sched_on__cpu 667 668 /* 669 * Remove stack frame created in SAVE_REGS() 670 */ 671 addq $CLONGSIZE, %rsp 672 ret 673 SET_SIZE(resume_from_zombie) 674 675#elif defined (__i386) 676 677 ENTRY(resume_from_zombie) 678 movl %gs:CPU_THREAD, %eax 679 movl $resume_from_zombie_return, %ecx 680 681 /* 682 * Save non-volatile registers, and set return address for current 683 * thread to resume_from_zombie_return. 684 * 685 * %edi = t (new thread) when done. 686 */ 687 SAVE_REGS(%eax, %ecx) 688 689#ifdef DEBUG 690 call assert_ints_enabled /* panics if we are cli'd */ 691#endif 692 movl %gs:CPU_THREAD, %esi /* %esi = curthread */ 693 694 /* clean up the fp unit. It might be left enabled */ 695 696 movl %cr0, %eax 697 testl $CR0_TS, %eax 698 jnz .zfpu_disabled /* if TS already set, nothing to do */ 699 fninit /* init fpu & discard pending error */ 700 orl $CR0_TS, %eax 701 movl %eax, %cr0 702.zfpu_disabled: 703 704 /* 705 * Temporarily switch to the idle thread's stack so that the zombie 706 * thread's stack can be reclaimed by the reaper. 707 */ 708 movl %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */ 709 movl T_SP(%eax), %esp /* get onto idle thread stack */ 710 711 /* 712 * Set the idle thread as the current thread. 713 */ 714 movl %eax, %gs:CPU_THREAD 715 716 /* 717 * switch in the hat context for the new thread 718 */ 719 GET_THREAD_HATP(%ecx, %edi, %ecx) 720 pushl %ecx 721 call hat_switch 722 addl $4, %esp 723 724 /* 725 * Put the zombie on death-row. 726 */ 727 pushl %esi 728 call reapq_add 729 addl $4, %esp 730 jmp _resume_from_idle /* finish job of resume */ 731 732resume_from_zombie_return: 733 RESTORE_REGS(%ecx) /* restore non-volatile registers */ 734 call __dtrace_probe___sched_on__cpu 735 736 /* 737 * Remove stack frame created in SAVE_REGS() 738 */ 739 addl $CLONGSIZE, %esp 740 ret 741 SET_SIZE(resume_from_zombie) 742 743#endif /* __amd64 */ 744#endif /* __lint */ 745 746#if defined(__lint) 747 748/* ARGSUSED */ 749void 750resume_from_intr(kthread_t *t) 751{} 752 753#else /* __lint */ 754 755#if defined(__amd64) 756 757 ENTRY(resume_from_intr) 758 movq %gs:CPU_THREAD, %rax 759 leaq resume_from_intr_return(%rip), %r11 760 761 /* 762 * Save non-volatile registers, and set return address for current 763 * thread to resume_from_intr_return. 764 * 765 * %r12 = t (new thread) when done 766 */ 767 SAVE_REGS(%rax, %r11) 768 769 movq %gs:CPU_THREAD, %r13 /* %r13 = curthread */ 770 movq %r12, %gs:CPU_THREAD /* set CPU's thread pointer */ 771 movq T_SP(%r12), %rsp /* restore resuming thread's sp */ 772 xorl %ebp, %ebp /* make $<threadlist behave better */ 773 774 /* 775 * Unlock outgoing thread's mutex dispatched by another processor. 776 */ 777 xorl %eax, %eax 778 xchgb %al, T_LOCK(%r13) 779 780 STORE_INTR_START(%r12) 781 782 /* 783 * Restore non-volatile registers, then have spl0 return to the 784 * resuming thread's PC after first setting the priority as low as 785 * possible and blocking all interrupt threads that may be active. 786 */ 787 movq T_PC(%r12), %rax /* saved return addr */ 788 RESTORE_REGS(%r11); 789 pushq %rax /* push return address for spl0() */ 790 call __dtrace_probe___sched_on__cpu 791 jmp spl0 792 793resume_from_intr_return: 794 /* 795 * Remove stack frame created in SAVE_REGS() 796 */ 797 addq $CLONGSIZE, %rsp 798 ret 799 SET_SIZE(resume_from_intr) 800 801#elif defined (__i386) 802 803 ENTRY(resume_from_intr) 804 movl %gs:CPU_THREAD, %eax 805 movl $resume_from_intr_return, %ecx 806 807 /* 808 * Save non-volatile registers, and set return address for current 809 * thread to resume_return. 810 * 811 * %edi = t (new thread) when done. 812 */ 813 SAVE_REGS(%eax, %ecx) 814 815#ifdef DEBUG 816 call assert_ints_enabled /* panics if we are cli'd */ 817#endif 818 movl %gs:CPU_THREAD, %esi /* %esi = curthread */ 819 movl %edi, %gs:CPU_THREAD /* set CPU's thread pointer */ 820 movl T_SP(%edi), %esp /* restore resuming thread's sp */ 821 xorl %ebp, %ebp /* make $<threadlist behave better */ 822 823 /* 824 * Unlock outgoing thread's mutex dispatched by another processor. 825 */ 826 xorl %eax,%eax 827 xchgb %al, T_LOCK(%esi) 828 829 STORE_INTR_START(%edi) 830 831 /* 832 * Restore non-volatile registers, then have spl0 return to the 833 * resuming thread's PC after first setting the priority as low as 834 * possible and blocking all interrupt threads that may be active. 835 */ 836 movl T_PC(%edi), %eax /* saved return addr */ 837 RESTORE_REGS(%ecx) 838 pushl %eax /* push return address for spl0() */ 839 call __dtrace_probe___sched_on__cpu 840 jmp spl0 841 842resume_from_intr_return: 843 /* 844 * Remove stack frame created in SAVE_REGS() 845 */ 846 addl $CLONGSIZE, %esp 847 ret 848 SET_SIZE(resume_from_intr) 849 850#endif /* __amd64 */ 851#endif /* __lint */ 852 853#if defined(__lint) 854 855void 856thread_start(void) 857{} 858 859#else /* __lint */ 860 861#if defined(__amd64) 862 863 ENTRY(thread_start) 864 popq %rax /* start() */ 865 popq %rdi /* arg */ 866 popq %rsi /* len */ 867 movq %rsp, %rbp 868 call *%rax 869 call thread_exit /* destroy thread if it returns. */ 870 /*NOTREACHED*/ 871 SET_SIZE(thread_start) 872 873#elif defined(__i386) 874 875 ENTRY(thread_start) 876 popl %eax 877 movl %esp, %ebp 878 addl $8, %ebp 879 call *%eax 880 addl $8, %esp 881 call thread_exit /* destroy thread if it returns. */ 882 /*NOTREACHED*/ 883 SET_SIZE(thread_start) 884 885#endif /* __i386 */ 886 887#endif /* __lint */ 888