1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * Process switching routines. 30 */ 31 32#if !defined(lint) 33#include "assym.h" 34#else /* lint */ 35#include <sys/thread.h> 36#endif /* lint */ 37 38#include <sys/param.h> 39#include <sys/asm_linkage.h> 40#include <sys/mmu.h> 41#include <sys/pcb.h> 42#include <sys/machthread.h> 43#include <sys/privregs.h> 44#include <sys/vtrace.h> 45#include <vm/hat_sfmmu.h> 46 47/* 48 * resume(kthread_id_t) 49 * 50 * a thread can only run on one processor at a time. there 51 * exists a window on MPs where the current thread on one 52 * processor is capable of being dispatched by another processor. 53 * some overlap between outgoing and incoming threads can happen 54 * when they are the same thread. in this case where the threads 55 * are the same, resume() on one processor will spin on the incoming 56 * thread until resume() on the other processor has finished with 57 * the outgoing thread. 58 * 59 * The MMU context changes when the resuming thread resides in a different 60 * process. Kernel threads are known by resume to reside in process 0. 61 * The MMU context, therefore, only changes when resuming a thread in 62 * a process different from curproc. 63 * 64 * resume_from_intr() is called when the thread being resumed was not 65 * passivated by resume (e.g. was interrupted). This means that the 66 * resume lock is already held and that a restore context is not needed. 67 * Also, the MMU context is not changed on the resume in this case. 68 * 69 * resume_from_zombie() is the same as resume except the calling thread 70 * is a zombie and must be put on the deathrow list after the CPU is 71 * off the stack. 72 */ 73 74#if defined(lint) 75 76/* ARGSUSED */ 77void 78resume(kthread_id_t t) 79{} 80 81#else /* lint */ 82 83 ENTRY(resume) 84 save %sp, -SA(MINFRAME), %sp ! save ins and locals 85 86 call __dtrace_probe___sched_off__cpu ! DTrace probe 87 mov %i0, %o0 ! arg for DTrace probe 88 89 membar #Sync ! flush writebuffers 90 flushw ! flushes all but this window 91 92 stn %i7, [THREAD_REG + T_PC] ! save return address 93 stn %fp, [THREAD_REG + T_SP] ! save sp 94 95 ! 96 ! Save GSR (Graphics Status Register). 97 ! 98 ! Read fprs, call fp_save if FPRS_FEF set. 99 ! This handles floating-point state saving. 100 ! The fprs could be turned on by hw bcopy software, 101 ! *or* by fp_disabled. Handle it either way. 102 ! 103 ldn [THREAD_REG + T_LWP], %o4 ! get lwp pointer 104 rd %fprs, %g4 ! read fprs 105 brnz,pt %o4, 0f ! if user thread skip 106 ldn [THREAD_REG + T_CPU], %i1 ! get CPU pointer 107 108 ! 109 ! kernel thread 110 ! 111 ! we save fprs at the beginning the stack so we know 112 ! where to check at resume time 113 ldn [THREAD_REG + T_STACK], %i2 114 ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer 115 andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set 116 bz,pt %icc, 1f ! nope, skip 117 st %g4, [%i2 + SA(MINFRAME) + FPU_FPRS] ! save fprs 118 119 ! save kernel fp state in stack 120 add %i2, SA(MINFRAME), %o0 ! o0 = kfpu_t ptr 121 rd %gsr, %g5 122 call fp_save 123 stx %g5, [%o0 + FPU_GSR] ! store GSR 124 ba,a,pt %icc, 1f 125 nop 126 1270: 128 ! user thread 129 ! o4 = lwp ptr 130 ! g4 = fprs 131 ! i1 = CPU ptr 132 ldn [%o4 + LWP_FPU], %o0 ! fp pointer 133 stn %fp, [THREAD_REG + T_SP] ! save sp 134 andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set 135 st %g4, [%o0 + FPU_FPRS] ! store FPRS 136#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 137 sethi %hi(fpu_exists), %g5 138 ld [%g5 + %lo(fpu_exists)], %g5 139 brz,pn %g5, 1f 140 ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer 141#endif 142 bz,pt %icc, 1f ! most apps don't use fp 143 ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer 144 ldn [%o4 + LWP_FPU], %o0 ! fp pointer 145 rd %gsr, %g5 146 call fp_save ! doesn't touch globals 147 stx %g5, [%o0 + FPU_GSR] ! store GSR 1481: 149 ! 150 ! Perform context switch callback if set. 151 ! This handles coprocessor state saving. 152 ! i1 = cpu ptr 153 ! g3 = ctx pointer 154 ! 155 wr %g0, %g0, %fprs ! disable fpu and clear fprs 156 brz,pt %g3, 2f ! skip call when zero 157 ldn [%i0 + T_PROCP], %i3 ! delay slot - get proc pointer 158 call savectx 159 mov THREAD_REG, %o0 ! delay - arg = thread pointer 1602: 161 ldn [THREAD_REG + T_PROCP], %i2 ! load old curproc - for mmu 162 163 ! 164 ! Temporarily switch to idle thread's stack 165 ! 166 ldn [%i1 + CPU_IDLE_THREAD], %o0 ! idle thread pointer 167 ldn [%o0 + T_SP], %o1 ! get onto idle thread stack 168 sub %o1, SA(MINFRAME), %sp ! save room for ins and locals 169 clr %fp 170 171 ! 172 ! Set the idle thread as the current thread 173 ! 174 mov THREAD_REG, %l3 ! save %g7 (current thread) 175 mov %o0, THREAD_REG ! set %g7 to idle 176 stn %o0, [%i1 + CPU_THREAD] ! set CPU's thread to idle 177 178 ! 179 ! Clear and unlock previous thread's t_lock 180 ! to allow it to be dispatched by another processor. 181 ! 182 clrb [%l3 + T_LOCK] ! clear tp->t_lock 183 184 ! 185 ! IMPORTANT: Registers at this point must be: 186 ! %i0 = new thread 187 ! %i1 = cpu pointer 188 ! %i2 = old proc pointer 189 ! %i3 = new proc pointer 190 ! 191 ! Here we are in the idle thread, have dropped the old thread. 192 ! 193 ALTENTRY(_resume_from_idle) 194 195 ! SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) 196 SET_KCONTEXTREG(%o0, %g1, %g2, %g3, %o3, l1, l2, l3) 197 198 cmp %i2, %i3 ! resuming the same process? 199 be,pt %xcc, 5f ! yes. 200 nop 201 202 ldx [%i3 + P_AS], %o0 ! load p->p_as 203 ldx [%o0 + A_HAT], %i5 ! %i5 = new proc hat 204 205 ! 206 ! update cpusran field 207 ! 208 ld [%i1 + CPU_ID], %o4 209 add %i5, SFMMU_CPUSRAN, %o5 210 CPU_INDEXTOSET(%o5, %o4, %g1) 211 ldx [%o5], %o2 ! %o2 = cpusran field 212 mov 1, %g2 213 sllx %g2, %o4, %o4 ! %o4 = bit for this cpu 214 andcc %o4, %o2, %g0 215 bnz,pn %xcc, 0f ! bit already set, go to 0 216 nop 2173: 218 or %o2, %o4, %o1 ! or in this cpu's bit mask 219 casx [%o5], %o2, %o1 220 cmp %o2, %o1 221 bne,a,pn %xcc, 3b 222 ldx [%o5], %o2 ! o2 = cpusran field 223 membar #LoadLoad|#StoreLoad 224 2250: 226 ! 227 ! disable interrupts 228 ! 229 ! if resume from user to kernel thread 230 ! call sfmmu_setctx_sec 231 ! if resume from kernel (or a different user) thread to user thread 232 ! call sfmmu_alloc_ctx 233 ! sfmmu_load_mmustate 234 ! 235 ! enable interrupts 236 ! 237 ! %i5 = new proc hat 238 ! 239 240 sethi %hi(ksfmmup), %o2 241 ldx [%o2 + %lo(ksfmmup)], %o2 242 243 rdpr %pstate, %i4 244 cmp %i5, %o2 ! new proc hat == ksfmmup ? 245 bne,pt %xcc, 3f ! new proc is not kernel as, go to 3 246 wrpr %i4, PSTATE_IE, %pstate 247 248 SET_KAS_CTXSEC_ARGS(%i5, %o0, %o1) 249 250 ! new proc is kernel as 251 252 call sfmmu_setctx_sec ! switch to kernel context 253 or %o0, %o1, %o0 254 255 ba,a,pt %icc, 4f 256 257 ! 258 ! Switch to user address space. 259 ! 2603: 261 mov %i5, %o0 ! %o0 = sfmmup 262 mov %i1, %o2 ! %o2 = CPU 263 set SFMMU_PRIVATE, %o3 ! %o3 = sfmmu private flag 264 call sfmmu_alloc_ctx 265 mov %g0, %o1 ! %o1 = allocate flag = 0 266 267 brz,a,pt %o0, 4f ! %o0 == 0, no private alloc'ed 268 nop 269 270 ldn [%i5 + SFMMU_SCDP], %o0 ! using shared contexts? 271 brz,a,pt %o0, 4f 272 nop 273 274 ldn [%o0 + SCD_SFMMUP], %o0 ! %o0 = scdp->scd_sfmmup 275 mov %i1, %o2 ! %o2 = CPU 276 set SFMMU_SHARED, %o3 ! %o3 = sfmmu shared flag 277 call sfmmu_alloc_ctx 278 mov 1, %o1 ! %o1 = allocate flag = 1 279 2804: 281 call sfmmu_load_mmustate ! program MMU registers 282 mov %i5, %o0 283 284 wrpr %g0, %i4, %pstate ! enable interrupts 285 2865: 287 ! 288 ! spin until dispatched thread's mutex has 289 ! been unlocked. this mutex is unlocked when 290 ! it becomes safe for the thread to run. 291 ! 292 ldstub [%i0 + T_LOCK], %o0 ! lock curthread's t_lock 2936: 294 brnz,pn %o0, 7f ! lock failed 295 ldx [%i0 + T_PC], %i7 ! delay - restore resuming thread's pc 296 297 ! 298 ! Fix CPU structure to indicate new running thread. 299 ! Set pointer in new thread to the CPU structure. 300 ! XXX - Move migration statistic out of here 301 ! 302 ldx [%i0 + T_CPU], %g2 ! last CPU to run the new thread 303 cmp %g2, %i1 ! test for migration 304 be,pt %xcc, 4f ! no migration 305 ldn [%i0 + T_LWP], %o1 ! delay - get associated lwp (if any) 306 ldx [%i1 + CPU_STATS_SYS_CPUMIGRATE], %g2 307 inc %g2 308 stx %g2, [%i1 + CPU_STATS_SYS_CPUMIGRATE] 309 stx %i1, [%i0 + T_CPU] ! set new thread's CPU pointer 3104: 311 stx %i0, [%i1 + CPU_THREAD] ! set CPU's thread pointer 312 membar #StoreLoad ! synchronize with mutex_exit() 313 mov %i0, THREAD_REG ! update global thread register 314 stx %o1, [%i1 + CPU_LWP] ! set CPU's lwp ptr 315 brz,a,pn %o1, 1f ! if no lwp, branch and clr mpcb 316 stx %g0, [%i1 + CPU_MPCB] 317 ! 318 ! user thread 319 ! o1 = lwp 320 ! i0 = new thread 321 ! 322 ldx [%i0 + T_STACK], %o0 323 stx %o0, [%i1 + CPU_MPCB] ! set CPU's mpcb pointer 324#ifdef CPU_MPCB_PA 325 ldx [%o0 + MPCB_PA], %o0 326 stx %o0, [%i1 + CPU_MPCB_PA] 327#endif 328 ! Switch to new thread's stack 329 ldx [%i0 + T_SP], %o0 ! restore resuming thread's sp 330 sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore 331 mov %o0, %fp 332 ! 333 ! Restore resuming thread's GSR reg and floating-point regs 334 ! Note that the ld to the gsr register ensures that the loading of 335 ! the floating point saved state has completed without necessity 336 ! of a membar #Sync. 337 ! 338#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 339 sethi %hi(fpu_exists), %g3 340 ld [%g3 + %lo(fpu_exists)], %g3 341 brz,pn %g3, 2f 342 ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx? 343#endif 344 ldx [%o1 + LWP_FPU], %o0 ! fp pointer 345 ld [%o0 + FPU_FPRS], %g5 ! get fpu_fprs 346 andcc %g5, FPRS_FEF, %g0 ! is FPRS_FEF set? 347 bz,a,pt %icc, 9f ! no, skip fp_restore 348 wr %g0, FPRS_FEF, %fprs ! enable fprs so fp_zero works 349 350 ldx [THREAD_REG + T_CPU], %o4 ! cpu pointer 351 call fp_restore 352 wr %g5, %g0, %fprs ! enable fpu and restore fprs 353 354 ldx [%o0 + FPU_GSR], %g5 ! load saved GSR data 355 wr %g5, %g0, %gsr ! restore %gsr data 356 ba,pt %icc,2f 357 ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx? 358 3599: 360 ! 361 ! Zero resuming thread's fp registers, for *all* non-fp program 362 ! Remove all possibility of using the fp regs as a "covert channel". 363 ! 364 call fp_zero 365 wr %g0, %g0, %gsr 366 ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx? 367 ba,pt %icc, 2f 368 wr %g0, %g0, %fprs ! disable fprs 369 3701: 371#ifdef CPU_MPCB_PA 372 mov -1, %o1 373 stx %o1, [%i1 + CPU_MPCB_PA] 374#endif 375 ! 376 ! kernel thread 377 ! i0 = new thread 378 ! 379 ! Switch to new thread's stack 380 ! 381 ldx [%i0 + T_SP], %o0 ! restore resuming thread's sp 382 sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore 383 mov %o0, %fp 384 ! 385 ! Restore resuming thread's GSR reg and floating-point regs 386 ! Note that the ld to the gsr register ensures that the loading of 387 ! the floating point saved state has completed without necessity 388 ! of a membar #Sync. 389 ! 390 ldx [%i0 + T_STACK], %o0 391 ld [%o0 + SA(MINFRAME) + FPU_FPRS], %g5 ! load fprs 392 ldx [%i0 + T_CTX], %i5 ! should thread restorectx? 393 andcc %g5, FPRS_FEF, %g0 ! did we save fp in stack? 394 bz,a,pt %icc, 2f 395 wr %g0, %g0, %fprs ! clr fprs 396 397 wr %g5, %g0, %fprs ! enable fpu and restore fprs 398 call fp_restore 399 add %o0, SA(MINFRAME), %o0 ! o0 = kpu_t ptr 400 ldx [%o0 + FPU_GSR], %g5 ! load saved GSR data 401 wr %g5, %g0, %gsr ! restore %gsr data 402 4032: 404 ! 405 ! Restore resuming thread's context 406 ! i5 = ctx ptr 407 ! 408 brz,a,pt %i5, 8f ! skip restorectx() when zero 409 ld [%i1 + CPU_BASE_SPL], %o0 410 call restorectx ! thread can not sleep on temp stack 411 mov THREAD_REG, %o0 ! delay slot - arg = thread pointer 412 ! 413 ! Set priority as low as possible, blocking all interrupt threads 414 ! that may be active. 415 ! 416 ld [%i1 + CPU_BASE_SPL], %o0 4178: 418 wrpr %o0, 0, %pil 419 wrpr %g0, WSTATE_KERN, %wstate 420 ! 421 ! If we are resuming an interrupt thread, store a starting timestamp 422 ! in the thread structure. 423 ! 424 lduh [THREAD_REG + T_FLAGS], %o0 425 andcc %o0, T_INTR_THREAD, %g0 426 bnz,pn %xcc, 0f 427 nop 4285: 429 call __dtrace_probe___sched_on__cpu ! DTrace probe 430 nop 431 432 ret ! resume curthread 433 restore 4340: 435 add THREAD_REG, T_INTR_START, %o2 4361: 437 ldx [%o2], %o1 438 rdpr %tick, %o0 439 sllx %o0, 1, %o0 440 srlx %o0, 1, %o0 ! shift off NPT bit 441 casx [%o2], %o1, %o0 442 cmp %o0, %o1 443 be,pt %xcc, 5b 444 nop 445 ! If an interrupt occurred while we were attempting to store 446 ! the timestamp, try again. 447 ba,pt %xcc, 1b 448 nop 449 450 ! 451 ! lock failed - spin with regular load to avoid cache-thrashing. 452 ! 4537: 454 brnz,a,pt %o0, 7b ! spin while locked 455 ldub [%i0 + T_LOCK], %o0 456 ba %xcc, 6b 457 ldstub [%i0 + T_LOCK], %o0 ! delay - lock curthread's mutex 458 SET_SIZE(_resume_from_idle) 459 SET_SIZE(resume) 460 461#endif /* lint */ 462 463#if defined(lint) 464 465/* ARGSUSED */ 466void 467resume_from_zombie(kthread_id_t t) 468{} 469 470#else /* lint */ 471 472 ENTRY(resume_from_zombie) 473 save %sp, -SA(MINFRAME), %sp ! save ins and locals 474 475 call __dtrace_probe___sched_off__cpu ! DTrace probe 476 mov %i0, %o0 ! arg for DTrace probe 477 478 ldn [THREAD_REG + T_CPU], %i1 ! cpu pointer 479 480 flushw ! flushes all but this window 481 ldn [THREAD_REG + T_PROCP], %i2 ! old procp for mmu ctx 482 483 ! 484 ! Temporarily switch to the idle thread's stack so that 485 ! the zombie thread's stack can be reclaimed by the reaper. 486 ! 487 ldn [%i1 + CPU_IDLE_THREAD], %o2 ! idle thread pointer 488 ldn [%o2 + T_SP], %o1 ! get onto idle thread stack 489 sub %o1, SA(MINFRAME), %sp ! save room for ins and locals 490 clr %fp 491 ! 492 ! Set the idle thread as the current thread. 493 ! Put the zombie on death-row. 494 ! 495 mov THREAD_REG, %o0 ! save %g7 = curthread for arg 496 mov %o2, THREAD_REG ! set %g7 to idle 497 stn %g0, [%i1 + CPU_MPCB] ! clear mpcb 498#ifdef CPU_MPCB_PA 499 mov -1, %o1 500 stx %o1, [%i1 + CPU_MPCB_PA] 501#endif 502 call reapq_add ! reapq_add(old_thread); 503 stn %o2, [%i1 + CPU_THREAD] ! delay - CPU's thread = idle 504 505 ! 506 ! resume_from_idle args: 507 ! %i0 = new thread 508 ! %i1 = cpu 509 ! %i2 = old proc 510 ! %i3 = new proc 511 ! 512 b _resume_from_idle ! finish job of resume 513 ldn [%i0 + T_PROCP], %i3 ! new process 514 SET_SIZE(resume_from_zombie) 515 516#endif /* lint */ 517 518#if defined(lint) 519 520/* ARGSUSED */ 521void 522resume_from_intr(kthread_id_t t) 523{} 524 525#else /* lint */ 526 527 ENTRY(resume_from_intr) 528 save %sp, -SA(MINFRAME), %sp ! save ins and locals 529 530 flushw ! flushes all but this window 531 stn %fp, [THREAD_REG + T_SP] ! delay - save sp 532 stn %i7, [THREAD_REG + T_PC] ! save return address 533 534 ldn [%i0 + T_PC], %i7 ! restore resuming thread's pc 535 ldn [THREAD_REG + T_CPU], %i1 ! cpu pointer 536 537 ! 538 ! Fix CPU structure to indicate new running thread. 539 ! The pinned thread we're resuming already has the CPU pointer set. 540 ! 541 mov THREAD_REG, %l3 ! save old thread 542 stn %i0, [%i1 + CPU_THREAD] ! set CPU's thread pointer 543 membar #StoreLoad ! synchronize with mutex_exit() 544 mov %i0, THREAD_REG ! update global thread register 545 546 ! 547 ! Switch to new thread's stack 548 ! 549 ldn [THREAD_REG + T_SP], %o0 ! restore resuming thread's sp 550 sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore 551 mov %o0, %fp 552 clrb [%l3 + T_LOCK] ! clear intr thread's tp->t_lock 553 554 ! 555 ! If we are resuming an interrupt thread, store a timestamp in the 556 ! thread structure. 557 ! 558 lduh [THREAD_REG + T_FLAGS], %o0 559 andcc %o0, T_INTR_THREAD, %g0 560 bnz,pn %xcc, 0f 561 ! 562 ! We're resuming a non-interrupt thread. 563 ! Clear CPU_INTRCNT and check if cpu_kprunrun set? 564 ! 565 ldub [%i1 + CPU_KPRUNRUN], %o5 ! delay 566 brnz,pn %o5, 3f ! call kpreempt(KPREEMPT_SYNC); 567 stub %g0, [%i1 + CPU_INTRCNT] 5681: 569 ret ! resume curthread 570 restore 5710: 572 ! 573 ! We're an interrupt thread. Update t_intr_start and cpu_intrcnt 574 ! 575 add THREAD_REG, T_INTR_START, %o2 5762: 577 ldx [%o2], %o1 578 rdpr %tick, %o0 579 sllx %o0, 1, %o0 580 srlx %o0, 1, %o0 ! shift off NPT bit 581 casx [%o2], %o1, %o0 582 cmp %o0, %o1 583 bne,pn %xcc, 2b 584 ldn [THREAD_REG + T_INTR], %l1 ! delay 585 ! Reset cpu_intrcnt if we aren't pinning anyone 586 brz,a,pt %l1, 2f 587 stub %g0, [%i1 + CPU_INTRCNT] 5882: 589 ba,pt %xcc, 1b 590 nop 5913: 592 ! 593 ! We're a non-interrupt thread and cpu_kprunrun is set. call kpreempt. 594 ! 595 call kpreempt 596 mov KPREEMPT_SYNC, %o0 597 ba,pt %xcc, 1b 598 nop 599 SET_SIZE(resume_from_intr) 600 601#endif /* lint */ 602 603 604/* 605 * thread_start() 606 * 607 * the current register window was crafted by thread_run() to contain 608 * an address of a procedure (in register %i7), and its args in registers 609 * %i0 through %i5. a stack trace of this thread will show the procedure 610 * that thread_start() invoked at the bottom of the stack. an exit routine 611 * is stored in %l0 and called when started thread returns from its called 612 * procedure. 613 */ 614 615#if defined(lint) 616 617void 618thread_start(void) 619{} 620 621#else /* lint */ 622 623 ENTRY(thread_start) 624 mov %i0, %o0 625 jmpl %i7, %o7 ! call thread_run()'s start() procedure. 626 mov %i1, %o1 627 628 call thread_exit ! destroy thread if it returns. 629 nop 630 unimp 0 631 SET_SIZE(thread_start) 632 633#endif /* lint */ 634