1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#pragma ident "%Z%%M% %I% %E% SMI" 27 28/* 29 * Process switching routines. 30 */ 31 32#if !defined(lint) 33#include "assym.h" 34#else /* lint */ 35#include <sys/thread.h> 36#endif /* lint */ 37 38#include <sys/param.h> 39#include <sys/asm_linkage.h> 40#include <sys/mmu.h> 41#include <sys/pcb.h> 42#include <sys/machthread.h> 43#include <sys/privregs.h> 44#include <sys/vtrace.h> 45#include <vm/hat_sfmmu.h> 46 47/* 48 * resume(kthread_id_t) 49 * 50 * a thread can only run on one processor at a time. there 51 * exists a window on MPs where the current thread on one 52 * processor is capable of being dispatched by another processor. 53 * some overlap between outgoing and incoming threads can happen 54 * when they are the same thread. in this case where the threads 55 * are the same, resume() on one processor will spin on the incoming 56 * thread until resume() on the other processor has finished with 57 * the outgoing thread. 58 * 59 * The MMU context changes when the resuming thread resides in a different 60 * process. Kernel threads are known by resume to reside in process 0. 61 * The MMU context, therefore, only changes when resuming a thread in 62 * a process different from curproc. 63 * 64 * resume_from_intr() is called when the thread being resumed was not 65 * passivated by resume (e.g. was interrupted). This means that the 66 * resume lock is already held and that a restore context is not needed. 67 * Also, the MMU context is not changed on the resume in this case. 68 * 69 * resume_from_zombie() is the same as resume except the calling thread 70 * is a zombie and must be put on the deathrow list after the CPU is 71 * off the stack. 72 */ 73 74#if defined(lint) 75 76/* ARGSUSED */ 77void 78resume(kthread_id_t t) 79{} 80 81#else /* lint */ 82 83 ENTRY(resume) 84 save %sp, -SA(MINFRAME), %sp ! save ins and locals 85 86 call __dtrace_probe___sched_off__cpu ! DTrace probe 87 mov %i0, %o0 ! arg for DTrace probe 88 89 membar #Sync ! flush writebuffers 90 flushw ! flushes all but this window 91 92 stn %i7, [THREAD_REG + T_PC] ! save return address 93 stn %fp, [THREAD_REG + T_SP] ! save sp 94 95 ! 96 ! Save GSR (Graphics Status Register). 97 ! 98 ! Read fprs, call fp_save if FPRS_FEF set. 99 ! This handles floating-point state saving. 100 ! The fprs could be turned on by hw bcopy software, 101 ! *or* by fp_disabled. Handle it either way. 102 ! 103 ldn [THREAD_REG + T_LWP], %o4 ! get lwp pointer 104 rd %fprs, %g4 ! read fprs 105 brnz,pt %o4, 0f ! if user thread skip 106 ldn [THREAD_REG + T_CPU], %i1 ! get CPU pointer 107 108 ! 109 ! kernel thread 110 ! 111 ! we save fprs at the beginning the stack so we know 112 ! where to check at resume time 113 ldn [THREAD_REG + T_STACK], %i2 114 ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer 115 andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set 116 bz,pt %icc, 1f ! nope, skip 117 st %g4, [%i2 + SA(MINFRAME) + FPU_FPRS] ! save fprs 118 119 ! save kernel fp state in stack 120 add %i2, SA(MINFRAME), %o0 ! o0 = kfpu_t ptr 121 rd %gsr, %g5 122 call fp_save 123 stx %g5, [%o0 + FPU_GSR] ! store GSR 124 ba,a,pt %icc, 1f 125 nop 126 1270: 128 ! user thread 129 ! o4 = lwp ptr 130 ! g4 = fprs 131 ! i1 = CPU ptr 132 ldn [%o4 + LWP_FPU], %o0 ! fp pointer 133 stn %fp, [THREAD_REG + T_SP] ! save sp 134 andcc %g4, FPRS_FEF, %g0 ! is FPRS_FEF set 135 st %g4, [%o0 + FPU_FPRS] ! store FPRS 136#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 137 sethi %hi(fpu_exists), %g5 138 ld [%g5 + %lo(fpu_exists)], %g5 139 brz,pn %g5, 1f 140 ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer 141#endif 142 bz,pt %icc, 1f ! most apps don't use fp 143 ldn [THREAD_REG + T_CTX], %g3 ! get ctx pointer 144 ldn [%o4 + LWP_FPU], %o0 ! fp pointer 145 rd %gsr, %g5 146 call fp_save ! doesn't touch globals 147 stx %g5, [%o0 + FPU_GSR] ! store GSR 1481: 149 ! 150 ! Perform context switch callback if set. 151 ! This handles coprocessor state saving. 152 ! i1 = cpu ptr 153 ! g3 = ctx pointer 154 ! 155 wr %g0, %g0, %fprs ! disable fpu and clear fprs 156 brz,pt %g3, 2f ! skip call when zero 157 ldn [%i0 + T_PROCP], %i3 ! delay slot - get proc pointer 158 call savectx 159 mov THREAD_REG, %o0 ! delay - arg = thread pointer 1602: 161 ldn [THREAD_REG + T_PROCP], %i2 ! load old curproc - for mmu 162 163 ! 164 ! Temporarily switch to idle thread's stack 165 ! 166 ldn [%i1 + CPU_IDLE_THREAD], %o0 ! idle thread pointer 167 ldn [%o0 + T_SP], %o1 ! get onto idle thread stack 168 sub %o1, SA(MINFRAME), %sp ! save room for ins and locals 169 clr %fp 170 171 ! 172 ! Set the idle thread as the current thread 173 ! 174 mov THREAD_REG, %l3 ! save %g7 (current thread) 175 mov %o0, THREAD_REG ! set %g7 to idle 176 stn %o0, [%i1 + CPU_THREAD] ! set CPU's thread to idle 177 178 ! 179 ! Clear and unlock previous thread's t_lock 180 ! to allow it to be dispatched by another processor. 181 ! 182 clrb [%l3 + T_LOCK] ! clear tp->t_lock 183 184 ! 185 ! IMPORTANT: Registers at this point must be: 186 ! %i0 = new thread 187 ! %i1 = cpu pointer 188 ! %i2 = old proc pointer 189 ! %i3 = new proc pointer 190 ! 191 ! Here we are in the idle thread, have dropped the old thread. 192 ! 193 ALTENTRY(_resume_from_idle) 194 195 ! SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3) 196 SET_KCONTEXTREG(%o0, %g1, %g2, %g3, %o3, l1, l2, l3) 197 198 cmp %i2, %i3 ! resuming the same process? 199 be,pt %xcc, 5f ! yes. 200 nop 201 202 ldx [%i3 + P_AS], %o0 ! load p->p_as 203 ldx [%o0 + A_HAT], %i5 ! %i5 = new proc hat 204 205 ! 206 ! update cpusran field 207 ! 208 ld [%i1 + CPU_ID], %o4 209 add %i5, SFMMU_CPUSRAN, %o5 210 CPU_INDEXTOSET(%o5, %o4, %g1) 211 ldx [%o5], %o2 ! %o2 = cpusran field 212 mov 1, %g2 213 sllx %g2, %o4, %o4 ! %o4 = bit for this cpu 214 andcc %o4, %o2, %g0 215 bnz,pn %xcc, 0f ! bit already set, go to 0 216 nop 2173: 218 or %o2, %o4, %o1 ! or in this cpu's bit mask 219 casx [%o5], %o2, %o1 220 cmp %o2, %o1 221 bne,a,pn %xcc, 3b 222 ldx [%o5], %o2 ! o2 = cpusran field 223 membar #LoadLoad|#StoreLoad 224 2250: 226 ! 227 ! disable interrupts 228 ! 229 ! if resume from user to kernel thread 230 ! call sfmmu_setctx_sec 231 ! if resume from kernel (or a different user) thread to user thread 232 ! call sfmmu_alloc_ctx 233 ! sfmmu_load_mmustate 234 ! 235 ! enable interrupts 236 ! 237 ! %i5 = new proc hat 238 ! 239 240 sethi %hi(ksfmmup), %o2 241 ldx [%o2 + %lo(ksfmmup)], %o2 242 243 rdpr %pstate, %i4 244 cmp %i5, %o2 ! new proc hat == ksfmmup ? 245 bne,pt %xcc, 3f ! new proc is not kernel as, go to 3 246 wrpr %i4, PSTATE_IE, %pstate 247 248 SET_KAS_CTXSEC_ARGS(%i5, %o0, %o1) 249 250 ! new proc is kernel as 251 252 call sfmmu_setctx_sec ! switch to kernel context 253 or %o0, %o1, %o0 254 255 ba,a,pt %icc, 4f 256 257 ! 258 ! Switch to user address space. 259 ! 2603: 261 mov %i5, %o0 ! %o0 = sfmmup 262 mov %i1, %o2 ! %o2 = CPU 263 call sfmmu_alloc_ctx 264 mov %g0, %o1 ! %o1 = allocate flag = 0 2654: 266 call sfmmu_load_mmustate ! program MMU registers 267 mov %i5, %o0 268 269 wrpr %g0, %i4, %pstate ! enable interrupts 270 2715: 272 ! 273 ! spin until dispatched thread's mutex has 274 ! been unlocked. this mutex is unlocked when 275 ! it becomes safe for the thread to run. 276 ! 277 ldstub [%i0 + T_LOCK], %o0 ! lock curthread's t_lock 2786: 279 brnz,pn %o0, 7f ! lock failed 280 ldx [%i0 + T_PC], %i7 ! delay - restore resuming thread's pc 281 282 ! 283 ! Fix CPU structure to indicate new running thread. 284 ! Set pointer in new thread to the CPU structure. 285 ! XXX - Move migration statistic out of here 286 ! 287 ldx [%i0 + T_CPU], %g2 ! last CPU to run the new thread 288 cmp %g2, %i1 ! test for migration 289 be,pt %xcc, 4f ! no migration 290 ldn [%i0 + T_LWP], %o1 ! delay - get associated lwp (if any) 291 ldx [%i1 + CPU_STATS_SYS_CPUMIGRATE], %g2 292 inc %g2 293 stx %g2, [%i1 + CPU_STATS_SYS_CPUMIGRATE] 294 stx %i1, [%i0 + T_CPU] ! set new thread's CPU pointer 2954: 296 stx %i0, [%i1 + CPU_THREAD] ! set CPU's thread pointer 297 membar #StoreLoad ! synchronize with mutex_exit() 298 mov %i0, THREAD_REG ! update global thread register 299 stx %o1, [%i1 + CPU_LWP] ! set CPU's lwp ptr 300 brz,a,pn %o1, 1f ! if no lwp, branch and clr mpcb 301 stx %g0, [%i1 + CPU_MPCB] 302 ! 303 ! user thread 304 ! o1 = lwp 305 ! i0 = new thread 306 ! 307 ldx [%i0 + T_STACK], %o0 308 stx %o0, [%i1 + CPU_MPCB] ! set CPU's mpcb pointer 309#ifdef CPU_MPCB_PA 310 ldx [%o0 + MPCB_PA], %o0 311 stx %o0, [%i1 + CPU_MPCB_PA] 312#endif 313 ! Switch to new thread's stack 314 ldx [%i0 + T_SP], %o0 ! restore resuming thread's sp 315 sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore 316 mov %o0, %fp 317 ! 318 ! Restore resuming thread's GSR reg and floating-point regs 319 ! Note that the ld to the gsr register ensures that the loading of 320 ! the floating point saved state has completed without necessity 321 ! of a membar #Sync. 322 ! 323#if defined(DEBUG) || defined(NEED_FPU_EXISTS) 324 sethi %hi(fpu_exists), %g3 325 ld [%g3 + %lo(fpu_exists)], %g3 326 brz,pn %g3, 2f 327 ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx? 328#endif 329 ldx [%o1 + LWP_FPU], %o0 ! fp pointer 330 ld [%o0 + FPU_FPRS], %g5 ! get fpu_fprs 331 andcc %g5, FPRS_FEF, %g0 ! is FPRS_FEF set? 332 bz,a,pt %icc, 9f ! no, skip fp_restore 333 wr %g0, FPRS_FEF, %fprs ! enable fprs so fp_zero works 334 335 ldx [THREAD_REG + T_CPU], %o4 ! cpu pointer 336 call fp_restore 337 wr %g5, %g0, %fprs ! enable fpu and restore fprs 338 339 ldx [%o0 + FPU_GSR], %g5 ! load saved GSR data 340 wr %g5, %g0, %gsr ! restore %gsr data 341 ba,pt %icc,2f 342 ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx? 343 3449: 345 ! 346 ! Zero resuming thread's fp registers, for *all* non-fp program 347 ! Remove all possibility of using the fp regs as a "covert channel". 348 ! 349 call fp_zero 350 wr %g0, %g0, %gsr 351 ldx [%i0 + T_CTX], %i5 ! should resumed thread restorectx? 352 ba,pt %icc, 2f 353 wr %g0, %g0, %fprs ! disable fprs 354 3551: 356#ifdef CPU_MPCB_PA 357 mov -1, %o1 358 stx %o1, [%i1 + CPU_MPCB_PA] 359#endif 360 ! 361 ! kernel thread 362 ! i0 = new thread 363 ! 364 ! Switch to new thread's stack 365 ! 366 ldx [%i0 + T_SP], %o0 ! restore resuming thread's sp 367 sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore 368 mov %o0, %fp 369 ! 370 ! Restore resuming thread's GSR reg and floating-point regs 371 ! Note that the ld to the gsr register ensures that the loading of 372 ! the floating point saved state has completed without necessity 373 ! of a membar #Sync. 374 ! 375 ldx [%i0 + T_STACK], %o0 376 ld [%o0 + SA(MINFRAME) + FPU_FPRS], %g5 ! load fprs 377 ldx [%i0 + T_CTX], %i5 ! should thread restorectx? 378 andcc %g5, FPRS_FEF, %g0 ! did we save fp in stack? 379 bz,a,pt %icc, 2f 380 wr %g0, %g0, %fprs ! clr fprs 381 382 wr %g5, %g0, %fprs ! enable fpu and restore fprs 383 call fp_restore 384 add %o0, SA(MINFRAME), %o0 ! o0 = kpu_t ptr 385 ldx [%o0 + FPU_GSR], %g5 ! load saved GSR data 386 wr %g5, %g0, %gsr ! restore %gsr data 387 3882: 389 ! 390 ! Restore resuming thread's context 391 ! i5 = ctx ptr 392 ! 393 brz,a,pt %i5, 8f ! skip restorectx() when zero 394 ld [%i1 + CPU_BASE_SPL], %o0 395 call restorectx ! thread can not sleep on temp stack 396 mov THREAD_REG, %o0 ! delay slot - arg = thread pointer 397 ! 398 ! Set priority as low as possible, blocking all interrupt threads 399 ! that may be active. 400 ! 401 ld [%i1 + CPU_BASE_SPL], %o0 4028: 403 wrpr %o0, 0, %pil 404 wrpr %g0, WSTATE_KERN, %wstate 405 ! 406 ! If we are resuming an interrupt thread, store a starting timestamp 407 ! in the thread structure. 408 ! 409 lduh [THREAD_REG + T_FLAGS], %o0 410 andcc %o0, T_INTR_THREAD, %g0 411 bnz,pn %xcc, 0f 412 nop 4135: 414 call __dtrace_probe___sched_on__cpu ! DTrace probe 415 nop 416 417 ret ! resume curthread 418 restore 4190: 420 add THREAD_REG, T_INTR_START, %o2 4211: 422 ldx [%o2], %o1 423 rdpr %tick, %o0 424 sllx %o0, 1, %o0 425 srlx %o0, 1, %o0 ! shift off NPT bit 426 casx [%o2], %o1, %o0 427 cmp %o0, %o1 428 be,pt %xcc, 5b 429 nop 430 ! If an interrupt occurred while we were attempting to store 431 ! the timestamp, try again. 432 ba,pt %xcc, 1b 433 nop 434 435 ! 436 ! lock failed - spin with regular load to avoid cache-thrashing. 437 ! 4387: 439 brnz,a,pt %o0, 7b ! spin while locked 440 ldub [%i0 + T_LOCK], %o0 441 ba %xcc, 6b 442 ldstub [%i0 + T_LOCK], %o0 ! delay - lock curthread's mutex 443 SET_SIZE(_resume_from_idle) 444 SET_SIZE(resume) 445 446#endif /* lint */ 447 448#if defined(lint) 449 450/* ARGSUSED */ 451void 452resume_from_zombie(kthread_id_t t) 453{} 454 455#else /* lint */ 456 457 ENTRY(resume_from_zombie) 458 save %sp, -SA(MINFRAME), %sp ! save ins and locals 459 460 call __dtrace_probe___sched_off__cpu ! DTrace probe 461 mov %i0, %o0 ! arg for DTrace probe 462 463 ldn [THREAD_REG + T_CPU], %i1 ! cpu pointer 464 465 flushw ! flushes all but this window 466 ldn [THREAD_REG + T_PROCP], %i2 ! old procp for mmu ctx 467 468 ! 469 ! Temporarily switch to the idle thread's stack so that 470 ! the zombie thread's stack can be reclaimed by the reaper. 471 ! 472 ldn [%i1 + CPU_IDLE_THREAD], %o2 ! idle thread pointer 473 ldn [%o2 + T_SP], %o1 ! get onto idle thread stack 474 sub %o1, SA(MINFRAME), %sp ! save room for ins and locals 475 clr %fp 476 ! 477 ! Set the idle thread as the current thread. 478 ! Put the zombie on death-row. 479 ! 480 mov THREAD_REG, %o0 ! save %g7 = curthread for arg 481 mov %o2, THREAD_REG ! set %g7 to idle 482 stn %g0, [%i1 + CPU_MPCB] ! clear mpcb 483#ifdef CPU_MPCB_PA 484 mov -1, %o1 485 stx %o1, [%i1 + CPU_MPCB_PA] 486#endif 487 call reapq_add ! reapq_add(old_thread); 488 stn %o2, [%i1 + CPU_THREAD] ! delay - CPU's thread = idle 489 490 ! 491 ! resume_from_idle args: 492 ! %i0 = new thread 493 ! %i1 = cpu 494 ! %i2 = old proc 495 ! %i3 = new proc 496 ! 497 b _resume_from_idle ! finish job of resume 498 ldn [%i0 + T_PROCP], %i3 ! new process 499 SET_SIZE(resume_from_zombie) 500 501#endif /* lint */ 502 503#if defined(lint) 504 505/* ARGSUSED */ 506void 507resume_from_intr(kthread_id_t t) 508{} 509 510#else /* lint */ 511 512 ENTRY(resume_from_intr) 513 save %sp, -SA(MINFRAME), %sp ! save ins and locals 514 515 flushw ! flushes all but this window 516 stn %fp, [THREAD_REG + T_SP] ! delay - save sp 517 stn %i7, [THREAD_REG + T_PC] ! save return address 518 519 ldn [%i0 + T_PC], %i7 ! restore resuming thread's pc 520 ldn [THREAD_REG + T_CPU], %i1 ! cpu pointer 521 522 ! 523 ! Fix CPU structure to indicate new running thread. 524 ! The pinned thread we're resuming already has the CPU pointer set. 525 ! 526 mov THREAD_REG, %l3 ! save old thread 527 stn %i0, [%i1 + CPU_THREAD] ! set CPU's thread pointer 528 membar #StoreLoad ! synchronize with mutex_exit() 529 mov %i0, THREAD_REG ! update global thread register 530 531 ! 532 ! Switch to new thread's stack 533 ! 534 ldn [THREAD_REG + T_SP], %o0 ! restore resuming thread's sp 535 sub %o0, SA(MINFRAME), %sp ! in case of intr or trap before restore 536 mov %o0, %fp 537 clrb [%l3 + T_LOCK] ! clear intr thread's tp->t_lock 538 539 ! 540 ! If we are resuming an interrupt thread, store a timestamp in the 541 ! thread structure. 542 ! 543 lduh [THREAD_REG + T_FLAGS], %o0 544 andcc %o0, T_INTR_THREAD, %g0 545 bnz,pn %xcc, 0f 546 ! 547 ! We're resuming a non-interrupt thread. 548 ! Clear CPU_INTRCNT and check if cpu_kprunrun set? 549 ! 550 ldub [%i1 + CPU_KPRUNRUN], %o5 ! delay 551 brnz,pn %o5, 3f ! call kpreempt(KPREEMPT_SYNC); 552 stub %g0, [%i1 + CPU_INTRCNT] 5531: 554 ret ! resume curthread 555 restore 5560: 557 ! 558 ! We're an interrupt thread. Update t_intr_start and cpu_intrcnt 559 ! 560 add THREAD_REG, T_INTR_START, %o2 5612: 562 ldx [%o2], %o1 563 rdpr %tick, %o0 564 sllx %o0, 1, %o0 565 srlx %o0, 1, %o0 ! shift off NPT bit 566 casx [%o2], %o1, %o0 567 cmp %o0, %o1 568 bne,pn %xcc, 2b 569 ldn [THREAD_REG + T_INTR], %l1 ! delay 570 ! Reset cpu_intrcnt if we aren't pinning anyone 571 brz,a,pt %l1, 2f 572 stub %g0, [%i1 + CPU_INTRCNT] 5732: 574 ba,pt %xcc, 1b 575 nop 5763: 577 ! 578 ! We're a non-interrupt thread and cpu_kprunrun is set. call kpreempt. 579 ! 580 call kpreempt 581 mov KPREEMPT_SYNC, %o0 582 ba,pt %xcc, 1b 583 nop 584 SET_SIZE(resume_from_intr) 585 586#endif /* lint */ 587 588 589/* 590 * thread_start() 591 * 592 * the current register window was crafted by thread_run() to contain 593 * an address of a procedure (in register %i7), and its args in registers 594 * %i0 through %i5. a stack trace of this thread will show the procedure 595 * that thread_start() invoked at the bottom of the stack. an exit routine 596 * is stored in %l0 and called when started thread returns from its called 597 * procedure. 598 */ 599 600#if defined(lint) 601 602void 603thread_start(void) 604{} 605 606#else /* lint */ 607 608 ENTRY(thread_start) 609 mov %i0, %o0 610 jmpl %i7, %o7 ! call thread_run()'s start() procedure. 611 mov %i1, %o1 612 613 call thread_exit ! destroy thread if it returns. 614 nop 615 unimp 0 616 SET_SIZE(thread_start) 617 618#endif /* lint */ 619