1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25#if defined(lint) 26#include <sys/types.h> 27#include <sys/thread.h> 28#else /* lint */ 29#include "assym.h" 30#endif /* lint */ 31 32#include <sys/cmn_err.h> 33#include <sys/ftrace.h> 34#include <sys/asm_linkage.h> 35#include <sys/machthread.h> 36#include <sys/machcpuvar.h> 37#include <sys/intreg.h> 38#include <sys/ivintr.h> 39 40#ifdef TRAPTRACE 41#include <sys/traptrace.h> 42#endif /* TRAPTRACE */ 43 44#if defined(lint) 45 46/* ARGSUSED */ 47void 48pil_interrupt(int level) 49{} 50 51#else /* lint */ 52 53 54/* 55 * (TT 0x40..0x4F, TL>0) Interrupt Level N Handler (N == 1..15) 56 * Register passed from LEVEL_INTERRUPT(level) 57 * %g4 - interrupt request level 58 */ 59 ENTRY_NP(pil_interrupt) 60 ! 61 ! Register usage 62 ! %g1 - cpu 63 ! %g2 - pointer to intr_vec_t (iv) 64 ! %g4 - pil 65 ! %g3, %g5, %g6, %g7 - temps 66 ! 67 ! Grab the first or list head intr_vec_t off the intr_head[pil] 68 ! and panic immediately if list head is NULL. Otherwise, update 69 ! intr_head[pil] to next intr_vec_t on the list and clear softint 70 ! %clear_softint, if next intr_vec_t is NULL. 71 ! 72 CPU_ADDR(%g1, %g5) ! %g1 = cpu 73 ! 74 ALTENTRY(pil_interrupt_common) 75 sll %g4, CPTRSHIFT, %g5 ! %g5 = offset to the pil entry 76 add %g1, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head 77 add %g6, %g5, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] 78 ldn [%g6], %g2 ! %g2 = cpu->m_cpu.intr_head[pil] 79 brnz,pt %g2, 0f ! check list head (iv) is NULL 80 nop 81 ba ptl1_panic ! panic, list head (iv) is NULL 82 mov PTL1_BAD_INTR_VEC, %g1 830: 84 lduh [%g2 + IV_FLAGS], %g7 ! %g7 = iv->iv_flags 85 and %g7, IV_SOFTINT_MT, %g3 ! %g3 = iv->iv_flags & IV_SOFTINT_MT 86 brz,pt %g3, 1f ! check for multi target softint 87 add %g2, IV_PIL_NEXT, %g7 ! g7% = &iv->iv_pil_next 88 ld [%g1 + CPU_ID], %g3 ! for multi target softint, use cpuid 89 sll %g3, CPTRSHIFT, %g3 ! convert cpuid to offset address 90 add %g7, %g3, %g7 ! %g5 = &iv->iv_xpil_next[cpuid] 911: 92 ldn [%g7], %g3 ! %g3 = next intr_vec_t 93 brnz,pn %g3, 2f ! branch if next intr_vec_t non NULL 94 stn %g3, [%g6] ! update cpu->m_cpu.intr_head[pil] 95 add %g1, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail 96 stn %g0, [%g5 + %g6] ! clear cpu->m_cpu.intr_tail[pil] 97 mov 1, %g5 ! %g5 = 1 98 sll %g5, %g4, %g5 ! %g5 = 1 << pil 99 wr %g5, CLEAR_SOFTINT ! clear interrupt on this pil 1002: 101#ifdef TRAPTRACE 102 TRACE_PTR(%g5, %g6) 103 TRACE_SAVE_TL_GL_REGS(%g5, %g6) 104 rdpr %tt, %g6 105 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt 106 rdpr %tpc, %g6 107 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc 108 rdpr %tstate, %g6 109 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate 110 stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp 111 stna %g2, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = first intr_vec 112 stna %g3, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = next intr_vec 113 GET_TRACE_TICK(%g6, %g3) 114 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick 115 sll %g4, CPTRSHIFT, %g3 116 add %g1, INTR_HEAD, %g6 117 ldn [%g6 + %g3], %g6 ! %g6=cpu->m_cpu.intr_head[pil] 118 stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil] 119 add %g1, INTR_TAIL, %g6 120 ldn [%g6 + %g3], %g6 ! %g6=cpu->m_cpu.intr_tail[pil] 121 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil] 122 stna %g4, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil 123 TRACE_NEXT(%g5, %g6, %g3) 124#endif /* TRAPTRACE */ 125 ! 126 ! clear the iv_pending flag for this interrupt request 127 ! 128 lduh [%g2 + IV_FLAGS], %g3 ! %g3 = iv->iv_flags 129 andn %g3, IV_SOFTINT_PEND, %g3 ! %g3 = !(iv->iv_flags & PEND) 130 sth %g3, [%g2 + IV_FLAGS] ! clear IV_SOFTINT_PEND flag 131 stn %g0, [%g7] ! clear iv->iv_pil_next or 132 ! iv->iv_pil_xnext 133 134 ! 135 ! Prepare for sys_trap() 136 ! 137 ! Registers passed to sys_trap() 138 ! %g1 - interrupt handler at TL==0 139 ! %g2 - pointer to current intr_vec_t (iv), 140 ! job queue for intr_thread or current_thread 141 ! %g3 - pil 142 ! %g4 - initial pil for handler 143 ! 144 ! figure which handler to run and which %pil it starts at 145 ! intr_thread starts at DISP_LEVEL to prevent preemption 146 ! current_thread starts at PIL_MAX to protect cpu_intr_actv 147 ! 148 mov %g4, %g3 ! %g3 = %g4, pil 149 cmp %g4, LOCK_LEVEL 150 bg,a,pt %xcc, 3f ! branch if pil > LOCK_LEVEL 151 mov PIL_MAX, %g4 ! %g4 = PIL_MAX (15) 152 sethi %hi(intr_thread), %g1 ! %g1 = intr_thread 153 mov DISP_LEVEL, %g4 ! %g4 = DISP_LEVEL (11) 154 ba,pt %xcc, sys_trap 155 or %g1, %lo(intr_thread), %g1 1563: 157 sethi %hi(current_thread), %g1 ! %g1 = current_thread 158 ba,pt %xcc, sys_trap 159 or %g1, %lo(current_thread), %g1 160 SET_SIZE(pil_interrupt_common) 161 SET_SIZE(pil_interrupt) 162 163#endif /* lint */ 164 165 166#ifndef lint 167_spurious: 168 .asciz "!interrupt 0x%x at level %d not serviced" 169 170/* 171 * SERVE_INTR_PRE is called once, just before the first invocation 172 * of SERVE_INTR. 173 * 174 * Registers on entry: 175 * 176 * iv_p, cpu, regs: may be out-registers 177 * ls1, ls2: local scratch registers 178 * os1, os2, os3: scratch registers, may be out 179 */ 180 181#define SERVE_INTR_PRE(iv_p, cpu, ls1, ls2, os1, os2, os3, regs) \ 182 mov iv_p, ls1; \ 183 mov iv_p, ls2; \ 184 SERVE_INTR_TRACE(iv_p, os1, os2, os3, regs); 185 186/* 187 * SERVE_INTR is called immediately after either SERVE_INTR_PRE or 188 * SERVE_INTR_NEXT, without intervening code. No register values 189 * may be modified. 190 * 191 * After calling SERVE_INTR, the caller must check if os3 is set. If 192 * so, there is another interrupt to process. The caller must call 193 * SERVE_INTR_NEXT, immediately followed by SERVE_INTR. 194 * 195 * Before calling SERVE_INTR_NEXT, the caller may perform accounting 196 * and other actions which need to occur after invocation of an interrupt 197 * handler. However, the values of ls1 and os3 *must* be preserved and 198 * passed unmodified into SERVE_INTR_NEXT. 199 * 200 * Registers on return from SERVE_INTR: 201 * 202 * ls1 - the pil just processed 203 * ls2 - the pointer to intr_vec_t (iv) just processed 204 * os3 - if set, another interrupt needs to be processed 205 * cpu, ls1, os3 - must be preserved if os3 is set 206 */ 207 208#define SERVE_INTR(os5, cpu, ls1, ls2, os1, os2, os3, os4) \ 209 ldn [ls1 + IV_HANDLER], os2; \ 210 ldn [ls1 + IV_ARG1], %o0; \ 211 ldn [ls1 + IV_ARG2], %o1; \ 212 call os2; \ 213 lduh [ls1 + IV_PIL], ls1; \ 214 brnz,pt %o0, 2f; \ 215 mov CE_WARN, %o0; \ 216 set _spurious, %o1; \ 217 mov ls2, %o2; \ 218 call cmn_err; \ 219 rdpr %pil, %o3; \ 2202: ldn [THREAD_REG + T_CPU], cpu; \ 221 sll ls1, 3, os1; \ 222 add os1, CPU_STATS_SYS_INTR - 8, os2; \ 223 ldx [cpu + os2], os3; \ 224 inc os3; \ 225 stx os3, [cpu + os2]; \ 226 sll ls1, CPTRSHIFT, os2; \ 227 add cpu, INTR_HEAD, os1; \ 228 add os1, os2, os1; \ 229 ldn [os1], os3; 230 231/* 232 * Registers on entry: 233 * 234 * cpu - cpu pointer (clobbered, set to cpu upon completion) 235 * ls1, os3 - preserved from prior call to SERVE_INTR 236 * ls2 - local scratch reg (not preserved) 237 * os1, os2, os4, os5 - scratch reg, can be out (not preserved) 238 */ 239#define SERVE_INTR_NEXT(os5, cpu, ls1, ls2, os1, os2, os3, os4) \ 240 sll ls1, CPTRSHIFT, os4; \ 241 add cpu, INTR_HEAD, os1; \ 242 rdpr %pstate, ls2; \ 243 wrpr ls2, PSTATE_IE, %pstate; \ 244 lduh [os3 + IV_FLAGS], os2; \ 245 and os2, IV_SOFTINT_MT, os2; \ 246 brz,pt os2, 4f; \ 247 add os3, IV_PIL_NEXT, os2; \ 248 ld [cpu + CPU_ID], os5; \ 249 sll os5, CPTRSHIFT, os5; \ 250 add os2, os5, os2; \ 2514: ldn [os2], os5; \ 252 brnz,pn os5, 5f; \ 253 stn os5, [os1 + os4]; \ 254 add cpu, INTR_TAIL, os1; \ 255 stn %g0, [os1 + os4]; \ 256 mov 1, os1; \ 257 sll os1, ls1, os1; \ 258 wr os1, CLEAR_SOFTINT; \ 2595: lduh [os3 + IV_FLAGS], ls1; \ 260 andn ls1, IV_SOFTINT_PEND, ls1; \ 261 sth ls1, [os3 + IV_FLAGS]; \ 262 stn %g0, [os2]; \ 263 wrpr %g0, ls2, %pstate; \ 264 mov os3, ls1; \ 265 mov os3, ls2; \ 266 SERVE_INTR_TRACE2(os5, os1, os2, os3, os4); 267 268#ifdef TRAPTRACE 269/* 270 * inum - not modified, _spurious depends on it. 271 */ 272#define SERVE_INTR_TRACE(inum, os1, os2, os3, os4) \ 273 rdpr %pstate, os3; \ 274 andn os3, PSTATE_IE | PSTATE_AM, os2; \ 275 wrpr %g0, os2, %pstate; \ 276 TRACE_PTR(os1, os2); \ 277 ldn [os4 + PC_OFF], os2; \ 278 stna os2, [os1 + TRAP_ENT_TPC]%asi; \ 279 ldx [os4 + TSTATE_OFF], os2; \ 280 stxa os2, [os1 + TRAP_ENT_TSTATE]%asi; \ 281 mov os3, os4; \ 282 GET_TRACE_TICK(os2, os3); \ 283 stxa os2, [os1 + TRAP_ENT_TICK]%asi; \ 284 TRACE_SAVE_TL_GL_REGS(os1, os2); \ 285 set TT_SERVE_INTR, os2; \ 286 rdpr %pil, os3; \ 287 or os2, os3, os2; \ 288 stha os2, [os1 + TRAP_ENT_TT]%asi; \ 289 stna %sp, [os1 + TRAP_ENT_SP]%asi; \ 290 stna inum, [os1 + TRAP_ENT_TR]%asi; \ 291 stna %g0, [os1 + TRAP_ENT_F1]%asi; \ 292 stna %g0, [os1 + TRAP_ENT_F2]%asi; \ 293 stna %g0, [os1 + TRAP_ENT_F3]%asi; \ 294 stna %g0, [os1 + TRAP_ENT_F4]%asi; \ 295 TRACE_NEXT(os1, os2, os3); \ 296 wrpr %g0, os4, %pstate 297#else /* TRAPTRACE */ 298#define SERVE_INTR_TRACE(inum, os1, os2, os3, os4) 299#endif /* TRAPTRACE */ 300 301#ifdef TRAPTRACE 302/* 303 * inum - not modified, _spurious depends on it. 304 */ 305#define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4) \ 306 rdpr %pstate, os3; \ 307 andn os3, PSTATE_IE | PSTATE_AM, os2; \ 308 wrpr %g0, os2, %pstate; \ 309 TRACE_PTR(os1, os2); \ 310 stna %g0, [os1 + TRAP_ENT_TPC]%asi; \ 311 stxa %g0, [os1 + TRAP_ENT_TSTATE]%asi; \ 312 mov os3, os4; \ 313 GET_TRACE_TICK(os2, os3); \ 314 stxa os2, [os1 + TRAP_ENT_TICK]%asi; \ 315 TRACE_SAVE_TL_GL_REGS(os1, os2); \ 316 set TT_SERVE_INTR, os2; \ 317 rdpr %pil, os3; \ 318 or os2, os3, os2; \ 319 stha os2, [os1 + TRAP_ENT_TT]%asi; \ 320 stna %sp, [os1 + TRAP_ENT_SP]%asi; \ 321 stna inum, [os1 + TRAP_ENT_TR]%asi; \ 322 stna %g0, [os1 + TRAP_ENT_F1]%asi; \ 323 stna %g0, [os1 + TRAP_ENT_F2]%asi; \ 324 stna %g0, [os1 + TRAP_ENT_F3]%asi; \ 325 stna %g0, [os1 + TRAP_ENT_F4]%asi; \ 326 TRACE_NEXT(os1, os2, os3); \ 327 wrpr %g0, os4, %pstate 328#else /* TRAPTRACE */ 329#define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4) 330#endif /* TRAPTRACE */ 331 332#endif /* lint */ 333 334#if defined(lint) 335 336/*ARGSUSED*/ 337void 338intr_thread(struct regs *regs, uint64_t iv_p, uint_t pil) 339{} 340 341#else /* lint */ 342 343#define INTRCNT_LIMIT 16 344 345/* 346 * Handle an interrupt in a new thread. 347 * Entry: 348 * %o0 = pointer to regs structure 349 * %o1 = pointer to current intr_vec_t (iv) to be processed 350 * %o2 = pil 351 * %sp = on current thread's kernel stack 352 * %o7 = return linkage to trap code 353 * %g7 = current thread 354 * %pstate = normal globals, interrupts enabled, 355 * privileged, fp disabled 356 * %pil = DISP_LEVEL 357 * 358 * Register Usage 359 * %l0 = return linkage 360 * %l1 = pil 361 * %l2 - %l3 = scratch 362 * %l4 - %l7 = reserved for sys_trap 363 * %o2 = cpu 364 * %o3 = intr thread 365 * %o0 = scratch 366 * %o4 - %o5 = scratch 367 */ 368 ENTRY_NP(intr_thread) 369 mov %o7, %l0 370 mov %o2, %l1 371 ! 372 ! See if we are interrupting another interrupt thread. 373 ! 374 lduh [THREAD_REG + T_FLAGS], %o3 375 andcc %o3, T_INTR_THREAD, %g0 376 bz,pt %xcc, 1f 377 ldn [THREAD_REG + T_CPU], %o2 ! delay - load CPU pointer 378 379 ! We have interrupted an interrupt thread. Take a timestamp, 380 ! compute its interval, and update its cumulative counter. 381 add THREAD_REG, T_INTR_START, %o5 3820: 383 ldx [%o5], %o3 384 brz,pn %o3, 1f 385 ! We came in on top of an interrupt thread that had no timestamp. 386 ! This could happen if, for instance, an interrupt thread which had 387 ! previously blocked is being set up to run again in resume(), but 388 ! resume() hasn't yet stored a timestamp for it. Or, it could be in 389 ! swtch() after its slice has been accounted for. 390 ! Only account for the time slice if the starting timestamp is non-zero. 391 RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__) 392 sub %o4, %o3, %o4 ! o4 has interval 393 394 ! A high-level interrupt in current_thread() interrupting here 395 ! will account for the interrupted thread's time slice, but 396 ! only if t_intr_start is non-zero. Since this code is going to account 397 ! for the time slice, we want to "atomically" load the thread's 398 ! starting timestamp, calculate the interval with %tick, and zero 399 ! its starting timestamp. 400 ! To do this, we do a casx on the t_intr_start field, and store 0 to it. 401 ! If it has changed since we loaded it above, we need to re-compute the 402 ! interval, since a changed t_intr_start implies current_thread placed 403 ! a new, later timestamp there after running a high-level interrupt, 404 ! and the %tick val in %o4 had become stale. 405 mov %g0, %l2 406 casx [%o5], %o3, %l2 407 408 ! If %l2 == %o3, our casx was successful. If not, the starting timestamp 409 ! changed between loading it (after label 0b) and computing the 410 ! interval above. 411 cmp %l2, %o3 412 bne,pn %xcc, 0b 413 414 ! Check for Energy Star mode 415 lduh [%o2 + CPU_DIVISOR], %l2 ! delay -- %l2 = clock divisor 416 cmp %l2, 1 417 bg,a,pn %xcc, 2f 418 mulx %o4, %l2, %o4 ! multiply interval by clock divisor iff > 1 4192: 420 ! We now know that a valid interval for the interrupted interrupt 421 ! thread is in %o4. Update its cumulative counter. 422 ldub [THREAD_REG + T_PIL], %l3 ! load PIL 423 sllx %l3, 4, %l3 ! convert PIL index to byte offset 424 add %l3, CPU_MCPU, %l3 ! CPU_INTRSTAT is too big for use 425 add %l3, MCPU_INTRSTAT, %l3 ! as const, add offsets separately 426 ldx [%o2 + %l3], %o5 ! old counter in o5 427 add %o5, %o4, %o5 ! new counter in o5 428 stx %o5, [%o2 + %l3] ! store new counter 429 430 ! Also update intracct[] 431 lduh [%o2 + CPU_MSTATE], %l3 432 sllx %l3, 3, %l3 433 add %l3, CPU_INTRACCT, %l3 434 add %l3, %o2, %l3 4350: 436 ldx [%l3], %o5 437 add %o5, %o4, %o3 438 casx [%l3], %o5, %o3 439 cmp %o5, %o3 440 bne,pn %xcc, 0b 441 nop 442 4431: 444 ! 445 ! Get set to run interrupt thread. 446 ! There should always be an interrupt thread since we allocate one 447 ! for each level on the CPU. 448 ! 449 ! Note that the code in kcpc_overflow_intr -relies- on the ordering 450 ! of events here -- in particular that t->t_lwp of the interrupt thread 451 ! is set to the pinned thread *before* curthread is changed. 452 ! 453 ldn [%o2 + CPU_INTR_THREAD], %o3 ! interrupt thread pool 454 ldn [%o3 + T_LINK], %o4 ! unlink thread from CPU's list 455 stn %o4, [%o2 + CPU_INTR_THREAD] 456 ! 457 ! Set bit for this level in CPU's active interrupt bitmask. 458 ! 459 ld [%o2 + CPU_INTR_ACTV], %o5 460 mov 1, %o4 461 sll %o4, %l1, %o4 462#ifdef DEBUG 463 ! 464 ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL))) 465 ! 466 andcc %o5, %o4, %g0 467 bz,pt %xcc, 0f 468 nop 469 ! Do not call panic if a panic is already in progress. 470 sethi %hi(panic_quiesce), %l2 471 ld [%l2 + %lo(panic_quiesce)], %l2 472 brnz,pn %l2, 0f 473 nop 474 sethi %hi(intr_thread_actv_bit_set), %o0 475 call panic 476 or %o0, %lo(intr_thread_actv_bit_set), %o0 4770: 478#endif /* DEBUG */ 479 or %o5, %o4, %o5 480 st %o5, [%o2 + CPU_INTR_ACTV] 481 ! 482 ! Consider the new thread part of the same LWP so that 483 ! window overflow code can find the PCB. 484 ! 485 ldn [THREAD_REG + T_LWP], %o4 486 stn %o4, [%o3 + T_LWP] 487 ! 488 ! Threads on the interrupt thread free list could have state already 489 ! set to TS_ONPROC, but it helps in debugging if they're TS_FREE 490 ! Could eliminate the next two instructions with a little work. 491 ! 492 mov TS_ONPROC, %o4 493 st %o4, [%o3 + T_STATE] 494 ! 495 ! Push interrupted thread onto list from new thread. 496 ! Set the new thread as the current one. 497 ! Set interrupted thread's T_SP because if it is the idle thread, 498 ! resume may use that stack between threads. 499 ! 500 stn %o7, [THREAD_REG + T_PC] ! mark pc for resume 501 stn %sp, [THREAD_REG + T_SP] ! mark stack for resume 502 stn THREAD_REG, [%o3 + T_INTR] ! push old thread 503 stn %o3, [%o2 + CPU_THREAD] ! set new thread 504 mov %o3, THREAD_REG ! set global curthread register 505 ldn [%o3 + T_STACK], %o4 ! interrupt stack pointer 506 sub %o4, STACK_BIAS, %sp 507 ! 508 ! Initialize thread priority level from intr_pri 509 ! 510 sethi %hi(intr_pri), %o4 511 ldsh [%o4 + %lo(intr_pri)], %o4 ! grab base interrupt priority 512 add %l1, %o4, %o4 ! convert level to dispatch priority 513 sth %o4, [THREAD_REG + T_PRI] 514 stub %l1, [THREAD_REG + T_PIL] ! save pil for intr_passivate 515 516 ! Store starting timestamp in thread structure. 517 add THREAD_REG, T_INTR_START, %o3 5181: 519 ldx [%o3], %o5 520 RD_CLOCK_TICK(%o4,%l2,%l3,__LINE__) 521 casx [%o3], %o5, %o4 522 cmp %o4, %o5 523 ! If a high-level interrupt occurred while we were attempting to store 524 ! the timestamp, try again. 525 bne,pn %xcc, 1b 526 nop 527 528 wrpr %g0, %l1, %pil ! lower %pil to new level 529 ! 530 ! Fast event tracing. 531 ! 532 ld [%o2 + CPU_FTRACE_STATE], %o4 ! %o2 = curthread->t_cpu 533 btst FTRACE_ENABLED, %o4 534 be,pt %icc, 1f ! skip if ftrace disabled 535 mov %l1, %o5 536 ! 537 ! Tracing is enabled - write the trace entry. 538 ! 539 save %sp, -SA(MINFRAME), %sp 540 set ftrace_intr_thread_format_str, %o0 541 mov %i0, %o1 542 mov %i1, %o2 543 mov %i5, %o3 544 call ftrace_3 545 ldn [%i0 + PC_OFF], %o4 546 restore 5471: 548 ! 549 ! call the handler 550 ! 551 SERVE_INTR_PRE(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0) 552 ! 553 ! %o0 and %o1 are now available as scratch registers. 554 ! 5550: 556 SERVE_INTR(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0) 557 ! 558 ! If %o3 is set, we must call serve_intr_next, and both %l1 and %o3 559 ! must be preserved. %l1 holds our pil, %l3 holds our inum. 560 ! 561 ! Note: %l1 is the pil level we're processing, but we may have a 562 ! higher effective pil because a higher-level interrupt may have 563 ! blocked. 564 ! 565 wrpr %g0, DISP_LEVEL, %pil 566 ! 567 ! Take timestamp, compute interval, update cumulative counter. 568 ! 569 add THREAD_REG, T_INTR_START, %o5 5701: 571 ldx [%o5], %o0 572#ifdef DEBUG 573 brnz %o0, 9f 574 nop 575 ! Do not call panic if a panic is already in progress. 576 sethi %hi(panic_quiesce), %o1 577 ld [%o1 + %lo(panic_quiesce)], %o1 578 brnz,pn %o1, 9f 579 nop 580 sethi %hi(intr_thread_t_intr_start_zero), %o0 581 call panic 582 or %o0, %lo(intr_thread_t_intr_start_zero), %o0 5839: 584#endif /* DEBUG */ 585 RD_CLOCK_TICK(%o1,%l2,%l3,__LINE__) 586 sub %o1, %o0, %l2 ! l2 has interval 587 ! 588 ! The general outline of what the code here does is: 589 ! 1. load t_intr_start, %tick, and calculate the delta 590 ! 2. replace t_intr_start with %tick (if %o3 is set) or 0. 591 ! 592 ! The problem is that a high-level interrupt could arrive at any time. 593 ! It will account for (%tick - t_intr_start) for us when it starts, 594 ! unless we have set t_intr_start to zero, and then set t_intr_start 595 ! to a new %tick when it finishes. To account for this, our first step 596 ! is to load t_intr_start and the last is to use casx to store the new 597 ! t_intr_start. This guarantees atomicity in reading t_intr_start, 598 ! reading %tick, and updating t_intr_start. 599 ! 600 movrz %o3, %g0, %o1 601 casx [%o5], %o0, %o1 602 cmp %o0, %o1 603 bne,pn %xcc, 1b 604 ! 605 ! Check for Energy Star mode 606 ! 607 lduh [%o2 + CPU_DIVISOR], %o0 ! delay -- %o0 = clock divisor 608 cmp %o0, 1 609 bg,a,pn %xcc, 2f 610 mulx %l2, %o0, %l2 ! multiply interval by clock divisor iff > 1 6112: 612 ! 613 ! Update cpu_intrstat. If o3 is set then we will be processing another 614 ! interrupt. Above we have set t_intr_start to %tick, not 0. This 615 ! means a high-level interrupt can arrive and update the same stats 616 ! we're updating. Need to use casx. 617 ! 618 sllx %l1, 4, %o1 ! delay - PIL as byte offset 619 add %o1, CPU_MCPU, %o1 ! CPU_INTRSTAT const too big 620 add %o1, MCPU_INTRSTAT, %o1 ! add parts separately 621 add %o1, %o2, %o1 6221: 623 ldx [%o1], %o5 ! old counter in o5 624 add %o5, %l2, %o0 ! new counter in o0 625 stx %o0, [%o1 + 8] ! store into intrstat[pil][1] 626 casx [%o1], %o5, %o0 ! and into intrstat[pil][0] 627 cmp %o5, %o0 628 bne,pn %xcc, 1b 629 nop 630 631 ! Also update intracct[] 632 lduh [%o2 + CPU_MSTATE], %o1 633 sllx %o1, 3, %o1 634 add %o1, CPU_INTRACCT, %o1 635 add %o1, %o2, %o1 6361: 637 ldx [%o1], %o5 638 add %o5, %l2, %o0 639 casx [%o1], %o5, %o0 640 cmp %o5, %o0 641 bne,pn %xcc, 1b 642 nop 643 644 ! 645 ! Don't keep a pinned process pinned indefinitely. Bump cpu_intrcnt 646 ! for each interrupt handler we invoke. If we hit INTRCNT_LIMIT, then 647 ! we've crossed the threshold and we should unpin the pinned threads 648 ! by preempt()ing ourselves, which will bubble up the t_intr chain 649 ! until hitting the non-interrupt thread, which will then in turn 650 ! preempt itself allowing the interrupt processing to resume. Finally, 651 ! the scheduler takes over and picks the next thread to run. 652 ! 653 ! If our CPU is quiesced, we cannot preempt because the idle thread 654 ! won't ever re-enter the scheduler, and the interrupt will be forever 655 ! blocked. 656 ! 657 ! If t_intr is NULL, we're not pinning anyone, so we use a simpler 658 ! algorithm. Just check for cpu_kprunrun, and if set then preempt. 659 ! This insures we enter the scheduler if a higher-priority thread 660 ! has become runnable. 661 ! 662 lduh [%o2 + CPU_FLAGS], %o5 ! don't preempt if quiesced 663 andcc %o5, CPU_QUIESCED, %g0 664 bnz,pn %xcc, 1f 665 666 ldn [THREAD_REG + T_INTR], %o5 ! pinning anything? 667 brz,pn %o5, 3f ! if not, don't inc intrcnt 668 669 ldub [%o2 + CPU_INTRCNT], %o5 ! delay - %o5 = cpu_intrcnt 670 inc %o5 671 cmp %o5, INTRCNT_LIMIT ! have we hit the limit? 672 bl,a,pt %xcc, 1f ! no preempt if < INTRCNT_LIMIT 673 stub %o5, [%o2 + CPU_INTRCNT] ! delay annul - inc CPU_INTRCNT 674 bg,pn %xcc, 2f ! don't inc stats again 675 ! 676 ! We've reached the limit. Set cpu_intrcnt and cpu_kprunrun, and do 677 ! CPU_STATS_ADDQ(cp, sys, intrunpin, 1). Then call preempt. 678 ! 679 mov 1, %o4 ! delay 680 stub %o4, [%o2 + CPU_KPRUNRUN] 681 ldx [%o2 + CPU_STATS_SYS_INTRUNPIN], %o4 682 inc %o4 683 stx %o4, [%o2 + CPU_STATS_SYS_INTRUNPIN] 684 ba 2f 685 stub %o5, [%o2 + CPU_INTRCNT] ! delay 6863: 687 ! Code for t_intr == NULL 688 ldub [%o2 + CPU_KPRUNRUN], %o5 689 brz,pt %o5, 1f ! don't preempt unless kprunrun 6902: 691 ! Time to call preempt 692 mov %o2, %l3 ! delay - save %o2 693 call preempt 694 mov %o3, %l2 ! delay - save %o3. 695 mov %l3, %o2 ! restore %o2 696 mov %l2, %o3 ! restore %o3 697 wrpr %g0, DISP_LEVEL, %pil ! up from cpu_base_spl 6981: 699 ! 700 ! Do we need to call serve_intr_next and do this again? 701 ! 702 brz,a,pt %o3, 0f 703 ld [%o2 + CPU_INTR_ACTV], %o5 ! delay annulled 704 ! 705 ! Restore %pil before calling serve_intr() again. We must check 706 ! CPU_BASE_SPL and set %pil to max(our-pil, CPU_BASE_SPL) 707 ! 708 ld [%o2 + CPU_BASE_SPL], %o4 709 cmp %o4, %l1 710 movl %xcc, %l1, %o4 711 wrpr %g0, %o4, %pil 712 SERVE_INTR_NEXT(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0) 713 ba 0b ! compute new stats 714 nop 7150: 716 ! 717 ! Clear bit for this level in CPU's interrupt active bitmask. 718 ! 719 mov 1, %o4 720 sll %o4, %l1, %o4 721#ifdef DEBUG 722 ! 723 ! ASSERT(CPU->cpu_intr_actv & (1 << PIL)) 724 ! 725 andcc %o4, %o5, %g0 726 bnz,pt %xcc, 0f 727 nop 728 ! Do not call panic if a panic is already in progress. 729 sethi %hi(panic_quiesce), %l2 730 ld [%l2 + %lo(panic_quiesce)], %l2 731 brnz,pn %l2, 0f 732 nop 733 sethi %hi(intr_thread_actv_bit_not_set), %o0 734 call panic 735 or %o0, %lo(intr_thread_actv_bit_not_set), %o0 7360: 737#endif /* DEBUG */ 738 andn %o5, %o4, %o5 739 st %o5, [%o2 + CPU_INTR_ACTV] 740 ! 741 ! If there is still an interrupted thread underneath this one, 742 ! then the interrupt was never blocked and the return is fairly 743 ! simple. Otherwise jump to intr_thread_exit. 744 ! 745 ldn [THREAD_REG + T_INTR], %o4 ! pinned thread 746 brz,pn %o4, intr_thread_exit ! branch if none 747 nop 748 ! 749 ! link the thread back onto the interrupt thread pool 750 ! 751 ldn [%o2 + CPU_INTR_THREAD], %o3 752 stn %o3, [THREAD_REG + T_LINK] 753 stn THREAD_REG, [%o2 + CPU_INTR_THREAD] 754 ! 755 ! set the thread state to free so kernel debuggers don't see it 756 ! 757 mov TS_FREE, %o5 758 st %o5, [THREAD_REG + T_STATE] 759 ! 760 ! Switch back to the interrupted thread and return 761 ! 762 stn %o4, [%o2 + CPU_THREAD] 763 membar #StoreLoad ! sync with mutex_exit() 764 mov %o4, THREAD_REG 765 766 ! If we pinned an interrupt thread, store its starting timestamp. 767 lduh [THREAD_REG + T_FLAGS], %o5 768 andcc %o5, T_INTR_THREAD, %g0 769 bz,pt %xcc, 1f 770 ldn [THREAD_REG + T_SP], %sp ! delay - restore %sp 771 772 add THREAD_REG, T_INTR_START, %o3 ! o3 has &curthread->t_intr_star 7730: 774 ldx [%o3], %o4 ! o4 = t_intr_start before 775 RD_CLOCK_TICK(%o5,%l2,%l3,__LINE__) 776 casx [%o3], %o4, %o5 ! put o5 in ts if o4 == ts after 777 cmp %o4, %o5 778 ! If a high-level interrupt occurred while we were attempting to store 779 ! the timestamp, try again. 780 bne,pn %xcc, 0b 781 ldn [THREAD_REG + T_SP], %sp ! delay - restore %sp 7821: 783 ! If the thread being restarted isn't pinning anyone, and no interrupts 784 ! are pending, zero out cpu_intrcnt 785 ldn [THREAD_REG + T_INTR], %o4 786 brnz,pn %o4, 2f 787 rd SOFTINT, %o4 ! delay 788 set SOFTINT_MASK, %o5 789 andcc %o4, %o5, %g0 790 bz,a,pt %xcc, 2f 791 stub %g0, [%o2 + CPU_INTRCNT] ! delay annul 7922: 793 jmp %l0 + 8 794 nop 795 SET_SIZE(intr_thread) 796 /* Not Reached */ 797 798 ! 799 ! An interrupt returned on what was once (and still might be) 800 ! an interrupt thread stack, but the interrupted process is no longer 801 ! there. This means the interrupt must have blocked. 802 ! 803 ! There is no longer a thread under this one, so put this thread back 804 ! on the CPU's free list and resume the idle thread which will dispatch 805 ! the next thread to run. 806 ! 807 ! All traps below DISP_LEVEL are disabled here, but the mondo interrupt 808 ! is enabled. 809 ! 810 ENTRY_NP(intr_thread_exit) 811#ifdef TRAPTRACE 812 rdpr %pstate, %l2 813 andn %l2, PSTATE_IE | PSTATE_AM, %o4 814 wrpr %g0, %o4, %pstate ! cpu to known state 815 TRACE_PTR(%o4, %o5) 816 GET_TRACE_TICK(%o5, %o0) 817 stxa %o5, [%o4 + TRAP_ENT_TICK]%asi 818 TRACE_SAVE_TL_GL_REGS(%o4, %o5) 819 set TT_INTR_EXIT, %o5 820 stha %o5, [%o4 + TRAP_ENT_TT]%asi 821 stna %g0, [%o4 + TRAP_ENT_TPC]%asi 822 stxa %g0, [%o4 + TRAP_ENT_TSTATE]%asi 823 stna %sp, [%o4 + TRAP_ENT_SP]%asi 824 stna THREAD_REG, [%o4 + TRAP_ENT_TR]%asi 825 ld [%o2 + CPU_BASE_SPL], %o5 826 stna %o5, [%o4 + TRAP_ENT_F1]%asi 827 stna %g0, [%o4 + TRAP_ENT_F2]%asi 828 stna %g0, [%o4 + TRAP_ENT_F3]%asi 829 stna %g0, [%o4 + TRAP_ENT_F4]%asi 830 TRACE_NEXT(%o4, %o5, %o0) 831 wrpr %g0, %l2, %pstate 832#endif /* TRAPTRACE */ 833 ! cpu_stats.sys.intrblk++ 834 ldx [%o2 + CPU_STATS_SYS_INTRBLK], %o4 835 inc %o4 836 stx %o4, [%o2 + CPU_STATS_SYS_INTRBLK] 837 ! 838 ! Put thread back on the interrupt thread list. 839 ! 840 841 ! 842 ! Set the CPU's base SPL level. 843 ! 844#ifdef DEBUG 845 ! 846 ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL))) 847 ! 848 ld [%o2 + CPU_INTR_ACTV], %o5 849 mov 1, %o4 850 sll %o4, %l1, %o4 851 and %o5, %o4, %o4 852 brz,pt %o4, 0f 853 nop 854 ! Do not call panic if a panic is already in progress. 855 sethi %hi(panic_quiesce), %l2 856 ld [%l2 + %lo(panic_quiesce)], %l2 857 brnz,pn %l2, 0f 858 nop 859 sethi %hi(intr_thread_exit_actv_bit_set), %o0 860 call panic 861 or %o0, %lo(intr_thread_exit_actv_bit_set), %o0 8620: 863#endif /* DEBUG */ 864 call _intr_set_spl ! set CPU's base SPL level 865 ld [%o2 + CPU_INTR_ACTV], %o5 ! delay - load active mask 866 ! 867 ! set the thread state to free so kernel debuggers don't see it 868 ! 869 mov TS_FREE, %o4 870 st %o4, [THREAD_REG + T_STATE] 871 ! 872 ! Put thread on either the interrupt pool or the free pool and 873 ! call swtch() to resume another thread. 874 ! 875 ldn [%o2 + CPU_INTR_THREAD], %o5 ! get list pointer 876 stn %o5, [THREAD_REG + T_LINK] 877 call swtch ! switch to best thread 878 stn THREAD_REG, [%o2 + CPU_INTR_THREAD] ! delay - put thread on list 879 ba,a,pt %xcc, . ! swtch() shouldn't return 880 SET_SIZE(intr_thread_exit) 881 882 .global ftrace_intr_thread_format_str 883ftrace_intr_thread_format_str: 884 .asciz "intr_thread(): regs=0x%lx, int=0x%lx, pil=0x%lx" 885#ifdef DEBUG 886intr_thread_actv_bit_set: 887 .asciz "intr_thread(): cpu_intr_actv bit already set for PIL" 888intr_thread_actv_bit_not_set: 889 .asciz "intr_thread(): cpu_intr_actv bit not set for PIL" 890intr_thread_exit_actv_bit_set: 891 .asciz "intr_thread_exit(): cpu_intr_actv bit erroneously set for PIL" 892intr_thread_t_intr_start_zero: 893 .asciz "intr_thread(): t_intr_start zero upon handler return" 894#endif /* DEBUG */ 895#endif /* lint */ 896 897#if defined(lint) 898 899/* 900 * Handle an interrupt in the current thread 901 * Entry: 902 * %o0 = pointer to regs structure 903 * %o1 = pointer to current intr_vec_t (iv) to be processed 904 * %o2 = pil 905 * %sp = on current thread's kernel stack 906 * %o7 = return linkage to trap code 907 * %g7 = current thread 908 * %pstate = normal globals, interrupts enabled, 909 * privileged, fp disabled 910 * %pil = PIL_MAX 911 * 912 * Register Usage 913 * %l0 = return linkage 914 * %l1 = old stack 915 * %l2 - %l3 = scratch 916 * %l4 - %l7 = reserved for sys_trap 917 * %o3 = cpu 918 * %o0 = scratch 919 * %o4 - %o5 = scratch 920 */ 921/* ARGSUSED */ 922void 923current_thread(struct regs *regs, uint64_t iv_p, uint_t pil) 924{} 925 926#else /* lint */ 927 928 ENTRY_NP(current_thread) 929 930 mov %o7, %l0 931 ldn [THREAD_REG + T_CPU], %o3 932 933 ldn [THREAD_REG + T_ONFAULT], %l2 934 brz,pt %l2, no_onfault ! branch if no onfault label set 935 nop 936 stn %g0, [THREAD_REG + T_ONFAULT]! clear onfault label 937 ldn [THREAD_REG + T_LOFAULT], %l3 938 stn %g0, [THREAD_REG + T_LOFAULT]! clear lofault data 939 940 sub %o2, LOCK_LEVEL + 1, %o5 941 sll %o5, CPTRSHIFT, %o5 942 add %o5, CPU_OFD, %o4 ! %o4 has on_fault data offset 943 stn %l2, [%o3 + %o4] ! save onfault label for pil %o2 944 add %o5, CPU_LFD, %o4 ! %o4 has lofault data offset 945 stn %l3, [%o3 + %o4] ! save lofault data for pil %o2 946 947no_onfault: 948 ldn [THREAD_REG + T_ONTRAP], %l2 949 brz,pt %l2, 6f ! branch if no on_trap protection 950 nop 951 stn %g0, [THREAD_REG + T_ONTRAP]! clear on_trap protection 952 sub %o2, LOCK_LEVEL + 1, %o5 953 sll %o5, CPTRSHIFT, %o5 954 add %o5, CPU_OTD, %o4 ! %o4 has on_trap data offset 955 stn %l2, [%o3 + %o4] ! save on_trap label for pil %o2 956 957 ! 958 ! Set bit for this level in CPU's active interrupt bitmask. 959 ! 9606: ld [%o3 + CPU_INTR_ACTV], %o5 ! o5 has cpu_intr_actv b4 chng 961 mov 1, %o4 962 sll %o4, %o2, %o4 ! construct mask for level 963#ifdef DEBUG 964 ! 965 ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL))) 966 ! 967 andcc %o5, %o4, %g0 968 bz,pt %xcc, 0f 969 nop 970 ! Do not call panic if a panic is already in progress. 971 sethi %hi(panic_quiesce), %l2 972 ld [%l2 + %lo(panic_quiesce)], %l2 973 brnz,pn %l2, 0f 974 nop 975 sethi %hi(current_thread_actv_bit_set), %o0 976 call panic 977 or %o0, %lo(current_thread_actv_bit_set), %o0 9780: 979#endif /* DEBUG */ 980 or %o5, %o4, %o4 981 ! 982 ! See if we are interrupting another high-level interrupt. 983 ! 984 srl %o5, LOCK_LEVEL + 1, %o5 ! only look at high-level bits 985 brz,pt %o5, 1f 986 st %o4, [%o3 + CPU_INTR_ACTV] ! delay - store active mask 987 ! 988 ! We have interrupted another high-level interrupt. Find its PIL, 989 ! compute the interval it ran for, and update its cumulative counter. 990 ! 991 ! Register usage: 992 993 ! o2 = PIL of this interrupt 994 ! o5 = high PIL bits of INTR_ACTV (not including this PIL) 995 ! l1 = bitmask used to find other active high-level PIL 996 ! o4 = index of bit set in l1 997 ! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the 998 ! interrupted high-level interrupt. 999 ! Create mask for cpu_intr_actv. Begin by looking for bits set 1000 ! at one level below the current PIL. Since %o5 contains the active 1001 ! mask already shifted right by (LOCK_LEVEL + 1), we start by looking 1002 ! at bit (current_pil - (LOCK_LEVEL + 2)). 1003 sub %o2, LOCK_LEVEL + 2, %o4 1004 mov 1, %l1 1005 sll %l1, %o4, %l1 10062: 1007#ifdef DEBUG 1008 ! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge) 1009 brnz,pt %l1, 9f 1010 nop 1011 1012 ! Don't panic if a panic is already in progress. 1013 sethi %hi(panic_quiesce), %l3 1014 ld [%l3 + %lo(panic_quiesce)], %l3 1015 brnz,pn %l3, 9f 1016 nop 1017 sethi %hi(current_thread_nested_PIL_not_found), %o0 1018 call panic 1019 or %o0, %lo(current_thread_nested_PIL_not_found), %o0 10209: 1021#endif /* DEBUG */ 1022 andcc %l1, %o5, %g0 ! test mask against high-level bits of 1023 bnz %xcc, 3f ! cpu_intr_actv 1024 nop 1025 srl %l1, 1, %l1 ! No match. Try next lower PIL. 1026 ba,pt %xcc, 2b 1027 sub %o4, 1, %o4 ! delay - decrement PIL 10283: 1029 sll %o4, 3, %o4 ! index to byte offset 1030 add %o4, CPU_MCPU, %l1 ! CPU_PIL_HIGH_START is too large 1031 add %l1, MCPU_PIL_HIGH_START, %l1 1032 ldx [%o3 + %l1], %l3 ! load starting timestamp 1033#ifdef DEBUG 1034 brnz,pt %l3, 9f 1035 nop 1036 ! Don't panic if a panic is already in progress. 1037 sethi %hi(panic_quiesce), %l1 1038 ld [%l1 + %lo(panic_quiesce)], %l1 1039 brnz,pn %l1, 9f 1040 nop 1041 srl %o4, 3, %o1 ! Find interrupted PIL for panic 1042 add %o1, LOCK_LEVEL + 1, %o1 1043 sethi %hi(current_thread_nested_pil_zero), %o0 1044 call panic 1045 or %o0, %lo(current_thread_nested_pil_zero), %o0 10469: 1047#endif /* DEBUG */ 1048 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%l1, %l2) 1049 sub %l1, %l3, %l3 ! interval in %l3 1050 ! 1051 ! Check for Energy Star mode 1052 ! 1053 lduh [%o3 + CPU_DIVISOR], %l1 ! %l1 = clock divisor 1054 cmp %l1, 1 1055 bg,a,pn %xcc, 2f 1056 mulx %l3, %l1, %l3 ! multiply interval by clock divisor iff > 1 10572: 1058 ! 1059 ! We need to find the CPU offset of the cumulative counter. We start 1060 ! with %o4, which has (PIL - (LOCK_LEVEL + 1)) * 8. We need PIL * 16, 1061 ! so we shift left 1, then add (LOCK_LEVEL + 1) * 16, which is 1062 ! CPU_INTRSTAT_LOW_PIL_OFFSET. 1063 ! 1064 sll %o4, 1, %o4 1065 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT const too large 1066 add %o4, MCPU_INTRSTAT, %o4 ! add parts separately 1067 add %o4, CPU_INTRSTAT_LOW_PIL_OFFSET, %o4 1068 ldx [%o3 + %o4], %l1 ! old counter in l1 1069 add %l1, %l3, %l1 ! new counter in l1 1070 stx %l1, [%o3 + %o4] ! store new counter 1071 1072 ! Also update intracct[] 1073 lduh [%o3 + CPU_MSTATE], %o4 1074 sllx %o4, 3, %o4 1075 add %o4, CPU_INTRACCT, %o4 1076 ldx [%o3 + %o4], %l1 1077 add %l1, %l3, %l1 1078 ! Another high-level interrupt is active below this one, so 1079 ! there is no need to check for an interrupt thread. That will be 1080 ! done by the lowest priority high-level interrupt active. 1081 ba,pt %xcc, 5f 1082 stx %l1, [%o3 + %o4] ! delay - store new counter 10831: 1084 ! If we haven't interrupted another high-level interrupt, we may be 1085 ! interrupting a low level interrupt thread. If so, compute its interval 1086 ! and update its cumulative counter. 1087 lduh [THREAD_REG + T_FLAGS], %o4 1088 andcc %o4, T_INTR_THREAD, %g0 1089 bz,pt %xcc, 4f 1090 nop 1091 1092 ! We have interrupted an interrupt thread. Take timestamp, compute 1093 ! interval, update cumulative counter. 1094 1095 ! Check t_intr_start. If it is zero, either intr_thread() or 1096 ! current_thread() (at a lower PIL, of course) already did 1097 ! the accounting for the underlying interrupt thread. 1098 ldx [THREAD_REG + T_INTR_START], %o5 1099 brz,pn %o5, 4f 1100 nop 1101 1102 stx %g0, [THREAD_REG + T_INTR_START] 1103 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2) 1104 sub %o4, %o5, %o5 ! o5 has the interval 1105 1106 ! Check for Energy Star mode 1107 lduh [%o3 + CPU_DIVISOR], %o4 ! %o4 = clock divisor 1108 cmp %o4, 1 1109 bg,a,pn %xcc, 2f 1110 mulx %o5, %o4, %o5 ! multiply interval by clock divisor iff > 1 11112: 1112 ldub [THREAD_REG + T_PIL], %o4 1113 sllx %o4, 4, %o4 ! PIL index to byte offset 1114 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT const too large 1115 add %o4, MCPU_INTRSTAT, %o4 ! add parts separately 1116 ldx [%o3 + %o4], %l2 ! old counter in l2 1117 add %l2, %o5, %l2 ! new counter in l2 1118 stx %l2, [%o3 + %o4] ! store new counter 1119 1120 ! Also update intracct[] 1121 lduh [%o3 + CPU_MSTATE], %o4 1122 sllx %o4, 3, %o4 1123 add %o4, CPU_INTRACCT, %o4 1124 ldx [%o3 + %o4], %l2 1125 add %l2, %o5, %l2 1126 stx %l2, [%o3 + %o4] 11274: 1128 ! 1129 ! Handle high-level interrupts on separate interrupt stack. 1130 ! No other high-level interrupts are active, so switch to int stack. 1131 ! 1132 mov %sp, %l1 1133 ldn [%o3 + CPU_INTR_STACK], %l3 1134 sub %l3, STACK_BIAS, %sp 1135 11365: 1137#ifdef DEBUG 1138 ! 1139 ! ASSERT(%o2 > LOCK_LEVEL) 1140 ! 1141 cmp %o2, LOCK_LEVEL 1142 bg,pt %xcc, 3f 1143 nop 1144 mov CE_PANIC, %o0 1145 sethi %hi(current_thread_wrong_pil), %o1 1146 call cmn_err ! %o2 has the %pil already 1147 or %o1, %lo(current_thread_wrong_pil), %o1 1148#endif 11493: 1150 ! Store starting timestamp for this PIL in CPU structure at 1151 ! cpu.cpu_m.pil_high_start[PIL - (LOCK_LEVEL + 1)] 1152 sub %o2, LOCK_LEVEL + 1, %o4 ! convert PIL to array index 1153 sllx %o4, 3, %o4 ! index to byte offset 1154 add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large 1155 add %o4, MCPU_PIL_HIGH_START, %o4 1156 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %l2) 1157 stx %o5, [%o3 + %o4] 1158 1159 wrpr %g0, %o2, %pil ! enable interrupts 1160 1161 ! 1162 ! call the handler 1163 ! 1164 SERVE_INTR_PRE(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0) 11651: 1166 SERVE_INTR(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0) 1167 1168 brz,a,pt %o2, 0f ! if %o2, more intrs await 1169 rdpr %pil, %o2 ! delay annulled 1170 SERVE_INTR_NEXT(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0) 1171 ba 1b 1172 nop 11730: 1174 wrpr %g0, PIL_MAX, %pil ! disable interrupts (1-15) 1175 1176 cmp %o2, PIL_15 1177 bne,pt %xcc, 3f 1178 nop 1179 1180 sethi %hi(cpc_level15_inum), %o1 1181 ldx [%o1 + %lo(cpc_level15_inum)], %o1 ! arg for intr_enqueue_req 1182 brz %o1, 3f 1183 nop 1184 1185 rdpr %pstate, %g5 1186 andn %g5, PSTATE_IE, %g1 1187 wrpr %g0, %g1, %pstate ! Disable vec interrupts 1188 1189 call intr_enqueue_req ! preserves %g5 1190 mov PIL_15, %o0 1191 1192 ! clear perfcntr overflow 1193 mov 1, %o0 1194 sllx %o0, PIL_15, %o0 1195 wr %o0, CLEAR_SOFTINT 1196 1197 wrpr %g0, %g5, %pstate ! Enable vec interrupts 1198 11993: 1200 cmp %o2, PIL_14 1201 be tick_rtt ! cpu-specific tick processing 1202 nop 1203 .global current_thread_complete 1204current_thread_complete: 1205 ! 1206 ! Register usage: 1207 ! 1208 ! %l1 = stack pointer 1209 ! %l2 = CPU_INTR_ACTV >> (LOCK_LEVEL + 1) 1210 ! %o2 = PIL 1211 ! %o3 = CPU pointer 1212 ! %o4, %o5, %l3, %l4, %l5 = scratch 1213 ! 1214 ldn [THREAD_REG + T_CPU], %o3 1215 ! 1216 ! Clear bit for this level in CPU's interrupt active bitmask. 1217 ! 1218 ld [%o3 + CPU_INTR_ACTV], %l2 1219 mov 1, %o5 1220 sll %o5, %o2, %o5 1221#ifdef DEBUG 1222 ! 1223 ! ASSERT(CPU->cpu_intr_actv & (1 << PIL)) 1224 ! 1225 andcc %l2, %o5, %g0 1226 bnz,pt %xcc, 0f 1227 nop 1228 ! Do not call panic if a panic is already in progress. 1229 sethi %hi(panic_quiesce), %l2 1230 ld [%l2 + %lo(panic_quiesce)], %l2 1231 brnz,pn %l2, 0f 1232 nop 1233 sethi %hi(current_thread_actv_bit_not_set), %o0 1234 call panic 1235 or %o0, %lo(current_thread_actv_bit_not_set), %o0 12360: 1237#endif /* DEBUG */ 1238 andn %l2, %o5, %l2 1239 st %l2, [%o3 + CPU_INTR_ACTV] 1240 1241 ! Take timestamp, compute interval, update cumulative counter. 1242 sub %o2, LOCK_LEVEL + 1, %o4 ! PIL to array index 1243 sllx %o4, 3, %o4 ! index to byte offset 1244 add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large 1245 add %o4, MCPU_PIL_HIGH_START, %o4 1246 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o5, %o0) 1247 ldx [%o3 + %o4], %o0 1248#ifdef DEBUG 1249 ! ASSERT(cpu.cpu_m.pil_high_start[pil - (LOCK_LEVEL + 1)] != 0) 1250 brnz,pt %o0, 9f 1251 nop 1252 ! Don't panic if a panic is already in progress. 1253 sethi %hi(panic_quiesce), %l2 1254 ld [%l2 + %lo(panic_quiesce)], %l2 1255 brnz,pn %l2, 9f 1256 nop 1257 sethi %hi(current_thread_timestamp_zero), %o0 1258 call panic 1259 or %o0, %lo(current_thread_timestamp_zero), %o0 12609: 1261#endif /* DEBUG */ 1262 stx %g0, [%o3 + %o4] 1263 sub %o5, %o0, %o5 ! interval in o5 1264 1265 ! Check for Energy Star mode 1266 lduh [%o3 + CPU_DIVISOR], %o4 ! %o4 = clock divisor 1267 cmp %o4, 1 1268 bg,a,pn %xcc, 2f 1269 mulx %o5, %o4, %o5 ! multiply interval by clock divisor iff > 1 12702: 1271 sllx %o2, 4, %o4 ! PIL index to byte offset 1272 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT too large 1273 add %o4, MCPU_INTRSTAT, %o4 ! add parts separately 1274 ldx [%o3 + %o4], %o0 ! old counter in o0 1275 add %o0, %o5, %o0 ! new counter in o0 1276 stx %o0, [%o3 + %o4] ! store new counter 1277 1278 ! Also update intracct[] 1279 lduh [%o3 + CPU_MSTATE], %o4 1280 sllx %o4, 3, %o4 1281 add %o4, CPU_INTRACCT, %o4 1282 ldx [%o3 + %o4], %o0 1283 add %o0, %o5, %o0 1284 stx %o0, [%o3 + %o4] 1285 1286 ! 1287 ! get back on current thread's stack 1288 ! 1289 srl %l2, LOCK_LEVEL + 1, %l2 1290 tst %l2 ! any more high-level ints? 1291 movz %xcc, %l1, %sp 1292 ! 1293 ! Current register usage: 1294 ! o2 = PIL 1295 ! o3 = CPU pointer 1296 ! l0 = return address 1297 ! l2 = intr_actv shifted right 1298 ! 1299 bz,pt %xcc, 3f ! if l2 was zero, no more ints 1300 nop 1301 ! 1302 ! We found another high-level interrupt active below the one that just 1303 ! returned. Store a starting timestamp for it in the CPU structure. 1304 ! 1305 ! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the 1306 ! interrupted high-level interrupt. 1307 ! Create mask for cpu_intr_actv. Begin by looking for bits set 1308 ! at one level below the current PIL. Since %l2 contains the active 1309 ! mask already shifted right by (LOCK_LEVEL + 1), we start by looking 1310 ! at bit (current_pil - (LOCK_LEVEL + 2)). 1311 ! %l1 = mask, %o5 = index of bit set in mask 1312 ! 1313 mov 1, %l1 1314 sub %o2, LOCK_LEVEL + 2, %o5 1315 sll %l1, %o5, %l1 ! l1 = mask for level 13161: 1317#ifdef DEBUG 1318 ! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge) 1319 brnz,pt %l1, 9f 1320 nop 1321 sethi %hi(current_thread_nested_PIL_not_found), %o0 1322 call panic 1323 or %o0, %lo(current_thread_nested_PIL_not_found), %o0 13249: 1325#endif /* DEBUG */ 1326 andcc %l1, %l2, %g0 ! test mask against high-level bits of 1327 bnz %xcc, 2f ! cpu_intr_actv 1328 nop 1329 srl %l1, 1, %l1 ! No match. Try next lower PIL. 1330 ba,pt %xcc, 1b 1331 sub %o5, 1, %o5 ! delay - decrement PIL 13322: 1333 sll %o5, 3, %o5 ! convert array index to byte offset 1334 add %o5, CPU_MCPU, %o5 ! CPU_PIL_HIGH_START is too large 1335 add %o5, MCPU_PIL_HIGH_START, %o5 1336 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2) 1337 ! Another high-level interrupt is active below this one, so 1338 ! there is no need to check for an interrupt thread. That will be 1339 ! done by the lowest priority high-level interrupt active. 1340 ba,pt %xcc, 7f 1341 stx %o4, [%o3 + %o5] ! delay - store timestamp 13423: 1343 ! If we haven't interrupted another high-level interrupt, we may have 1344 ! interrupted a low level interrupt thread. If so, store a starting 1345 ! timestamp in its thread structure. 1346 lduh [THREAD_REG + T_FLAGS], %o4 1347 andcc %o4, T_INTR_THREAD, %g0 1348 bz,pt %xcc, 7f 1349 nop 1350 1351 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o4, %l2) 1352 stx %o4, [THREAD_REG + T_INTR_START] 1353 13547: 1355 sub %o2, LOCK_LEVEL + 1, %o4 1356 sll %o4, CPTRSHIFT, %o5 1357 1358 ! Check on_trap saved area and restore as needed 1359 add %o5, CPU_OTD, %o4 1360 ldn [%o3 + %o4], %l2 1361 brz,pt %l2, no_ontrp_restore 1362 nop 1363 stn %l2, [THREAD_REG + T_ONTRAP] ! restore 1364 stn %g0, [%o3 + %o4] ! clear 1365 1366no_ontrp_restore: 1367 ! Check on_fault saved area and restore as needed 1368 add %o5, CPU_OFD, %o4 1369 ldn [%o3 + %o4], %l2 1370 brz,pt %l2, 8f 1371 nop 1372 stn %l2, [THREAD_REG + T_ONFAULT] ! restore 1373 stn %g0, [%o3 + %o4] ! clear 1374 add %o5, CPU_LFD, %o4 1375 ldn [%o3 + %o4], %l2 1376 stn %l2, [THREAD_REG + T_LOFAULT] ! restore 1377 stn %g0, [%o3 + %o4] ! clear 1378 1379 13808: 1381 ! Enable interrupts and return 1382 jmp %l0 + 8 1383 wrpr %g0, %o2, %pil ! enable interrupts 1384 SET_SIZE(current_thread) 1385 1386 1387#ifdef DEBUG 1388current_thread_wrong_pil: 1389 .asciz "current_thread: unexpected pil level: %d" 1390current_thread_actv_bit_set: 1391 .asciz "current_thread(): cpu_intr_actv bit already set for PIL" 1392current_thread_actv_bit_not_set: 1393 .asciz "current_thread(): cpu_intr_actv bit not set for PIL" 1394current_thread_nested_pil_zero: 1395 .asciz "current_thread(): timestamp zero for nested PIL %d" 1396current_thread_timestamp_zero: 1397 .asciz "current_thread(): timestamp zero upon handler return" 1398current_thread_nested_PIL_not_found: 1399 .asciz "current_thread: couldn't find nested high-level PIL" 1400#endif /* DEBUG */ 1401#endif /* lint */ 1402 1403/* 1404 * Return a thread's interrupt level. 1405 * Since this isn't saved anywhere but in %l4 on interrupt entry, we 1406 * must dig it out of the save area. 1407 * 1408 * Caller 'swears' that this really is an interrupt thread. 1409 * 1410 * int 1411 * intr_level(t) 1412 * kthread_id_t t; 1413 */ 1414 1415#if defined(lint) 1416 1417/* ARGSUSED */ 1418int 1419intr_level(kthread_id_t t) 1420{ return (0); } 1421 1422#else /* lint */ 1423 1424 ENTRY_NP(intr_level) 1425 retl 1426 ldub [%o0 + T_PIL], %o0 ! return saved pil 1427 SET_SIZE(intr_level) 1428 1429#endif /* lint */ 1430 1431#if defined(lint) 1432 1433/* ARGSUSED */ 1434int 1435disable_pil_intr() 1436{ return (0); } 1437 1438#else /* lint */ 1439 1440 ENTRY_NP(disable_pil_intr) 1441 rdpr %pil, %o0 1442 retl 1443 wrpr %g0, PIL_MAX, %pil ! disable interrupts (1-15) 1444 SET_SIZE(disable_pil_intr) 1445 1446#endif /* lint */ 1447 1448#if defined(lint) 1449 1450/* ARGSUSED */ 1451void 1452enable_pil_intr(int pil_save) 1453{} 1454 1455#else /* lint */ 1456 1457 ENTRY_NP(enable_pil_intr) 1458 retl 1459 wrpr %o0, %pil 1460 SET_SIZE(enable_pil_intr) 1461 1462#endif /* lint */ 1463 1464#if defined(lint) 1465 1466/* ARGSUSED */ 1467uint_t 1468disable_vec_intr(void) 1469{ return (0); } 1470 1471#else /* lint */ 1472 1473 ENTRY_NP(disable_vec_intr) 1474 rdpr %pstate, %o0 1475 andn %o0, PSTATE_IE, %g1 1476 retl 1477 wrpr %g0, %g1, %pstate ! disable interrupt 1478 SET_SIZE(disable_vec_intr) 1479 1480#endif /* lint */ 1481 1482#if defined(lint) 1483 1484/* ARGSUSED */ 1485void 1486enable_vec_intr(uint_t pstate_save) 1487{} 1488 1489#else /* lint */ 1490 1491 ENTRY_NP(enable_vec_intr) 1492 retl 1493 wrpr %g0, %o0, %pstate 1494 SET_SIZE(enable_vec_intr) 1495 1496#endif /* lint */ 1497 1498#if defined(lint) 1499 1500void 1501cbe_level14(void) 1502{} 1503 1504#else /* lint */ 1505 1506 ENTRY_NP(cbe_level14) 1507 save %sp, -SA(MINFRAME), %sp ! get a new window 1508 ! 1509 ! Make sure that this is from TICK_COMPARE; if not just return 1510 ! 1511 rd SOFTINT, %l1 1512 set (TICK_INT_MASK | STICK_INT_MASK), %o2 1513 andcc %l1, %o2, %g0 1514 bz,pn %icc, 2f 1515 nop 1516 1517 CPU_ADDR(%o1, %o2) 1518 call cyclic_fire 1519 mov %o1, %o0 15202: 1521 ret 1522 restore %g0, 1, %o0 1523 SET_SIZE(cbe_level14) 1524 1525#endif /* lint */ 1526 1527 1528#if defined(lint) 1529 1530/* ARGSUSED */ 1531void 1532kdi_setsoftint(uint64_t iv_p) 1533{} 1534 1535#else /* lint */ 1536 1537 ENTRY_NP(kdi_setsoftint) 1538 save %sp, -SA(MINFRAME), %sp ! get a new window 1539 rdpr %pstate, %l5 1540 andn %l5, PSTATE_IE, %l1 1541 wrpr %l1, %pstate ! disable interrupt 1542 ! 1543 ! We have a pointer to an interrupt vector data structure. 1544 ! Put the request on the cpu's softint priority list and 1545 ! set %set_softint. 1546 ! 1547 ! Register usage 1548 ! %i0 - pointer to intr_vec_t (iv) 1549 ! %l2 - requested pil 1550 ! %l4 - cpu 1551 ! %l5 - pstate 1552 ! %l1, %l3, %l6 - temps 1553 ! 1554 ! check if a softint is pending for this softint, 1555 ! if one is pending, don't bother queuing another. 1556 ! 1557 lduh [%i0 + IV_FLAGS], %l1 ! %l1 = iv->iv_flags 1558 and %l1, IV_SOFTINT_PEND, %l6 ! %l6 = iv->iv_flags & IV_SOFTINT_PEND 1559 brnz,pn %l6, 4f ! branch if softint is already pending 1560 or %l1, IV_SOFTINT_PEND, %l2 1561 sth %l2, [%i0 + IV_FLAGS] ! Set IV_SOFTINT_PEND flag 1562 1563 CPU_ADDR(%l4, %l2) ! %l4 = cpu 1564 lduh [%i0 + IV_PIL], %l2 ! %l2 = iv->iv_pil 1565 1566 ! 1567 ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list 1568 ! 1569 sll %l2, CPTRSHIFT, %l0 ! %l0 = offset to pil entry 1570 add %l4, INTR_TAIL, %l6 ! %l6 = &cpu->m_cpu.intr_tail 1571 ldn [%l6 + %l0], %l1 ! %l1 = cpu->m_cpu.intr_tail[pil] 1572 ! current tail (ct) 1573 brz,pt %l1, 2f ! branch if current tail is NULL 1574 stn %i0, [%l6 + %l0] ! make intr_vec_t (iv) as new tail 1575 ! 1576 ! there's pending intr_vec_t already 1577 ! 1578 lduh [%l1 + IV_FLAGS], %l6 ! %l6 = ct->iv_flags 1579 and %l6, IV_SOFTINT_MT, %l6 ! %l6 = ct->iv_flags & IV_SOFTINT_MT 1580 brz,pt %l6, 1f ! check for Multi target softint flag 1581 add %l1, IV_PIL_NEXT, %l3 ! %l3 = &ct->iv_pil_next 1582 ld [%l4 + CPU_ID], %l6 ! for multi target softint, use cpuid 1583 sll %l6, CPTRSHIFT, %l6 ! calculate offset address from cpuid 1584 add %l3, %l6, %l3 ! %l3 = &ct->iv_xpil_next[cpuid] 15851: 1586 ! 1587 ! update old tail 1588 ! 1589 ba,pt %xcc, 3f 1590 stn %i0, [%l3] ! [%l3] = iv, set pil_next field 15912: 1592 ! 1593 ! no pending intr_vec_t; make intr_vec_t as new head 1594 ! 1595 add %l4, INTR_HEAD, %l6 ! %l6 = &cpu->m_cpu.intr_head[pil] 1596 stn %i0, [%l6 + %l0] ! cpu->m_cpu.intr_head[pil] = iv 15973: 1598 ! 1599 ! Write %set_softint with (1<<pil) to cause a "pil" level trap 1600 ! 1601 mov 1, %l1 ! %l1 = 1 1602 sll %l1, %l2, %l1 ! %l1 = 1 << pil 1603 wr %l1, SET_SOFTINT ! trigger required pil softint 16044: 1605 wrpr %g0, %l5, %pstate ! %pstate = saved %pstate (in %l5) 1606 ret 1607 restore 1608 SET_SIZE(kdi_setsoftint) 1609 1610#endif /* lint */ 1611 1612#if defined(lint) 1613 1614/*ARGSUSED*/ 1615void 1616setsoftint_tl1(uint64_t iv_p, uint64_t dummy) 1617{} 1618 1619#else /* lint */ 1620 1621 ! 1622 ! Register usage 1623 ! Arguments: 1624 ! %g1 - Pointer to intr_vec_t (iv) 1625 ! 1626 ! Internal: 1627 ! %g2 - pil 1628 ! %g4 - cpu 1629 ! %g3,%g5-g7 - temps 1630 ! 1631 ENTRY_NP(setsoftint_tl1) 1632 ! 1633 ! We have a pointer to an interrupt vector data structure. 1634 ! Put the request on the cpu's softint priority list and 1635 ! set %set_softint. 1636 ! 1637 CPU_ADDR(%g4, %g2) ! %g4 = cpu 1638 lduh [%g1 + IV_PIL], %g2 ! %g2 = iv->iv_pil 1639 1640 ! 1641 ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list 1642 ! 1643 sll %g2, CPTRSHIFT, %g7 ! %g7 = offset to pil entry 1644 add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail 1645 ldn [%g6 + %g7], %g5 ! %g5 = cpu->m_cpu.intr_tail[pil] 1646 ! current tail (ct) 1647 brz,pt %g5, 1f ! branch if current tail is NULL 1648 stn %g1, [%g6 + %g7] ! make intr_rec_t (iv) as new tail 1649 ! 1650 ! there's pending intr_vec_t already 1651 ! 1652 lduh [%g5 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags 1653 and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT 1654 brz,pt %g6, 0f ! check for Multi target softint flag 1655 add %g5, IV_PIL_NEXT, %g3 ! %g3 = &ct->iv_pil_next 1656 ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid 1657 sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid 1658 add %g3, %g6, %g3 ! %g3 = &ct->iv_xpil_next[cpuid] 16590: 1660 ! 1661 ! update old tail 1662 ! 1663 ba,pt %xcc, 2f 1664 stn %g1, [%g3] ! [%g3] = iv, set pil_next field 16651: 1666 ! 1667 ! no pending intr_vec_t; make intr_vec_t as new head 1668 ! 1669 add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] 1670 stn %g1, [%g6 + %g7] ! cpu->m_cpu.intr_head[pil] = iv 16712: 1672#ifdef TRAPTRACE 1673 TRACE_PTR(%g5, %g6) 1674 GET_TRACE_TICK(%g6, %g3) 1675 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick 1676 TRACE_SAVE_TL_GL_REGS(%g5, %g6) 1677 rdpr %tt, %g6 1678 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt 1679 rdpr %tpc, %g6 1680 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc 1681 rdpr %tstate, %g6 1682 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate 1683 stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp 1684 stna %g1, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = iv 1685 ldn [%g1 + IV_PIL_NEXT], %g6 ! 1686 stna %g6, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = iv->iv_pil_next 1687 add %g4, INTR_HEAD, %g6 1688 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_head[pil] 1689 stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil] 1690 add %g4, INTR_TAIL, %g6 1691 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_tail[pil] 1692 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil] 1693 stna %g2, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil 1694 TRACE_NEXT(%g5, %g6, %g3) 1695#endif /* TRAPTRACE */ 1696 ! 1697 ! Write %set_softint with (1<<pil) to cause a "pil" level trap 1698 ! 1699 mov 1, %g5 ! %g5 = 1 1700 sll %g5, %g2, %g5 ! %g5 = 1 << pil 1701 wr %g5, SET_SOFTINT ! trigger required pil softint 1702 retry 1703 SET_SIZE(setsoftint_tl1) 1704 1705#endif /* lint */ 1706 1707#if defined(lint) 1708 1709/*ARGSUSED*/ 1710void 1711setvecint_tl1(uint64_t inum, uint64_t dummy) 1712{} 1713 1714#else /* lint */ 1715 1716 ! 1717 ! Register usage 1718 ! Arguments: 1719 ! %g1 - inumber 1720 ! 1721 ! Internal: 1722 ! %g1 - softint pil mask 1723 ! %g2 - pil of intr_vec_t 1724 ! %g3 - pointer to current intr_vec_t (iv) 1725 ! %g4 - cpu 1726 ! %g5, %g6,%g7 - temps 1727 ! 1728 ENTRY_NP(setvecint_tl1) 1729 ! 1730 ! Verify the inumber received (should be inum < MAXIVNUM). 1731 ! 1732 set MAXIVNUM, %g2 1733 cmp %g1, %g2 1734 bgeu,pn %xcc, .no_ivintr 1735 clr %g2 ! expected in .no_ivintr 1736 1737 ! 1738 ! Fetch data from intr_vec_table according to the inum. 1739 ! 1740 ! We have an interrupt number. Fetch the interrupt vector requests 1741 ! from the interrupt vector table for a given interrupt number and 1742 ! insert them into cpu's softint priority lists and set %set_softint. 1743 ! 1744 set intr_vec_table, %g5 ! %g5 = intr_vec_table 1745 sll %g1, CPTRSHIFT, %g6 ! %g6 = offset to inum entry in table 1746 add %g5, %g6, %g5 ! %g5 = &intr_vec_table[inum] 1747 ldn [%g5], %g3 ! %g3 = pointer to first entry of 1748 ! intr_vec_t list 1749 1750 ! Verify the first intr_vec_t pointer for a given inum and it should 1751 ! not be NULL. This used to be guarded by DEBUG but broken drivers can 1752 ! cause spurious tick interrupts when the softint register is programmed 1753 ! with 1 << 0 at the end of this routine. Now we always check for a 1754 ! valid intr_vec_t pointer. 1755 brz,pn %g3, .no_ivintr 1756 nop 1757 1758 ! 1759 ! Traverse the intr_vec_t link list, put each item on to corresponding 1760 ! CPU softint priority queue, and compose the final softint pil mask. 1761 ! 1762 ! At this point: 1763 ! %g3 = intr_vec_table[inum] 1764 ! 1765 CPU_ADDR(%g4, %g2) ! %g4 = cpu 1766 mov %g0, %g1 ! %g1 = 0, initialize pil mask to 0 17670: 1768 ! 1769 ! Insert next intr_vec_t (iv) to appropriate cpu's softint priority list 1770 ! 1771 ! At this point: 1772 ! %g1 = softint pil mask 1773 ! %g3 = pointer to next intr_vec_t (iv) 1774 ! %g4 = cpu 1775 ! 1776 lduh [%g3 + IV_PIL], %g2 ! %g2 = iv->iv_pil 1777 sll %g2, CPTRSHIFT, %g7 ! %g7 = offset to pil entry 1778 add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail 1779 ldn [%g6 + %g7], %g5 ! %g5 = cpu->m_cpu.intr_tail[pil] 1780 ! current tail (ct) 1781 brz,pt %g5, 2f ! branch if current tail is NULL 1782 stn %g3, [%g6 + %g7] ! make intr_vec_t (iv) as new tail 1783 ! cpu->m_cpu.intr_tail[pil] = iv 1784 ! 1785 ! there's pending intr_vec_t already 1786 ! 1787 lduh [%g5 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags 1788 and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT 1789 brz,pt %g6, 1f ! check for Multi target softint flag 1790 add %g5, IV_PIL_NEXT, %g5 ! %g5 = &ct->iv_pil_next 1791 ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid 1792 sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid 1793 add %g5, %g6, %g5 ! %g5 = &ct->iv_xpil_next[cpuid] 17941: 1795 ! 1796 ! update old tail 1797 ! 1798 ba,pt %xcc, 3f 1799 stn %g3, [%g5] ! [%g5] = iv, set pil_next field 18002: 1801 ! 1802 ! no pending intr_vec_t; make intr_vec_t as new head 1803 ! 1804 add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] 1805 stn %g3, [%g6 + %g7] ! cpu->m_cpu.intr_head[pil] = iv 18063: 1807#ifdef TRAPTRACE 1808 TRACE_PTR(%g5, %g6) 1809 TRACE_SAVE_TL_GL_REGS(%g5, %g6) 1810 rdpr %tt, %g6 1811 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt` 1812 rdpr %tpc, %g6 1813 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc 1814 rdpr %tstate, %g6 1815 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate 1816 stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp 1817 stna %g3, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = iv 1818 stna %g1, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = pil mask 1819 add %g4, INTR_HEAD, %g6 1820 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_head[pil] 1821 stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil] 1822 add %g4, INTR_TAIL, %g6 1823 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_tail[pil] 1824 stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil] 1825 stna %g2, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil 1826 GET_TRACE_TICK(%g6, %g7) 1827 stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick 1828 TRACE_NEXT(%g5, %g6, %g7) 1829#endif /* TRAPTRACE */ 1830 mov 1, %g6 ! %g6 = 1 1831 sll %g6, %g2, %g6 ! %g6 = 1 << pil 1832 or %g1, %g6, %g1 ! %g1 |= (1 << pil), pil mask 1833 ldn [%g3 + IV_VEC_NEXT], %g3 ! %g3 = pointer to next intr_vec_t (iv) 1834 brnz,pn %g3, 0b ! iv->iv_vec_next is non NULL, goto 0b 1835 nop 1836 wr %g1, SET_SOFTINT ! triggered one or more pil softints 1837 retry 1838 1839.no_ivintr: 1840 ! no_ivintr: arguments: rp, inum (%g1), pil (%g2 == 0) 1841 mov %g2, %g3 1842 mov %g1, %g2 1843 set no_ivintr, %g1 1844 ba,pt %xcc, sys_trap 1845 mov PIL_15, %g4 1846 SET_SIZE(setvecint_tl1) 1847 1848#endif /* lint */ 1849 1850#if defined(lint) 1851 1852/*ARGSUSED*/ 1853void 1854wr_clr_softint(uint_t value) 1855{} 1856 1857#else 1858 1859 ENTRY_NP(wr_clr_softint) 1860 retl 1861 wr %o0, CLEAR_SOFTINT 1862 SET_SIZE(wr_clr_softint) 1863 1864#endif /* lint */ 1865 1866#if defined(lint) 1867 1868/*ARGSUSED*/ 1869void 1870intr_enqueue_req(uint_t pil, uint64_t inum) 1871{} 1872 1873#else /* lint */ 1874 1875/* 1876 * intr_enqueue_req 1877 * 1878 * %o0 - pil 1879 * %o1 - pointer to intr_vec_t (iv) 1880 * %o5 - preserved 1881 * %g5 - preserved 1882 */ 1883 ENTRY_NP(intr_enqueue_req) 1884 ! 1885 CPU_ADDR(%g4, %g1) ! %g4 = cpu 1886 1887 ! 1888 ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list 1889 ! 1890 sll %o0, CPTRSHIFT, %o0 ! %o0 = offset to pil entry 1891 add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail 1892 ldn [%o0 + %g6], %g1 ! %g1 = cpu->m_cpu.intr_tail[pil] 1893 ! current tail (ct) 1894 brz,pt %g1, 2f ! branch if current tail is NULL 1895 stn %o1, [%g6 + %o0] ! make intr_vec_t (iv) as new tail 1896 1897 ! 1898 ! there's pending intr_vec_t already 1899 ! 1900 lduh [%g1 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags 1901 and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT 1902 brz,pt %g6, 1f ! check for Multi target softint flag 1903 add %g1, IV_PIL_NEXT, %g3 ! %g3 = &ct->iv_pil_next 1904 ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid 1905 sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid 1906 add %g3, %g6, %g3 ! %g3 = &ct->iv_xpil_next[cpuid] 19071: 1908 ! 1909 ! update old tail 1910 ! 1911 ba,pt %xcc, 3f 1912 stn %o1, [%g3] ! {%g5] = iv, set pil_next field 19132: 1914 ! 1915 ! no intr_vec_t's queued so make intr_vec_t as new head 1916 ! 1917 add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] 1918 stn %o1, [%g6 + %o0] ! cpu->m_cpu.intr_head[pil] = iv 19193: 1920 retl 1921 nop 1922 SET_SIZE(intr_enqueue_req) 1923 1924#endif /* lint */ 1925 1926/* 1927 * Set CPU's base SPL level, based on which interrupt levels are active. 1928 * Called at spl7 or above. 1929 */ 1930 1931#if defined(lint) 1932 1933void 1934set_base_spl(void) 1935{} 1936 1937#else /* lint */ 1938 1939 ENTRY_NP(set_base_spl) 1940 ldn [THREAD_REG + T_CPU], %o2 ! load CPU pointer 1941 ld [%o2 + CPU_INTR_ACTV], %o5 ! load active interrupts mask 1942 1943/* 1944 * WARNING: non-standard callinq sequence; do not call from C 1945 * %o2 = pointer to CPU 1946 * %o5 = updated CPU_INTR_ACTV 1947 */ 1948_intr_set_spl: ! intr_thread_exit enters here 1949 ! 1950 ! Determine highest interrupt level active. Several could be blocked 1951 ! at higher levels than this one, so must convert flags to a PIL 1952 ! Normally nothing will be blocked, so test this first. 1953 ! 1954 brz,pt %o5, 1f ! nothing active 1955 sra %o5, 11, %o3 ! delay - set %o3 to bits 15-11 1956 set _intr_flag_table, %o1 1957 tst %o3 ! see if any of the bits set 1958 ldub [%o1 + %o3], %o3 ! load bit number 1959 bnz,a,pn %xcc, 1f ! yes, add 10 and we're done 1960 add %o3, 11-1, %o3 ! delay - add bit number - 1 1961 1962 sra %o5, 6, %o3 ! test bits 10-6 1963 tst %o3 1964 ldub [%o1 + %o3], %o3 1965 bnz,a,pn %xcc, 1f 1966 add %o3, 6-1, %o3 1967 1968 sra %o5, 1, %o3 ! test bits 5-1 1969 ldub [%o1 + %o3], %o3 1970 1971 ! 1972 ! highest interrupt level number active is in %l6 1973 ! 19741: 1975 retl 1976 st %o3, [%o2 + CPU_BASE_SPL] ! delay - store base priority 1977 SET_SIZE(set_base_spl) 1978 1979/* 1980 * Table that finds the most significant bit set in a five bit field. 1981 * Each entry is the high-order bit number + 1 of it's index in the table. 1982 * This read-only data is in the text segment. 1983 */ 1984_intr_flag_table: 1985 .byte 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4 1986 .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 1987 .align 4 1988 1989#endif /* lint */ 1990 1991/* 1992 * int 1993 * intr_passivate(from, to) 1994 * kthread_id_t from; interrupt thread 1995 * kthread_id_t to; interrupted thread 1996 */ 1997 1998#if defined(lint) 1999 2000/* ARGSUSED */ 2001int 2002intr_passivate(kthread_id_t from, kthread_id_t to) 2003{ return (0); } 2004 2005#else /* lint */ 2006 2007 ENTRY_NP(intr_passivate) 2008 save %sp, -SA(MINFRAME), %sp ! get a new window 2009 2010 flushw ! force register windows to stack 2011 ! 2012 ! restore registers from the base of the stack of the interrupt thread. 2013 ! 2014 ldn [%i0 + T_STACK], %i2 ! get stack save area pointer 2015 ldn [%i2 + (0*GREGSIZE)], %l0 ! load locals 2016 ldn [%i2 + (1*GREGSIZE)], %l1 2017 ldn [%i2 + (2*GREGSIZE)], %l2 2018 ldn [%i2 + (3*GREGSIZE)], %l3 2019 ldn [%i2 + (4*GREGSIZE)], %l4 2020 ldn [%i2 + (5*GREGSIZE)], %l5 2021 ldn [%i2 + (6*GREGSIZE)], %l6 2022 ldn [%i2 + (7*GREGSIZE)], %l7 2023 ldn [%i2 + (8*GREGSIZE)], %o0 ! put ins from stack in outs 2024 ldn [%i2 + (9*GREGSIZE)], %o1 2025 ldn [%i2 + (10*GREGSIZE)], %o2 2026 ldn [%i2 + (11*GREGSIZE)], %o3 2027 ldn [%i2 + (12*GREGSIZE)], %o4 2028 ldn [%i2 + (13*GREGSIZE)], %o5 2029 ldn [%i2 + (14*GREGSIZE)], %i4 2030 ! copy stack/pointer without using %sp 2031 ldn [%i2 + (15*GREGSIZE)], %i5 2032 ! 2033 ! put registers into the save area at the top of the interrupted 2034 ! thread's stack, pointed to by %l7 in the save area just loaded. 2035 ! 2036 ldn [%i1 + T_SP], %i3 ! get stack save area pointer 2037 stn %l0, [%i3 + STACK_BIAS + (0*GREGSIZE)] ! save locals 2038 stn %l1, [%i3 + STACK_BIAS + (1*GREGSIZE)] 2039 stn %l2, [%i3 + STACK_BIAS + (2*GREGSIZE)] 2040 stn %l3, [%i3 + STACK_BIAS + (3*GREGSIZE)] 2041 stn %l4, [%i3 + STACK_BIAS + (4*GREGSIZE)] 2042 stn %l5, [%i3 + STACK_BIAS + (5*GREGSIZE)] 2043 stn %l6, [%i3 + STACK_BIAS + (6*GREGSIZE)] 2044 stn %l7, [%i3 + STACK_BIAS + (7*GREGSIZE)] 2045 stn %o0, [%i3 + STACK_BIAS + (8*GREGSIZE)] ! save ins using outs 2046 stn %o1, [%i3 + STACK_BIAS + (9*GREGSIZE)] 2047 stn %o2, [%i3 + STACK_BIAS + (10*GREGSIZE)] 2048 stn %o3, [%i3 + STACK_BIAS + (11*GREGSIZE)] 2049 stn %o4, [%i3 + STACK_BIAS + (12*GREGSIZE)] 2050 stn %o5, [%i3 + STACK_BIAS + (13*GREGSIZE)] 2051 stn %i4, [%i3 + STACK_BIAS + (14*GREGSIZE)] 2052 ! fp, %i7 copied using %i4 2053 stn %i5, [%i3 + STACK_BIAS + (15*GREGSIZE)] 2054 stn %g0, [%i2 + ((8+6)*GREGSIZE)] 2055 ! clear fp in save area 2056 2057 ! load saved pil for return 2058 ldub [%i0 + T_PIL], %i0 2059 ret 2060 restore 2061 SET_SIZE(intr_passivate) 2062 2063#endif /* lint */ 2064 2065#if defined(lint) 2066 2067/* 2068 * intr_get_time() is a resource for interrupt handlers to determine how 2069 * much time has been spent handling the current interrupt. Such a function 2070 * is needed because higher level interrupts can arrive during the 2071 * processing of an interrupt, thus making direct comparisons of %tick by 2072 * the handler inaccurate. intr_get_time() only returns time spent in the 2073 * current interrupt handler. 2074 * 2075 * The caller must be calling from an interrupt handler running at a pil 2076 * below or at lock level. Timings are not provided for high-level 2077 * interrupts. 2078 * 2079 * The first time intr_get_time() is called while handling an interrupt, 2080 * it returns the time since the interrupt handler was invoked. Subsequent 2081 * calls will return the time since the prior call to intr_get_time(). Time 2082 * is returned as ticks, adjusted for any clock divisor due to power 2083 * management. Use tick2ns() to convert ticks to nsec. Warning: ticks may 2084 * not be the same across CPUs. 2085 * 2086 * Theory Of Intrstat[][]: 2087 * 2088 * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two 2089 * uint64_ts per pil. 2090 * 2091 * intrstat[pil][0] is a cumulative count of the number of ticks spent 2092 * handling all interrupts at the specified pil on this CPU. It is 2093 * exported via kstats to the user. 2094 * 2095 * intrstat[pil][1] is always a count of ticks less than or equal to the 2096 * value in [0]. The difference between [1] and [0] is the value returned 2097 * by a call to intr_get_time(). At the start of interrupt processing, 2098 * [0] and [1] will be equal (or nearly so). As the interrupt consumes 2099 * time, [0] will increase, but [1] will remain the same. A call to 2100 * intr_get_time() will return the difference, then update [1] to be the 2101 * same as [0]. Future calls will return the time since the last call. 2102 * Finally, when the interrupt completes, [1] is updated to the same as [0]. 2103 * 2104 * Implementation: 2105 * 2106 * intr_get_time() works much like a higher level interrupt arriving. It 2107 * "checkpoints" the timing information by incrementing intrstat[pil][0] 2108 * to include elapsed running time, and by setting t_intr_start to %tick. 2109 * It then sets the return value to intrstat[pil][0] - intrstat[pil][1], 2110 * and updates intrstat[pil][1] to be the same as the new value of 2111 * intrstat[pil][0]. 2112 * 2113 * In the normal handling of interrupts, after an interrupt handler returns 2114 * and the code in intr_thread() updates intrstat[pil][0], it then sets 2115 * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1], 2116 * the timings are reset, i.e. intr_get_time() will return [0] - [1] which 2117 * is 0. 2118 * 2119 * Whenever interrupts arrive on a CPU which is handling a lower pil 2120 * interrupt, they update the lower pil's [0] to show time spent in the 2121 * handler that they've interrupted. This results in a growing discrepancy 2122 * between [0] and [1], which is returned the next time intr_get_time() is 2123 * called. Time spent in the higher-pil interrupt will not be returned in 2124 * the next intr_get_time() call from the original interrupt, because 2125 * the higher-pil interrupt's time is accumulated in intrstat[higherpil][]. 2126 */ 2127 2128/*ARGSUSED*/ 2129uint64_t 2130intr_get_time(void) 2131{ return 0; } 2132#else /* lint */ 2133 2134 ENTRY_NP(intr_get_time) 2135#ifdef DEBUG 2136 ! 2137 ! Lots of asserts, but just check panic_quiesce first. 2138 ! Don't bother with lots of tests if we're just ignoring them. 2139 ! 2140 sethi %hi(panic_quiesce), %o0 2141 ld [%o0 + %lo(panic_quiesce)], %o0 2142 brnz,pn %o0, 2f 2143 nop 2144 ! 2145 ! ASSERT(%pil <= LOCK_LEVEL) 2146 ! 2147 rdpr %pil, %o1 2148 cmp %o1, LOCK_LEVEL 2149 ble,pt %xcc, 0f 2150 sethi %hi(intr_get_time_high_pil), %o0 ! delay 2151 call panic 2152 or %o0, %lo(intr_get_time_high_pil), %o0 21530: 2154 ! 2155 ! ASSERT((t_flags & T_INTR_THREAD) != 0 && t_pil > 0) 2156 ! 2157 lduh [THREAD_REG + T_FLAGS], %o2 2158 andcc %o2, T_INTR_THREAD, %g0 2159 bz,pn %xcc, 1f 2160 ldub [THREAD_REG + T_PIL], %o1 ! delay 2161 brnz,pt %o1, 0f 21621: 2163 sethi %hi(intr_get_time_not_intr), %o0 2164 call panic 2165 or %o0, %lo(intr_get_time_not_intr), %o0 21660: 2167 ! 2168 ! ASSERT(t_intr_start != 0) 2169 ! 2170 ldx [THREAD_REG + T_INTR_START], %o1 2171 brnz,pt %o1, 2f 2172 sethi %hi(intr_get_time_no_start_time), %o0 ! delay 2173 call panic 2174 or %o0, %lo(intr_get_time_no_start_time), %o0 21752: 2176#endif /* DEBUG */ 2177 ! 2178 ! %o0 = elapsed time and return value 2179 ! %o1 = pil 2180 ! %o2 = scratch 2181 ! %o3 = scratch 2182 ! %o4 = scratch 2183 ! %o5 = cpu 2184 ! 2185 wrpr %g0, PIL_MAX, %pil ! make this easy -- block normal intrs 2186 ldn [THREAD_REG + T_CPU], %o5 2187 ldub [THREAD_REG + T_PIL], %o1 2188 ldx [THREAD_REG + T_INTR_START], %o3 ! %o3 = t_intr_start 2189 ! 2190 ! Calculate elapsed time since t_intr_start. Update t_intr_start, 2191 ! get delta, and multiply by cpu_divisor if necessary. 2192 ! 2193 RD_CLOCK_TICK_NO_SUSPEND_CHECK(%o2, %o0) 2194 stx %o2, [THREAD_REG + T_INTR_START] 2195 sub %o2, %o3, %o0 2196 2197 lduh [%o5 + CPU_DIVISOR], %o4 2198 cmp %o4, 1 2199 bg,a,pn %xcc, 1f 2200 mulx %o0, %o4, %o0 ! multiply interval by clock divisor iff > 1 22011: 2202 ! Update intracct[] 2203 lduh [%o5 + CPU_MSTATE], %o4 2204 sllx %o4, 3, %o4 2205 add %o4, CPU_INTRACCT, %o4 2206 ldx [%o5 + %o4], %o2 2207 add %o2, %o0, %o2 2208 stx %o2, [%o5 + %o4] 2209 2210 ! 2211 ! Increment cpu_m.intrstat[pil][0]. Calculate elapsed time since 2212 ! cpu_m.intrstat[pil][1], which is either when the interrupt was 2213 ! first entered, or the last time intr_get_time() was invoked. Then 2214 ! update cpu_m.intrstat[pil][1] to match [0]. 2215 ! 2216 sllx %o1, 4, %o3 2217 add %o3, CPU_MCPU, %o3 2218 add %o3, MCPU_INTRSTAT, %o3 2219 add %o3, %o5, %o3 ! %o3 = cpu_m.intrstat[pil][0] 2220 ldx [%o3], %o2 2221 add %o2, %o0, %o2 ! %o2 = new value for intrstat 2222 stx %o2, [%o3] 2223 ldx [%o3 + 8], %o4 ! %o4 = cpu_m.intrstat[pil][1] 2224 sub %o2, %o4, %o0 ! %o0 is elapsed time since %o4 2225 stx %o2, [%o3 + 8] ! make [1] match [0], resetting time 2226 2227 ld [%o5 + CPU_BASE_SPL], %o2 ! restore %pil to the greater 2228 cmp %o2, %o1 ! of either our pil %o1 or 2229 movl %xcc, %o1, %o2 ! cpu_base_spl. 2230 retl 2231 wrpr %g0, %o2, %pil 2232 SET_SIZE(intr_get_time) 2233 2234#ifdef DEBUG 2235intr_get_time_high_pil: 2236 .asciz "intr_get_time(): %pil > LOCK_LEVEL" 2237intr_get_time_not_intr: 2238 .asciz "intr_get_time(): not called from an interrupt thread" 2239intr_get_time_no_start_time: 2240 .asciz "intr_get_time(): t_intr_start == 0" 2241#endif /* DEBUG */ 2242#endif /* lint */ 2243