/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #if defined(lint) #include #include #else /* lint */ #include "assym.h" #endif /* lint */ #include #include #include #include #include #include #include #ifdef TRAPTRACE #include #endif /* TRAPTRACE */ #if defined(lint) /* ARGSUSED */ void pil_interrupt(int level) {} #else /* lint */ /* * (TT 0x40..0x4F, TL>0) Interrupt Level N Handler (N == 1..15) * Register passed from LEVEL_INTERRUPT(level) * %g4 - interrupt request level */ ENTRY_NP(pil_interrupt) ! ! Register usage ! %g1 - cpu ! %g2 - pointer to intr_vec_t (iv) ! %g4 - pil ! %g3, %g5, %g6, %g7 - temps ! ! Grab the first or list head intr_vec_t off the intr_head[pil] ! and panic immediately if list head is NULL. Otherwise, update ! intr_head[pil] to next intr_vec_t on the list and clear softint ! %clear_softint, if next intr_vec_t is NULL. ! CPU_ADDR(%g1, %g5) ! %g1 = cpu ! ALTENTRY(pil_interrupt_common) sll %g4, CPTRSHIFT, %g5 ! %g5 = offset to the pil entry add %g1, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head add %g6, %g5, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] ldn [%g6], %g2 ! %g2 = cpu->m_cpu.intr_head[pil] brnz,pt %g2, 0f ! check list head (iv) is NULL nop ba ptl1_panic ! panic, list head (iv) is NULL mov PTL1_BAD_INTR_VEC, %g1 0: lduh [%g2 + IV_FLAGS], %g7 ! %g7 = iv->iv_flags and %g7, IV_SOFTINT_MT, %g3 ! %g3 = iv->iv_flags & IV_SOFTINT_MT brz,pt %g3, 1f ! check for multi target softint add %g2, IV_PIL_NEXT, %g7 ! g7% = &iv->iv_pil_next ld [%g1 + CPU_ID], %g3 ! for multi target softint, use cpuid sll %g3, CPTRSHIFT, %g3 ! convert cpuid to offset address add %g7, %g3, %g7 ! %g5 = &iv->iv_xpil_next[cpuid] 1: ldn [%g7], %g3 ! %g3 = next intr_vec_t brnz,pn %g3, 2f ! branch if next intr_vec_t non NULL stn %g3, [%g6] ! update cpu->m_cpu.intr_head[pil] add %g1, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail stn %g0, [%g5 + %g6] ! clear cpu->m_cpu.intr_tail[pil] mov 1, %g5 ! %g5 = 1 sll %g5, %g4, %g5 ! %g5 = 1 << pil wr %g5, CLEAR_SOFTINT ! clear interrupt on this pil 2: #ifdef TRAPTRACE TRACE_PTR(%g5, %g6) TRACE_SAVE_TL_GL_REGS(%g5, %g6) rdpr %tt, %g6 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt rdpr %tpc, %g6 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc rdpr %tstate, %g6 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp stna %g2, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = first intr_vec stna %g3, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = next intr_vec GET_TRACE_TICK(%g6, %g3) stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick sll %g4, CPTRSHIFT, %g3 add %g1, INTR_HEAD, %g6 ldn [%g6 + %g3], %g6 ! %g6=cpu->m_cpu.intr_head[pil] stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil] add %g1, INTR_TAIL, %g6 ldn [%g6 + %g3], %g6 ! %g6=cpu->m_cpu.intr_tail[pil] stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil] stna %g4, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil TRACE_NEXT(%g5, %g6, %g3) #endif /* TRAPTRACE */ ! ! clear the iv_pending flag for this interrupt request ! lduh [%g2 + IV_FLAGS], %g3 ! %g3 = iv->iv_flags andn %g3, IV_SOFTINT_PEND, %g3 ! %g3 = !(iv->iv_flags & PEND) sth %g3, [%g2 + IV_FLAGS] ! clear IV_SOFTINT_PEND flag stn %g0, [%g7] ! clear iv->iv_pil_next or ! iv->iv_pil_xnext ! ! Prepare for sys_trap() ! ! Registers passed to sys_trap() ! %g1 - interrupt handler at TL==0 ! %g2 - pointer to current intr_vec_t (iv), ! job queue for intr_thread or current_thread ! %g3 - pil ! %g4 - initial pil for handler ! ! figure which handler to run and which %pil it starts at ! intr_thread starts at DISP_LEVEL to prevent preemption ! current_thread starts at PIL_MAX to protect cpu_intr_actv ! mov %g4, %g3 ! %g3 = %g4, pil cmp %g4, LOCK_LEVEL bg,a,pt %xcc, 3f ! branch if pil > LOCK_LEVEL mov PIL_MAX, %g4 ! %g4 = PIL_MAX (15) sethi %hi(intr_thread), %g1 ! %g1 = intr_thread mov DISP_LEVEL, %g4 ! %g4 = DISP_LEVEL (11) ba,pt %xcc, sys_trap or %g1, %lo(intr_thread), %g1 3: sethi %hi(current_thread), %g1 ! %g1 = current_thread ba,pt %xcc, sys_trap or %g1, %lo(current_thread), %g1 SET_SIZE(pil_interrupt_common) SET_SIZE(pil_interrupt) #endif /* lint */ #ifndef lint _spurious: .asciz "!interrupt 0x%x at level %d not serviced" /* * SERVE_INTR_PRE is called once, just before the first invocation * of SERVE_INTR. * * Registers on entry: * * iv_p, cpu, regs: may be out-registers * ls1, ls2: local scratch registers * os1, os2, os3: scratch registers, may be out */ #define SERVE_INTR_PRE(iv_p, cpu, ls1, ls2, os1, os2, os3, regs) \ mov iv_p, ls1; \ mov iv_p, ls2; \ SERVE_INTR_TRACE(iv_p, os1, os2, os3, regs); /* * SERVE_INTR is called immediately after either SERVE_INTR_PRE or * SERVE_INTR_NEXT, without intervening code. No register values * may be modified. * * After calling SERVE_INTR, the caller must check if os3 is set. If * so, there is another interrupt to process. The caller must call * SERVE_INTR_NEXT, immediately followed by SERVE_INTR. * * Before calling SERVE_INTR_NEXT, the caller may perform accounting * and other actions which need to occur after invocation of an interrupt * handler. However, the values of ls1 and os3 *must* be preserved and * passed unmodified into SERVE_INTR_NEXT. * * Registers on return from SERVE_INTR: * * ls1 - the pil just processed * ls2 - the pointer to intr_vec_t (iv) just processed * os3 - if set, another interrupt needs to be processed * cpu, ls1, os3 - must be preserved if os3 is set */ #define SERVE_INTR(os5, cpu, ls1, ls2, os1, os2, os3, os4) \ ldn [ls1 + IV_HANDLER], os2; \ ldn [ls1 + IV_ARG1], %o0; \ ldn [ls1 + IV_ARG2], %o1; \ call os2; \ lduh [ls1 + IV_PIL], ls1; \ brnz,pt %o0, 2f; \ mov CE_WARN, %o0; \ set _spurious, %o1; \ mov ls2, %o2; \ call cmn_err; \ rdpr %pil, %o3; \ 2: ldn [THREAD_REG + T_CPU], cpu; \ sll ls1, 3, os1; \ add os1, CPU_STATS_SYS_INTR - 8, os2; \ ldx [cpu + os2], os3; \ inc os3; \ stx os3, [cpu + os2]; \ sll ls1, CPTRSHIFT, os2; \ add cpu, INTR_HEAD, os1; \ add os1, os2, os1; \ ldn [os1], os3; /* * Registers on entry: * * cpu - cpu pointer (clobbered, set to cpu upon completion) * ls1, os3 - preserved from prior call to SERVE_INTR * ls2 - local scratch reg (not preserved) * os1, os2, os4, os5 - scratch reg, can be out (not preserved) */ #define SERVE_INTR_NEXT(os5, cpu, ls1, ls2, os1, os2, os3, os4) \ sll ls1, CPTRSHIFT, os4; \ add cpu, INTR_HEAD, os1; \ rdpr %pstate, ls2; \ wrpr ls2, PSTATE_IE, %pstate; \ lduh [os3 + IV_FLAGS], os2; \ and os2, IV_SOFTINT_MT, os2; \ brz,pt os2, 4f; \ add os3, IV_PIL_NEXT, os2; \ ld [cpu + CPU_ID], os5; \ sll os5, CPTRSHIFT, os5; \ add os2, os5, os2; \ 4: ldn [os2], os5; \ brnz,pn os5, 5f; \ stn os5, [os1 + os4]; \ add cpu, INTR_TAIL, os1; \ stn %g0, [os1 + os4]; \ mov 1, os1; \ sll os1, ls1, os1; \ wr os1, CLEAR_SOFTINT; \ 5: lduh [os3 + IV_FLAGS], ls1; \ andn ls1, IV_SOFTINT_PEND, ls1; \ sth ls1, [os3 + IV_FLAGS]; \ stn %g0, [os2]; \ wrpr %g0, ls2, %pstate; \ mov os3, ls1; \ mov os3, ls2; \ SERVE_INTR_TRACE2(os5, os1, os2, os3, os4); #ifdef TRAPTRACE /* * inum - not modified, _spurious depends on it. */ #define SERVE_INTR_TRACE(inum, os1, os2, os3, os4) \ rdpr %pstate, os3; \ andn os3, PSTATE_IE | PSTATE_AM, os2; \ wrpr %g0, os2, %pstate; \ TRACE_PTR(os1, os2); \ ldn [os4 + PC_OFF], os2; \ stna os2, [os1 + TRAP_ENT_TPC]%asi; \ ldx [os4 + TSTATE_OFF], os2; \ stxa os2, [os1 + TRAP_ENT_TSTATE]%asi; \ mov os3, os4; \ GET_TRACE_TICK(os2, os3); \ stxa os2, [os1 + TRAP_ENT_TICK]%asi; \ TRACE_SAVE_TL_GL_REGS(os1, os2); \ set TT_SERVE_INTR, os2; \ rdpr %pil, os3; \ or os2, os3, os2; \ stha os2, [os1 + TRAP_ENT_TT]%asi; \ stna %sp, [os1 + TRAP_ENT_SP]%asi; \ stna inum, [os1 + TRAP_ENT_TR]%asi; \ stna %g0, [os1 + TRAP_ENT_F1]%asi; \ stna %g0, [os1 + TRAP_ENT_F2]%asi; \ stna %g0, [os1 + TRAP_ENT_F3]%asi; \ stna %g0, [os1 + TRAP_ENT_F4]%asi; \ TRACE_NEXT(os1, os2, os3); \ wrpr %g0, os4, %pstate #else /* TRAPTRACE */ #define SERVE_INTR_TRACE(inum, os1, os2, os3, os4) #endif /* TRAPTRACE */ #ifdef TRAPTRACE /* * inum - not modified, _spurious depends on it. */ #define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4) \ rdpr %pstate, os3; \ andn os3, PSTATE_IE | PSTATE_AM, os2; \ wrpr %g0, os2, %pstate; \ TRACE_PTR(os1, os2); \ stna %g0, [os1 + TRAP_ENT_TPC]%asi; \ stxa %g0, [os1 + TRAP_ENT_TSTATE]%asi; \ mov os3, os4; \ GET_TRACE_TICK(os2, os3); \ stxa os2, [os1 + TRAP_ENT_TICK]%asi; \ TRACE_SAVE_TL_GL_REGS(os1, os2); \ set TT_SERVE_INTR, os2; \ rdpr %pil, os3; \ or os2, os3, os2; \ stha os2, [os1 + TRAP_ENT_TT]%asi; \ stna %sp, [os1 + TRAP_ENT_SP]%asi; \ stna inum, [os1 + TRAP_ENT_TR]%asi; \ stna %g0, [os1 + TRAP_ENT_F1]%asi; \ stna %g0, [os1 + TRAP_ENT_F2]%asi; \ stna %g0, [os1 + TRAP_ENT_F3]%asi; \ stna %g0, [os1 + TRAP_ENT_F4]%asi; \ TRACE_NEXT(os1, os2, os3); \ wrpr %g0, os4, %pstate #else /* TRAPTRACE */ #define SERVE_INTR_TRACE2(inum, os1, os2, os3, os4) #endif /* TRAPTRACE */ #endif /* lint */ #if defined(lint) /*ARGSUSED*/ void intr_thread(struct regs *regs, uint64_t iv_p, uint_t pil) {} #else /* lint */ #define INTRCNT_LIMIT 16 /* * Handle an interrupt in a new thread. * Entry: * %o0 = pointer to regs structure * %o1 = pointer to current intr_vec_t (iv) to be processed * %o2 = pil * %sp = on current thread's kernel stack * %o7 = return linkage to trap code * %g7 = current thread * %pstate = normal globals, interrupts enabled, * privileged, fp disabled * %pil = DISP_LEVEL * * Register Usage * %l0 = return linkage * %l1 = pil * %l2 - %l3 = scratch * %l4 - %l7 = reserved for sys_trap * %o2 = cpu * %o3 = intr thread * %o0 = scratch * %o4 - %o5 = scratch */ ENTRY_NP(intr_thread) mov %o7, %l0 mov %o2, %l1 ! ! See if we are interrupting another interrupt thread. ! lduh [THREAD_REG + T_FLAGS], %o3 andcc %o3, T_INTR_THREAD, %g0 bz,pt %xcc, 1f ldn [THREAD_REG + T_CPU], %o2 ! delay - load CPU pointer ! We have interrupted an interrupt thread. Take a timestamp, ! compute its interval, and update its cumulative counter. add THREAD_REG, T_INTR_START, %o5 0: ldx [%o5], %o3 brz,pn %o3, 1f ! We came in on top of an interrupt thread that had no timestamp. ! This could happen if, for instance, an interrupt thread which had ! previously blocked is being set up to run again in resume(), but ! resume() hasn't yet stored a timestamp for it. Or, it could be in ! swtch() after its slice has been accounted for. ! Only account for the time slice if the starting timestamp is non-zero. RD_TICK(%o4,%l2,%l3,__LINE__) sub %o4, %o3, %o4 ! o4 has interval ! A high-level interrupt in current_thread() interrupting here ! will account for the interrupted thread's time slice, but ! only if t_intr_start is non-zero. Since this code is going to account ! for the time slice, we want to "atomically" load the thread's ! starting timestamp, calculate the interval with %tick, and zero ! its starting timestamp. ! To do this, we do a casx on the t_intr_start field, and store 0 to it. ! If it has changed since we loaded it above, we need to re-compute the ! interval, since a changed t_intr_start implies current_thread placed ! a new, later timestamp there after running a high-level interrupt, ! and the %tick val in %o4 had become stale. mov %g0, %l2 casx [%o5], %o3, %l2 ! If %l2 == %o3, our casx was successful. If not, the starting timestamp ! changed between loading it (after label 0b) and computing the ! interval above. cmp %l2, %o3 bne,pn %xcc, 0b ! Check for Energy Star mode lduh [%o2 + CPU_DIVISOR], %l2 ! delay -- %l2 = clock divisor cmp %l2, 1 bg,a,pn %xcc, 2f mulx %o4, %l2, %o4 ! multiply interval by clock divisor iff > 1 2: ! We now know that a valid interval for the interrupted interrupt ! thread is in %o4. Update its cumulative counter. ldub [THREAD_REG + T_PIL], %l3 ! load PIL sllx %l3, 4, %l3 ! convert PIL index to byte offset add %l3, CPU_MCPU, %l3 ! CPU_INTRSTAT is too big for use add %l3, MCPU_INTRSTAT, %l3 ! as const, add offsets separately ldx [%o2 + %l3], %o5 ! old counter in o5 add %o5, %o4, %o5 ! new counter in o5 stx %o5, [%o2 + %l3] ! store new counter ! Also update intracct[] lduh [%o2 + CPU_MSTATE], %l3 sllx %l3, 3, %l3 add %l3, CPU_INTRACCT, %l3 add %l3, %o2, %l3 0: ldx [%l3], %o5 add %o5, %o4, %o3 casx [%l3], %o5, %o3 cmp %o5, %o3 bne,pn %xcc, 0b nop 1: ! ! Get set to run interrupt thread. ! There should always be an interrupt thread since we allocate one ! for each level on the CPU. ! ! Note that the code in kcpc_overflow_intr -relies- on the ordering ! of events here -- in particular that t->t_lwp of the interrupt thread ! is set to the pinned thread *before* curthread is changed. ! ldn [%o2 + CPU_INTR_THREAD], %o3 ! interrupt thread pool ldn [%o3 + T_LINK], %o4 ! unlink thread from CPU's list stn %o4, [%o2 + CPU_INTR_THREAD] ! ! Set bit for this level in CPU's active interrupt bitmask. ! ld [%o2 + CPU_INTR_ACTV], %o5 mov 1, %o4 sll %o4, %l1, %o4 #ifdef DEBUG ! ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL))) ! andcc %o5, %o4, %g0 bz,pt %xcc, 0f nop ! Do not call panic if a panic is already in progress. sethi %hi(panic_quiesce), %l2 ld [%l2 + %lo(panic_quiesce)], %l2 brnz,pn %l2, 0f nop sethi %hi(intr_thread_actv_bit_set), %o0 call panic or %o0, %lo(intr_thread_actv_bit_set), %o0 0: #endif /* DEBUG */ or %o5, %o4, %o5 st %o5, [%o2 + CPU_INTR_ACTV] ! ! Consider the new thread part of the same LWP so that ! window overflow code can find the PCB. ! ldn [THREAD_REG + T_LWP], %o4 stn %o4, [%o3 + T_LWP] ! ! Threads on the interrupt thread free list could have state already ! set to TS_ONPROC, but it helps in debugging if they're TS_FREE ! Could eliminate the next two instructions with a little work. ! mov TS_ONPROC, %o4 st %o4, [%o3 + T_STATE] ! ! Push interrupted thread onto list from new thread. ! Set the new thread as the current one. ! Set interrupted thread's T_SP because if it is the idle thread, ! resume may use that stack between threads. ! stn %o7, [THREAD_REG + T_PC] ! mark pc for resume stn %sp, [THREAD_REG + T_SP] ! mark stack for resume stn THREAD_REG, [%o3 + T_INTR] ! push old thread stn %o3, [%o2 + CPU_THREAD] ! set new thread mov %o3, THREAD_REG ! set global curthread register ldn [%o3 + T_STACK], %o4 ! interrupt stack pointer sub %o4, STACK_BIAS, %sp ! ! Initialize thread priority level from intr_pri ! sethi %hi(intr_pri), %o4 ldsh [%o4 + %lo(intr_pri)], %o4 ! grab base interrupt priority add %l1, %o4, %o4 ! convert level to dispatch priority sth %o4, [THREAD_REG + T_PRI] stub %l1, [THREAD_REG + T_PIL] ! save pil for intr_passivate ! Store starting timestamp in thread structure. add THREAD_REG, T_INTR_START, %o3 1: ldx [%o3], %o5 RD_TICK(%o4,%l2,%l3,__LINE__) casx [%o3], %o5, %o4 cmp %o4, %o5 ! If a high-level interrupt occurred while we were attempting to store ! the timestamp, try again. bne,pn %xcc, 1b nop wrpr %g0, %l1, %pil ! lower %pil to new level ! ! Fast event tracing. ! ld [%o2 + CPU_FTRACE_STATE], %o4 ! %o2 = curthread->t_cpu btst FTRACE_ENABLED, %o4 be,pt %icc, 1f ! skip if ftrace disabled mov %l1, %o5 ! ! Tracing is enabled - write the trace entry. ! save %sp, -SA(MINFRAME), %sp set ftrace_intr_thread_format_str, %o0 mov %i0, %o1 mov %i1, %o2 mov %i5, %o3 call ftrace_3 ldn [%i0 + PC_OFF], %o4 restore 1: ! ! call the handler ! SERVE_INTR_PRE(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0) ! ! %o0 and %o1 are now available as scratch registers. ! 0: SERVE_INTR(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0) ! ! If %o3 is set, we must call serve_intr_next, and both %l1 and %o3 ! must be preserved. %l1 holds our pil, %l3 holds our inum. ! ! Note: %l1 is the pil level we're processing, but we may have a ! higher effective pil because a higher-level interrupt may have ! blocked. ! wrpr %g0, DISP_LEVEL, %pil ! ! Take timestamp, compute interval, update cumulative counter. ! add THREAD_REG, T_INTR_START, %o5 1: ldx [%o5], %o0 #ifdef DEBUG brnz %o0, 9f nop ! Do not call panic if a panic is already in progress. sethi %hi(panic_quiesce), %o1 ld [%o1 + %lo(panic_quiesce)], %o1 brnz,pn %o1, 9f nop sethi %hi(intr_thread_t_intr_start_zero), %o0 call panic or %o0, %lo(intr_thread_t_intr_start_zero), %o0 9: #endif /* DEBUG */ RD_TICK(%o1,%l2,%l3,__LINE__) sub %o1, %o0, %l2 ! l2 has interval ! ! The general outline of what the code here does is: ! 1. load t_intr_start, %tick, and calculate the delta ! 2. replace t_intr_start with %tick (if %o3 is set) or 0. ! ! The problem is that a high-level interrupt could arrive at any time. ! It will account for (%tick - t_intr_start) for us when it starts, ! unless we have set t_intr_start to zero, and then set t_intr_start ! to a new %tick when it finishes. To account for this, our first step ! is to load t_intr_start and the last is to use casx to store the new ! t_intr_start. This guarantees atomicity in reading t_intr_start, ! reading %tick, and updating t_intr_start. ! movrz %o3, %g0, %o1 casx [%o5], %o0, %o1 cmp %o0, %o1 bne,pn %xcc, 1b ! ! Check for Energy Star mode ! lduh [%o2 + CPU_DIVISOR], %o0 ! delay -- %o0 = clock divisor cmp %o0, 1 bg,a,pn %xcc, 2f mulx %l2, %o0, %l2 ! multiply interval by clock divisor iff > 1 2: ! ! Update cpu_intrstat. If o3 is set then we will be processing another ! interrupt. Above we have set t_intr_start to %tick, not 0. This ! means a high-level interrupt can arrive and update the same stats ! we're updating. Need to use casx. ! sllx %l1, 4, %o1 ! delay - PIL as byte offset add %o1, CPU_MCPU, %o1 ! CPU_INTRSTAT const too big add %o1, MCPU_INTRSTAT, %o1 ! add parts separately add %o1, %o2, %o1 1: ldx [%o1], %o5 ! old counter in o5 add %o5, %l2, %o0 ! new counter in o0 stx %o0, [%o1 + 8] ! store into intrstat[pil][1] casx [%o1], %o5, %o0 ! and into intrstat[pil][0] cmp %o5, %o0 bne,pn %xcc, 1b nop ! Also update intracct[] lduh [%o2 + CPU_MSTATE], %o1 sllx %o1, 3, %o1 add %o1, CPU_INTRACCT, %o1 add %o1, %o2, %o1 1: ldx [%o1], %o5 add %o5, %l2, %o0 casx [%o1], %o5, %o0 cmp %o5, %o0 bne,pn %xcc, 1b nop ! ! Don't keep a pinned process pinned indefinitely. Bump cpu_intrcnt ! for each interrupt handler we invoke. If we hit INTRCNT_LIMIT, then ! we've crossed the threshold and we should unpin the pinned threads ! by preempt()ing ourselves, which will bubble up the t_intr chain ! until hitting the non-interrupt thread, which will then in turn ! preempt itself allowing the interrupt processing to resume. Finally, ! the scheduler takes over and picks the next thread to run. ! ! If our CPU is quiesced, we cannot preempt because the idle thread ! won't ever re-enter the scheduler, and the interrupt will be forever ! blocked. ! ! If t_intr is NULL, we're not pinning anyone, so we use a simpler ! algorithm. Just check for cpu_kprunrun, and if set then preempt. ! This insures we enter the scheduler if a higher-priority thread ! has become runnable. ! lduh [%o2 + CPU_FLAGS], %o5 ! don't preempt if quiesced andcc %o5, CPU_QUIESCED, %g0 bnz,pn %xcc, 1f ldn [THREAD_REG + T_INTR], %o5 ! pinning anything? brz,pn %o5, 3f ! if not, don't inc intrcnt ldub [%o2 + CPU_INTRCNT], %o5 ! delay - %o5 = cpu_intrcnt inc %o5 cmp %o5, INTRCNT_LIMIT ! have we hit the limit? bl,a,pt %xcc, 1f ! no preempt if < INTRCNT_LIMIT stub %o5, [%o2 + CPU_INTRCNT] ! delay annul - inc CPU_INTRCNT bg,pn %xcc, 2f ! don't inc stats again ! ! We've reached the limit. Set cpu_intrcnt and cpu_kprunrun, and do ! CPU_STATS_ADDQ(cp, sys, intrunpin, 1). Then call preempt. ! mov 1, %o4 ! delay stub %o4, [%o2 + CPU_KPRUNRUN] ldx [%o2 + CPU_STATS_SYS_INTRUNPIN], %o4 inc %o4 stx %o4, [%o2 + CPU_STATS_SYS_INTRUNPIN] ba 2f stub %o5, [%o2 + CPU_INTRCNT] ! delay 3: ! Code for t_intr == NULL ldub [%o2 + CPU_KPRUNRUN], %o5 brz,pt %o5, 1f ! don't preempt unless kprunrun 2: ! Time to call preempt mov %o2, %l3 ! delay - save %o2 call preempt mov %o3, %l2 ! delay - save %o3. mov %l3, %o2 ! restore %o2 mov %l2, %o3 ! restore %o3 wrpr %g0, DISP_LEVEL, %pil ! up from cpu_base_spl 1: ! ! Do we need to call serve_intr_next and do this again? ! brz,a,pt %o3, 0f ld [%o2 + CPU_INTR_ACTV], %o5 ! delay annulled ! ! Restore %pil before calling serve_intr() again. We must check ! CPU_BASE_SPL and set %pil to max(our-pil, CPU_BASE_SPL) ! ld [%o2 + CPU_BASE_SPL], %o4 cmp %o4, %l1 movl %xcc, %l1, %o4 wrpr %g0, %o4, %pil SERVE_INTR_NEXT(%o1, %o2, %l1, %l3, %o4, %o5, %o3, %o0) ba 0b ! compute new stats nop 0: ! ! Clear bit for this level in CPU's interrupt active bitmask. ! mov 1, %o4 sll %o4, %l1, %o4 #ifdef DEBUG ! ! ASSERT(CPU->cpu_intr_actv & (1 << PIL)) ! andcc %o4, %o5, %g0 bnz,pt %xcc, 0f nop ! Do not call panic if a panic is already in progress. sethi %hi(panic_quiesce), %l2 ld [%l2 + %lo(panic_quiesce)], %l2 brnz,pn %l2, 0f nop sethi %hi(intr_thread_actv_bit_not_set), %o0 call panic or %o0, %lo(intr_thread_actv_bit_not_set), %o0 0: #endif /* DEBUG */ andn %o5, %o4, %o5 st %o5, [%o2 + CPU_INTR_ACTV] ! ! If there is still an interrupted thread underneath this one, ! then the interrupt was never blocked and the return is fairly ! simple. Otherwise jump to intr_thread_exit. ! ldn [THREAD_REG + T_INTR], %o4 ! pinned thread brz,pn %o4, intr_thread_exit ! branch if none nop ! ! link the thread back onto the interrupt thread pool ! ldn [%o2 + CPU_INTR_THREAD], %o3 stn %o3, [THREAD_REG + T_LINK] stn THREAD_REG, [%o2 + CPU_INTR_THREAD] ! ! set the thread state to free so kernel debuggers don't see it ! mov TS_FREE, %o5 st %o5, [THREAD_REG + T_STATE] ! ! Switch back to the interrupted thread and return ! stn %o4, [%o2 + CPU_THREAD] membar #StoreLoad ! sync with mutex_exit() mov %o4, THREAD_REG ! If we pinned an interrupt thread, store its starting timestamp. lduh [THREAD_REG + T_FLAGS], %o5 andcc %o5, T_INTR_THREAD, %g0 bz,pt %xcc, 1f ldn [THREAD_REG + T_SP], %sp ! delay - restore %sp add THREAD_REG, T_INTR_START, %o3 ! o3 has &curthread->t_intr_star 0: ldx [%o3], %o4 ! o4 = t_intr_start before RD_TICK(%o5,%l2,%l3,__LINE__) casx [%o3], %o4, %o5 ! put o5 in ts if o4 == ts after cmp %o4, %o5 ! If a high-level interrupt occurred while we were attempting to store ! the timestamp, try again. bne,pn %xcc, 0b ldn [THREAD_REG + T_SP], %sp ! delay - restore %sp 1: ! If the thread being restarted isn't pinning anyone, and no interrupts ! are pending, zero out cpu_intrcnt ldn [THREAD_REG + T_INTR], %o4 brnz,pn %o4, 2f rd SOFTINT, %o4 ! delay set SOFTINT_MASK, %o5 andcc %o4, %o5, %g0 bz,a,pt %xcc, 2f stub %g0, [%o2 + CPU_INTRCNT] ! delay annul 2: jmp %l0 + 8 nop SET_SIZE(intr_thread) /* Not Reached */ ! ! An interrupt returned on what was once (and still might be) ! an interrupt thread stack, but the interrupted process is no longer ! there. This means the interrupt must have blocked. ! ! There is no longer a thread under this one, so put this thread back ! on the CPU's free list and resume the idle thread which will dispatch ! the next thread to run. ! ! All traps below DISP_LEVEL are disabled here, but the mondo interrupt ! is enabled. ! ENTRY_NP(intr_thread_exit) #ifdef TRAPTRACE rdpr %pstate, %l2 andn %l2, PSTATE_IE | PSTATE_AM, %o4 wrpr %g0, %o4, %pstate ! cpu to known state TRACE_PTR(%o4, %o5) GET_TRACE_TICK(%o5, %o0) stxa %o5, [%o4 + TRAP_ENT_TICK]%asi TRACE_SAVE_TL_GL_REGS(%o4, %o5) set TT_INTR_EXIT, %o5 stha %o5, [%o4 + TRAP_ENT_TT]%asi stna %g0, [%o4 + TRAP_ENT_TPC]%asi stxa %g0, [%o4 + TRAP_ENT_TSTATE]%asi stna %sp, [%o4 + TRAP_ENT_SP]%asi stna THREAD_REG, [%o4 + TRAP_ENT_TR]%asi ld [%o2 + CPU_BASE_SPL], %o5 stna %o5, [%o4 + TRAP_ENT_F1]%asi stna %g0, [%o4 + TRAP_ENT_F2]%asi stna %g0, [%o4 + TRAP_ENT_F3]%asi stna %g0, [%o4 + TRAP_ENT_F4]%asi TRACE_NEXT(%o4, %o5, %o0) wrpr %g0, %l2, %pstate #endif /* TRAPTRACE */ ! cpu_stats.sys.intrblk++ ldx [%o2 + CPU_STATS_SYS_INTRBLK], %o4 inc %o4 stx %o4, [%o2 + CPU_STATS_SYS_INTRBLK] ! ! Put thread back on the interrupt thread list. ! ! ! Set the CPU's base SPL level. ! #ifdef DEBUG ! ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL))) ! ld [%o2 + CPU_INTR_ACTV], %o5 mov 1, %o4 sll %o4, %l1, %o4 and %o5, %o4, %o4 brz,pt %o4, 0f nop ! Do not call panic if a panic is already in progress. sethi %hi(panic_quiesce), %l2 ld [%l2 + %lo(panic_quiesce)], %l2 brnz,pn %l2, 0f nop sethi %hi(intr_thread_exit_actv_bit_set), %o0 call panic or %o0, %lo(intr_thread_exit_actv_bit_set), %o0 0: #endif /* DEBUG */ call _intr_set_spl ! set CPU's base SPL level ld [%o2 + CPU_INTR_ACTV], %o5 ! delay - load active mask ! ! set the thread state to free so kernel debuggers don't see it ! mov TS_FREE, %o4 st %o4, [THREAD_REG + T_STATE] ! ! Put thread on either the interrupt pool or the free pool and ! call swtch() to resume another thread. ! ldn [%o2 + CPU_INTR_THREAD], %o5 ! get list pointer stn %o5, [THREAD_REG + T_LINK] call swtch ! switch to best thread stn THREAD_REG, [%o2 + CPU_INTR_THREAD] ! delay - put thread on list ba,a,pt %xcc, . ! swtch() shouldn't return SET_SIZE(intr_thread_exit) .global ftrace_intr_thread_format_str ftrace_intr_thread_format_str: .asciz "intr_thread(): regs=0x%lx, int=0x%lx, pil=0x%lx" #ifdef DEBUG intr_thread_actv_bit_set: .asciz "intr_thread(): cpu_intr_actv bit already set for PIL" intr_thread_actv_bit_not_set: .asciz "intr_thread(): cpu_intr_actv bit not set for PIL" intr_thread_exit_actv_bit_set: .asciz "intr_thread_exit(): cpu_intr_actv bit erroneously set for PIL" intr_thread_t_intr_start_zero: .asciz "intr_thread(): t_intr_start zero upon handler return" #endif /* DEBUG */ #endif /* lint */ #if defined(lint) /* * Handle an interrupt in the current thread * Entry: * %o0 = pointer to regs structure * %o1 = pointer to current intr_vec_t (iv) to be processed * %o2 = pil * %sp = on current thread's kernel stack * %o7 = return linkage to trap code * %g7 = current thread * %pstate = normal globals, interrupts enabled, * privileged, fp disabled * %pil = PIL_MAX * * Register Usage * %l0 = return linkage * %l1 = old stack * %l2 - %l3 = scratch * %l4 - %l7 = reserved for sys_trap * %o3 = cpu * %o0 = scratch * %o4 - %o5 = scratch */ /* ARGSUSED */ void current_thread(struct regs *regs, uint64_t iv_p, uint_t pil) {} #else /* lint */ ENTRY_NP(current_thread) mov %o7, %l0 ldn [THREAD_REG + T_CPU], %o3 ldn [THREAD_REG + T_ONFAULT], %l2 brz,pt %l2, no_onfault ! branch if no onfault label set nop stn %g0, [THREAD_REG + T_ONFAULT]! clear onfault label ldn [THREAD_REG + T_LOFAULT], %l3 stn %g0, [THREAD_REG + T_LOFAULT]! clear lofault data sub %o2, LOCK_LEVEL + 1, %o5 sll %o5, CPTRSHIFT, %o5 add %o5, CPU_OFD, %o4 ! %o4 has on_fault data offset stn %l2, [%o3 + %o4] ! save onfault label for pil %o2 add %o5, CPU_LFD, %o4 ! %o4 has lofault data offset stn %l3, [%o3 + %o4] ! save lofault data for pil %o2 no_onfault: ldn [THREAD_REG + T_ONTRAP], %l2 brz,pt %l2, 6f ! branch if no on_trap protection nop stn %g0, [THREAD_REG + T_ONTRAP]! clear on_trap protection sub %o2, LOCK_LEVEL + 1, %o5 sll %o5, CPTRSHIFT, %o5 add %o5, CPU_OTD, %o4 ! %o4 has on_trap data offset stn %l2, [%o3 + %o4] ! save on_trap label for pil %o2 ! ! Set bit for this level in CPU's active interrupt bitmask. ! 6: ld [%o3 + CPU_INTR_ACTV], %o5 ! o5 has cpu_intr_actv b4 chng mov 1, %o4 sll %o4, %o2, %o4 ! construct mask for level #ifdef DEBUG ! ! ASSERT(!(CPU->cpu_intr_actv & (1 << PIL))) ! andcc %o5, %o4, %g0 bz,pt %xcc, 0f nop ! Do not call panic if a panic is already in progress. sethi %hi(panic_quiesce), %l2 ld [%l2 + %lo(panic_quiesce)], %l2 brnz,pn %l2, 0f nop sethi %hi(current_thread_actv_bit_set), %o0 call panic or %o0, %lo(current_thread_actv_bit_set), %o0 0: #endif /* DEBUG */ or %o5, %o4, %o4 ! ! See if we are interrupting another high-level interrupt. ! srl %o5, LOCK_LEVEL + 1, %o5 ! only look at high-level bits brz,pt %o5, 1f st %o4, [%o3 + CPU_INTR_ACTV] ! delay - store active mask ! ! We have interrupted another high-level interrupt. Find its PIL, ! compute the interval it ran for, and update its cumulative counter. ! ! Register usage: ! o2 = PIL of this interrupt ! o5 = high PIL bits of INTR_ACTV (not including this PIL) ! l1 = bitmask used to find other active high-level PIL ! o4 = index of bit set in l1 ! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the ! interrupted high-level interrupt. ! Create mask for cpu_intr_actv. Begin by looking for bits set ! at one level below the current PIL. Since %o5 contains the active ! mask already shifted right by (LOCK_LEVEL + 1), we start by looking ! at bit (current_pil - (LOCK_LEVEL + 2)). sub %o2, LOCK_LEVEL + 2, %o4 mov 1, %l1 sll %l1, %o4, %l1 2: #ifdef DEBUG ! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge) brnz,pt %l1, 9f nop ! Don't panic if a panic is already in progress. sethi %hi(panic_quiesce), %l3 ld [%l3 + %lo(panic_quiesce)], %l3 brnz,pn %l3, 9f nop sethi %hi(current_thread_nested_PIL_not_found), %o0 call panic or %o0, %lo(current_thread_nested_PIL_not_found), %o0 9: #endif /* DEBUG */ andcc %l1, %o5, %g0 ! test mask against high-level bits of bnz %xcc, 3f ! cpu_intr_actv nop srl %l1, 1, %l1 ! No match. Try next lower PIL. ba,pt %xcc, 2b sub %o4, 1, %o4 ! delay - decrement PIL 3: sll %o4, 3, %o4 ! index to byte offset add %o4, CPU_MCPU, %l1 ! CPU_PIL_HIGH_START is too large add %l1, MCPU_PIL_HIGH_START, %l1 ldx [%o3 + %l1], %l3 ! load starting timestamp #ifdef DEBUG brnz,pt %l3, 9f nop ! Don't panic if a panic is already in progress. sethi %hi(panic_quiesce), %l1 ld [%l1 + %lo(panic_quiesce)], %l1 brnz,pn %l1, 9f nop srl %o4, 3, %o1 ! Find interrupted PIL for panic add %o1, LOCK_LEVEL + 1, %o1 sethi %hi(current_thread_nested_pil_zero), %o0 call panic or %o0, %lo(current_thread_nested_pil_zero), %o0 9: #endif /* DEBUG */ RD_TICK_NO_SUSPEND_CHECK(%l1, %l2) sub %l1, %l3, %l3 ! interval in %l3 ! ! Check for Energy Star mode ! lduh [%o3 + CPU_DIVISOR], %l1 ! %l1 = clock divisor cmp %l1, 1 bg,a,pn %xcc, 2f mulx %l3, %l1, %l3 ! multiply interval by clock divisor iff > 1 2: ! ! We need to find the CPU offset of the cumulative counter. We start ! with %o4, which has (PIL - (LOCK_LEVEL + 1)) * 8. We need PIL * 16, ! so we shift left 1, then add (LOCK_LEVEL + 1) * 16, which is ! CPU_INTRSTAT_LOW_PIL_OFFSET. ! sll %o4, 1, %o4 add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT const too large add %o4, MCPU_INTRSTAT, %o4 ! add parts separately add %o4, CPU_INTRSTAT_LOW_PIL_OFFSET, %o4 ldx [%o3 + %o4], %l1 ! old counter in l1 add %l1, %l3, %l1 ! new counter in l1 stx %l1, [%o3 + %o4] ! store new counter ! Also update intracct[] lduh [%o3 + CPU_MSTATE], %o4 sllx %o4, 3, %o4 add %o4, CPU_INTRACCT, %o4 ldx [%o3 + %o4], %l1 add %l1, %l3, %l1 ! Another high-level interrupt is active below this one, so ! there is no need to check for an interrupt thread. That will be ! done by the lowest priority high-level interrupt active. ba,pt %xcc, 5f stx %l1, [%o3 + %o4] ! delay - store new counter 1: ! If we haven't interrupted another high-level interrupt, we may be ! interrupting a low level interrupt thread. If so, compute its interval ! and update its cumulative counter. lduh [THREAD_REG + T_FLAGS], %o4 andcc %o4, T_INTR_THREAD, %g0 bz,pt %xcc, 4f nop ! We have interrupted an interrupt thread. Take timestamp, compute ! interval, update cumulative counter. ! Check t_intr_start. If it is zero, either intr_thread() or ! current_thread() (at a lower PIL, of course) already did ! the accounting for the underlying interrupt thread. ldx [THREAD_REG + T_INTR_START], %o5 brz,pn %o5, 4f nop stx %g0, [THREAD_REG + T_INTR_START] RD_TICK_NO_SUSPEND_CHECK(%o4, %l2) sub %o4, %o5, %o5 ! o5 has the interval ! Check for Energy Star mode lduh [%o3 + CPU_DIVISOR], %o4 ! %o4 = clock divisor cmp %o4, 1 bg,a,pn %xcc, 2f mulx %o5, %o4, %o5 ! multiply interval by clock divisor iff > 1 2: ldub [THREAD_REG + T_PIL], %o4 sllx %o4, 4, %o4 ! PIL index to byte offset add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT const too large add %o4, MCPU_INTRSTAT, %o4 ! add parts separately ldx [%o3 + %o4], %l2 ! old counter in l2 add %l2, %o5, %l2 ! new counter in l2 stx %l2, [%o3 + %o4] ! store new counter ! Also update intracct[] lduh [%o3 + CPU_MSTATE], %o4 sllx %o4, 3, %o4 add %o4, CPU_INTRACCT, %o4 ldx [%o3 + %o4], %l2 add %l2, %o5, %l2 stx %l2, [%o3 + %o4] 4: ! ! Handle high-level interrupts on separate interrupt stack. ! No other high-level interrupts are active, so switch to int stack. ! mov %sp, %l1 ldn [%o3 + CPU_INTR_STACK], %l3 sub %l3, STACK_BIAS, %sp 5: #ifdef DEBUG ! ! ASSERT(%o2 > LOCK_LEVEL) ! cmp %o2, LOCK_LEVEL bg,pt %xcc, 3f nop mov CE_PANIC, %o0 sethi %hi(current_thread_wrong_pil), %o1 call cmn_err ! %o2 has the %pil already or %o1, %lo(current_thread_wrong_pil), %o1 #endif 3: ! Store starting timestamp for this PIL in CPU structure at ! cpu.cpu_m.pil_high_start[PIL - (LOCK_LEVEL + 1)] sub %o2, LOCK_LEVEL + 1, %o4 ! convert PIL to array index sllx %o4, 3, %o4 ! index to byte offset add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large add %o4, MCPU_PIL_HIGH_START, %o4 RD_TICK_NO_SUSPEND_CHECK(%o5, %l2) stx %o5, [%o3 + %o4] wrpr %g0, %o2, %pil ! enable interrupts ! ! call the handler ! SERVE_INTR_PRE(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0) 1: SERVE_INTR(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0) brz,a,pt %o2, 0f ! if %o2, more intrs await rdpr %pil, %o2 ! delay annulled SERVE_INTR_NEXT(%o1, %o3, %l2, %l3, %o4, %o5, %o2, %o0) ba 1b nop 0: wrpr %g0, PIL_MAX, %pil ! disable interrupts (1-15) cmp %o2, PIL_15 bne,pt %xcc, 3f nop sethi %hi(cpc_level15_inum), %o1 ldx [%o1 + %lo(cpc_level15_inum)], %o1 ! arg for intr_enqueue_req brz %o1, 3f nop rdpr %pstate, %g5 andn %g5, PSTATE_IE, %g1 wrpr %g0, %g1, %pstate ! Disable vec interrupts call intr_enqueue_req ! preserves %g5 mov PIL_15, %o0 ! clear perfcntr overflow mov 1, %o0 sllx %o0, PIL_15, %o0 wr %o0, CLEAR_SOFTINT wrpr %g0, %g5, %pstate ! Enable vec interrupts 3: cmp %o2, PIL_14 be tick_rtt ! cpu-specific tick processing nop .global current_thread_complete current_thread_complete: ! ! Register usage: ! ! %l1 = stack pointer ! %l2 = CPU_INTR_ACTV >> (LOCK_LEVEL + 1) ! %o2 = PIL ! %o3 = CPU pointer ! %o4, %o5, %l3, %l4, %l5 = scratch ! ldn [THREAD_REG + T_CPU], %o3 ! ! Clear bit for this level in CPU's interrupt active bitmask. ! ld [%o3 + CPU_INTR_ACTV], %l2 mov 1, %o5 sll %o5, %o2, %o5 #ifdef DEBUG ! ! ASSERT(CPU->cpu_intr_actv & (1 << PIL)) ! andcc %l2, %o5, %g0 bnz,pt %xcc, 0f nop ! Do not call panic if a panic is already in progress. sethi %hi(panic_quiesce), %l2 ld [%l2 + %lo(panic_quiesce)], %l2 brnz,pn %l2, 0f nop sethi %hi(current_thread_actv_bit_not_set), %o0 call panic or %o0, %lo(current_thread_actv_bit_not_set), %o0 0: #endif /* DEBUG */ andn %l2, %o5, %l2 st %l2, [%o3 + CPU_INTR_ACTV] ! Take timestamp, compute interval, update cumulative counter. sub %o2, LOCK_LEVEL + 1, %o4 ! PIL to array index sllx %o4, 3, %o4 ! index to byte offset add %o4, CPU_MCPU, %o4 ! CPU_PIL_HIGH_START is too large add %o4, MCPU_PIL_HIGH_START, %o4 RD_TICK_NO_SUSPEND_CHECK(%o5, %o0) ldx [%o3 + %o4], %o0 #ifdef DEBUG ! ASSERT(cpu.cpu_m.pil_high_start[pil - (LOCK_LEVEL + 1)] != 0) brnz,pt %o0, 9f nop ! Don't panic if a panic is already in progress. sethi %hi(panic_quiesce), %l2 ld [%l2 + %lo(panic_quiesce)], %l2 brnz,pn %l2, 9f nop sethi %hi(current_thread_timestamp_zero), %o0 call panic or %o0, %lo(current_thread_timestamp_zero), %o0 9: #endif /* DEBUG */ stx %g0, [%o3 + %o4] sub %o5, %o0, %o5 ! interval in o5 ! Check for Energy Star mode lduh [%o3 + CPU_DIVISOR], %o4 ! %o4 = clock divisor cmp %o4, 1 bg,a,pn %xcc, 2f mulx %o5, %o4, %o5 ! multiply interval by clock divisor iff > 1 2: sllx %o2, 4, %o4 ! PIL index to byte offset add %o4, CPU_MCPU, %o4 ! CPU_INTRSTAT too large add %o4, MCPU_INTRSTAT, %o4 ! add parts separately ldx [%o3 + %o4], %o0 ! old counter in o0 add %o0, %o5, %o0 ! new counter in o0 stx %o0, [%o3 + %o4] ! store new counter ! Also update intracct[] lduh [%o3 + CPU_MSTATE], %o4 sllx %o4, 3, %o4 add %o4, CPU_INTRACCT, %o4 ldx [%o3 + %o4], %o0 add %o0, %o5, %o0 stx %o0, [%o3 + %o4] ! ! get back on current thread's stack ! srl %l2, LOCK_LEVEL + 1, %l2 tst %l2 ! any more high-level ints? movz %xcc, %l1, %sp ! ! Current register usage: ! o2 = PIL ! o3 = CPU pointer ! l0 = return address ! l2 = intr_actv shifted right ! bz,pt %xcc, 3f ! if l2 was zero, no more ints nop ! ! We found another high-level interrupt active below the one that just ! returned. Store a starting timestamp for it in the CPU structure. ! ! Use cpu_intr_actv to find the cpu_pil_high_start[] offset for the ! interrupted high-level interrupt. ! Create mask for cpu_intr_actv. Begin by looking for bits set ! at one level below the current PIL. Since %l2 contains the active ! mask already shifted right by (LOCK_LEVEL + 1), we start by looking ! at bit (current_pil - (LOCK_LEVEL + 2)). ! %l1 = mask, %o5 = index of bit set in mask ! mov 1, %l1 sub %o2, LOCK_LEVEL + 2, %o5 sll %l1, %o5, %l1 ! l1 = mask for level 1: #ifdef DEBUG ! ASSERT(%l1 != 0) (we didn't shift the bit off the right edge) brnz,pt %l1, 9f nop sethi %hi(current_thread_nested_PIL_not_found), %o0 call panic or %o0, %lo(current_thread_nested_PIL_not_found), %o0 9: #endif /* DEBUG */ andcc %l1, %l2, %g0 ! test mask against high-level bits of bnz %xcc, 2f ! cpu_intr_actv nop srl %l1, 1, %l1 ! No match. Try next lower PIL. ba,pt %xcc, 1b sub %o5, 1, %o5 ! delay - decrement PIL 2: sll %o5, 3, %o5 ! convert array index to byte offset add %o5, CPU_MCPU, %o5 ! CPU_PIL_HIGH_START is too large add %o5, MCPU_PIL_HIGH_START, %o5 RD_TICK_NO_SUSPEND_CHECK(%o4, %l2) ! Another high-level interrupt is active below this one, so ! there is no need to check for an interrupt thread. That will be ! done by the lowest priority high-level interrupt active. ba,pt %xcc, 7f stx %o4, [%o3 + %o5] ! delay - store timestamp 3: ! If we haven't interrupted another high-level interrupt, we may have ! interrupted a low level interrupt thread. If so, store a starting ! timestamp in its thread structure. lduh [THREAD_REG + T_FLAGS], %o4 andcc %o4, T_INTR_THREAD, %g0 bz,pt %xcc, 7f nop RD_TICK_NO_SUSPEND_CHECK(%o4, %l2) stx %o4, [THREAD_REG + T_INTR_START] 7: sub %o2, LOCK_LEVEL + 1, %o4 sll %o4, CPTRSHIFT, %o5 ! Check on_trap saved area and restore as needed add %o5, CPU_OTD, %o4 ldn [%o3 + %o4], %l2 brz,pt %l2, no_ontrp_restore nop stn %l2, [THREAD_REG + T_ONTRAP] ! restore stn %g0, [%o3 + %o4] ! clear no_ontrp_restore: ! Check on_fault saved area and restore as needed add %o5, CPU_OFD, %o4 ldn [%o3 + %o4], %l2 brz,pt %l2, 8f nop stn %l2, [THREAD_REG + T_ONFAULT] ! restore stn %g0, [%o3 + %o4] ! clear add %o5, CPU_LFD, %o4 ldn [%o3 + %o4], %l2 stn %l2, [THREAD_REG + T_LOFAULT] ! restore stn %g0, [%o3 + %o4] ! clear 8: ! Enable interrupts and return jmp %l0 + 8 wrpr %g0, %o2, %pil ! enable interrupts SET_SIZE(current_thread) #ifdef DEBUG current_thread_wrong_pil: .asciz "current_thread: unexpected pil level: %d" current_thread_actv_bit_set: .asciz "current_thread(): cpu_intr_actv bit already set for PIL" current_thread_actv_bit_not_set: .asciz "current_thread(): cpu_intr_actv bit not set for PIL" current_thread_nested_pil_zero: .asciz "current_thread(): timestamp zero for nested PIL %d" current_thread_timestamp_zero: .asciz "current_thread(): timestamp zero upon handler return" current_thread_nested_PIL_not_found: .asciz "current_thread: couldn't find nested high-level PIL" #endif /* DEBUG */ #endif /* lint */ /* * Return a thread's interrupt level. * Since this isn't saved anywhere but in %l4 on interrupt entry, we * must dig it out of the save area. * * Caller 'swears' that this really is an interrupt thread. * * int * intr_level(t) * kthread_id_t t; */ #if defined(lint) /* ARGSUSED */ int intr_level(kthread_id_t t) { return (0); } #else /* lint */ ENTRY_NP(intr_level) retl ldub [%o0 + T_PIL], %o0 ! return saved pil SET_SIZE(intr_level) #endif /* lint */ #if defined(lint) /* ARGSUSED */ int disable_pil_intr() { return (0); } #else /* lint */ ENTRY_NP(disable_pil_intr) rdpr %pil, %o0 retl wrpr %g0, PIL_MAX, %pil ! disable interrupts (1-15) SET_SIZE(disable_pil_intr) #endif /* lint */ #if defined(lint) /* ARGSUSED */ void enable_pil_intr(int pil_save) {} #else /* lint */ ENTRY_NP(enable_pil_intr) retl wrpr %o0, %pil SET_SIZE(enable_pil_intr) #endif /* lint */ #if defined(lint) /* ARGSUSED */ uint_t disable_vec_intr(void) { return (0); } #else /* lint */ ENTRY_NP(disable_vec_intr) rdpr %pstate, %o0 andn %o0, PSTATE_IE, %g1 retl wrpr %g0, %g1, %pstate ! disable interrupt SET_SIZE(disable_vec_intr) #endif /* lint */ #if defined(lint) /* ARGSUSED */ void enable_vec_intr(uint_t pstate_save) {} #else /* lint */ ENTRY_NP(enable_vec_intr) retl wrpr %g0, %o0, %pstate SET_SIZE(enable_vec_intr) #endif /* lint */ #if defined(lint) void cbe_level14(void) {} #else /* lint */ ENTRY_NP(cbe_level14) save %sp, -SA(MINFRAME), %sp ! get a new window ! ! Make sure that this is from TICK_COMPARE; if not just return ! rd SOFTINT, %l1 set (TICK_INT_MASK | STICK_INT_MASK), %o2 andcc %l1, %o2, %g0 bz,pn %icc, 2f nop CPU_ADDR(%o1, %o2) call cyclic_fire mov %o1, %o0 2: ret restore %g0, 1, %o0 SET_SIZE(cbe_level14) #endif /* lint */ #if defined(lint) /* ARGSUSED */ void kdi_setsoftint(uint64_t iv_p) {} #else /* lint */ ENTRY_NP(kdi_setsoftint) save %sp, -SA(MINFRAME), %sp ! get a new window rdpr %pstate, %l5 andn %l5, PSTATE_IE, %l1 wrpr %l1, %pstate ! disable interrupt ! ! We have a pointer to an interrupt vector data structure. ! Put the request on the cpu's softint priority list and ! set %set_softint. ! ! Register usage ! %i0 - pointer to intr_vec_t (iv) ! %l2 - requested pil ! %l4 - cpu ! %l5 - pstate ! %l1, %l3, %l6 - temps ! ! check if a softint is pending for this softint, ! if one is pending, don't bother queuing another. ! lduh [%i0 + IV_FLAGS], %l1 ! %l1 = iv->iv_flags and %l1, IV_SOFTINT_PEND, %l6 ! %l6 = iv->iv_flags & IV_SOFTINT_PEND brnz,pn %l6, 4f ! branch if softint is already pending or %l1, IV_SOFTINT_PEND, %l2 sth %l2, [%i0 + IV_FLAGS] ! Set IV_SOFTINT_PEND flag CPU_ADDR(%l4, %l2) ! %l4 = cpu lduh [%i0 + IV_PIL], %l2 ! %l2 = iv->iv_pil ! ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list ! sll %l2, CPTRSHIFT, %l0 ! %l0 = offset to pil entry add %l4, INTR_TAIL, %l6 ! %l6 = &cpu->m_cpu.intr_tail ldn [%l6 + %l0], %l1 ! %l1 = cpu->m_cpu.intr_tail[pil] ! current tail (ct) brz,pt %l1, 2f ! branch if current tail is NULL stn %i0, [%l6 + %l0] ! make intr_vec_t (iv) as new tail ! ! there's pending intr_vec_t already ! lduh [%l1 + IV_FLAGS], %l6 ! %l6 = ct->iv_flags and %l6, IV_SOFTINT_MT, %l6 ! %l6 = ct->iv_flags & IV_SOFTINT_MT brz,pt %l6, 1f ! check for Multi target softint flag add %l1, IV_PIL_NEXT, %l3 ! %l3 = &ct->iv_pil_next ld [%l4 + CPU_ID], %l6 ! for multi target softint, use cpuid sll %l6, CPTRSHIFT, %l6 ! calculate offset address from cpuid add %l3, %l6, %l3 ! %l3 = &ct->iv_xpil_next[cpuid] 1: ! ! update old tail ! ba,pt %xcc, 3f stn %i0, [%l3] ! [%l3] = iv, set pil_next field 2: ! ! no pending intr_vec_t; make intr_vec_t as new head ! add %l4, INTR_HEAD, %l6 ! %l6 = &cpu->m_cpu.intr_head[pil] stn %i0, [%l6 + %l0] ! cpu->m_cpu.intr_head[pil] = iv 3: ! ! Write %set_softint with (1<iv_pil ! ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list ! sll %g2, CPTRSHIFT, %g7 ! %g7 = offset to pil entry add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail ldn [%g6 + %g7], %g5 ! %g5 = cpu->m_cpu.intr_tail[pil] ! current tail (ct) brz,pt %g5, 1f ! branch if current tail is NULL stn %g1, [%g6 + %g7] ! make intr_rec_t (iv) as new tail ! ! there's pending intr_vec_t already ! lduh [%g5 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT brz,pt %g6, 0f ! check for Multi target softint flag add %g5, IV_PIL_NEXT, %g3 ! %g3 = &ct->iv_pil_next ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid add %g3, %g6, %g3 ! %g3 = &ct->iv_xpil_next[cpuid] 0: ! ! update old tail ! ba,pt %xcc, 2f stn %g1, [%g3] ! [%g3] = iv, set pil_next field 1: ! ! no pending intr_vec_t; make intr_vec_t as new head ! add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] stn %g1, [%g6 + %g7] ! cpu->m_cpu.intr_head[pil] = iv 2: #ifdef TRAPTRACE TRACE_PTR(%g5, %g6) GET_TRACE_TICK(%g6, %g3) stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick TRACE_SAVE_TL_GL_REGS(%g5, %g6) rdpr %tt, %g6 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt rdpr %tpc, %g6 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc rdpr %tstate, %g6 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp stna %g1, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = iv ldn [%g1 + IV_PIL_NEXT], %g6 ! stna %g6, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = iv->iv_pil_next add %g4, INTR_HEAD, %g6 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_head[pil] stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil] add %g4, INTR_TAIL, %g6 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_tail[pil] stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil] stna %g2, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil TRACE_NEXT(%g5, %g6, %g3) #endif /* TRAPTRACE */ ! ! Write %set_softint with (1<iv_pil sll %g2, CPTRSHIFT, %g7 ! %g7 = offset to pil entry add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail ldn [%g6 + %g7], %g5 ! %g5 = cpu->m_cpu.intr_tail[pil] ! current tail (ct) brz,pt %g5, 2f ! branch if current tail is NULL stn %g3, [%g6 + %g7] ! make intr_vec_t (iv) as new tail ! cpu->m_cpu.intr_tail[pil] = iv ! ! there's pending intr_vec_t already ! lduh [%g5 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT brz,pt %g6, 1f ! check for Multi target softint flag add %g5, IV_PIL_NEXT, %g5 ! %g5 = &ct->iv_pil_next ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid add %g5, %g6, %g5 ! %g5 = &ct->iv_xpil_next[cpuid] 1: ! ! update old tail ! ba,pt %xcc, 3f stn %g3, [%g5] ! [%g5] = iv, set pil_next field 2: ! ! no pending intr_vec_t; make intr_vec_t as new head ! add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] stn %g3, [%g6 + %g7] ! cpu->m_cpu.intr_head[pil] = iv 3: #ifdef TRAPTRACE TRACE_PTR(%g5, %g6) TRACE_SAVE_TL_GL_REGS(%g5, %g6) rdpr %tt, %g6 stha %g6, [%g5 + TRAP_ENT_TT]%asi ! trap_type = %tt` rdpr %tpc, %g6 stna %g6, [%g5 + TRAP_ENT_TPC]%asi ! trap_pc = %tpc rdpr %tstate, %g6 stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! trap_tstate = %tstate stna %sp, [%g5 + TRAP_ENT_SP]%asi ! trap_sp = %sp stna %g3, [%g5 + TRAP_ENT_TR]%asi ! trap_tr = iv stna %g1, [%g5 + TRAP_ENT_F1]%asi ! trap_f1 = pil mask add %g4, INTR_HEAD, %g6 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_head[pil] stna %g6, [%g5 + TRAP_ENT_F2]%asi ! trap_f2 = intr_head[pil] add %g4, INTR_TAIL, %g6 ldn [%g6 + %g7], %g6 ! %g6=cpu->m_cpu.intr_tail[pil] stna %g6, [%g5 + TRAP_ENT_F3]%asi ! trap_f3 = intr_tail[pil] stna %g2, [%g5 + TRAP_ENT_F4]%asi ! trap_f4 = pil GET_TRACE_TICK(%g6, %g7) stxa %g6, [%g5 + TRAP_ENT_TICK]%asi ! trap_tick = %tick TRACE_NEXT(%g5, %g6, %g7) #endif /* TRAPTRACE */ mov 1, %g6 ! %g6 = 1 sll %g6, %g2, %g6 ! %g6 = 1 << pil or %g1, %g6, %g1 ! %g1 |= (1 << pil), pil mask ldn [%g3 + IV_VEC_NEXT], %g3 ! %g3 = pointer to next intr_vec_t (iv) brnz,pn %g3, 0b ! iv->iv_vec_next is non NULL, goto 0b nop wr %g1, SET_SOFTINT ! triggered one or more pil softints retry .no_ivintr: ! no_ivintr: arguments: rp, inum (%g1), pil (%g2 == 0) mov %g2, %g3 mov %g1, %g2 set no_ivintr, %g1 ba,pt %xcc, sys_trap mov PIL_15, %g4 SET_SIZE(setvecint_tl1) #endif /* lint */ #if defined(lint) /*ARGSUSED*/ void wr_clr_softint(uint_t value) {} #else ENTRY_NP(wr_clr_softint) retl wr %o0, CLEAR_SOFTINT SET_SIZE(wr_clr_softint) #endif /* lint */ #if defined(lint) /*ARGSUSED*/ void intr_enqueue_req(uint_t pil, uint64_t inum) {} #else /* lint */ /* * intr_enqueue_req * * %o0 - pil * %o1 - pointer to intr_vec_t (iv) * %o5 - preserved * %g5 - preserved */ ENTRY_NP(intr_enqueue_req) ! CPU_ADDR(%g4, %g1) ! %g4 = cpu ! ! Insert intr_vec_t (iv) to appropriate cpu's softint priority list ! sll %o0, CPTRSHIFT, %o0 ! %o0 = offset to pil entry add %g4, INTR_TAIL, %g6 ! %g6 = &cpu->m_cpu.intr_tail ldn [%o0 + %g6], %g1 ! %g1 = cpu->m_cpu.intr_tail[pil] ! current tail (ct) brz,pt %g1, 2f ! branch if current tail is NULL stn %o1, [%g6 + %o0] ! make intr_vec_t (iv) as new tail ! ! there's pending intr_vec_t already ! lduh [%g1 + IV_FLAGS], %g6 ! %g6 = ct->iv_flags and %g6, IV_SOFTINT_MT, %g6 ! %g6 = ct->iv_flags & IV_SOFTINT_MT brz,pt %g6, 1f ! check for Multi target softint flag add %g1, IV_PIL_NEXT, %g3 ! %g3 = &ct->iv_pil_next ld [%g4 + CPU_ID], %g6 ! for multi target softint, use cpuid sll %g6, CPTRSHIFT, %g6 ! calculate offset address from cpuid add %g3, %g6, %g3 ! %g3 = &ct->iv_xpil_next[cpuid] 1: ! ! update old tail ! ba,pt %xcc, 3f stn %o1, [%g3] ! {%g5] = iv, set pil_next field 2: ! ! no intr_vec_t's queued so make intr_vec_t as new head ! add %g4, INTR_HEAD, %g6 ! %g6 = &cpu->m_cpu.intr_head[pil] stn %o1, [%g6 + %o0] ! cpu->m_cpu.intr_head[pil] = iv 3: retl nop SET_SIZE(intr_enqueue_req) #endif /* lint */ /* * Set CPU's base SPL level, based on which interrupt levels are active. * Called at spl7 or above. */ #if defined(lint) void set_base_spl(void) {} #else /* lint */ ENTRY_NP(set_base_spl) ldn [THREAD_REG + T_CPU], %o2 ! load CPU pointer ld [%o2 + CPU_INTR_ACTV], %o5 ! load active interrupts mask /* * WARNING: non-standard callinq sequence; do not call from C * %o2 = pointer to CPU * %o5 = updated CPU_INTR_ACTV */ _intr_set_spl: ! intr_thread_exit enters here ! ! Determine highest interrupt level active. Several could be blocked ! at higher levels than this one, so must convert flags to a PIL ! Normally nothing will be blocked, so test this first. ! brz,pt %o5, 1f ! nothing active sra %o5, 11, %o3 ! delay - set %o3 to bits 15-11 set _intr_flag_table, %o1 tst %o3 ! see if any of the bits set ldub [%o1 + %o3], %o3 ! load bit number bnz,a,pn %xcc, 1f ! yes, add 10 and we're done add %o3, 11-1, %o3 ! delay - add bit number - 1 sra %o5, 6, %o3 ! test bits 10-6 tst %o3 ldub [%o1 + %o3], %o3 bnz,a,pn %xcc, 1f add %o3, 6-1, %o3 sra %o5, 1, %o3 ! test bits 5-1 ldub [%o1 + %o3], %o3 ! ! highest interrupt level number active is in %l6 ! 1: retl st %o3, [%o2 + CPU_BASE_SPL] ! delay - store base priority SET_SIZE(set_base_spl) /* * Table that finds the most significant bit set in a five bit field. * Each entry is the high-order bit number + 1 of it's index in the table. * This read-only data is in the text segment. */ _intr_flag_table: .byte 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4 .byte 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 .align 4 #endif /* lint */ /* * int * intr_passivate(from, to) * kthread_id_t from; interrupt thread * kthread_id_t to; interrupted thread */ #if defined(lint) /* ARGSUSED */ int intr_passivate(kthread_id_t from, kthread_id_t to) { return (0); } #else /* lint */ ENTRY_NP(intr_passivate) save %sp, -SA(MINFRAME), %sp ! get a new window flushw ! force register windows to stack ! ! restore registers from the base of the stack of the interrupt thread. ! ldn [%i0 + T_STACK], %i2 ! get stack save area pointer ldn [%i2 + (0*GREGSIZE)], %l0 ! load locals ldn [%i2 + (1*GREGSIZE)], %l1 ldn [%i2 + (2*GREGSIZE)], %l2 ldn [%i2 + (3*GREGSIZE)], %l3 ldn [%i2 + (4*GREGSIZE)], %l4 ldn [%i2 + (5*GREGSIZE)], %l5 ldn [%i2 + (6*GREGSIZE)], %l6 ldn [%i2 + (7*GREGSIZE)], %l7 ldn [%i2 + (8*GREGSIZE)], %o0 ! put ins from stack in outs ldn [%i2 + (9*GREGSIZE)], %o1 ldn [%i2 + (10*GREGSIZE)], %o2 ldn [%i2 + (11*GREGSIZE)], %o3 ldn [%i2 + (12*GREGSIZE)], %o4 ldn [%i2 + (13*GREGSIZE)], %o5 ldn [%i2 + (14*GREGSIZE)], %i4 ! copy stack/pointer without using %sp ldn [%i2 + (15*GREGSIZE)], %i5 ! ! put registers into the save area at the top of the interrupted ! thread's stack, pointed to by %l7 in the save area just loaded. ! ldn [%i1 + T_SP], %i3 ! get stack save area pointer stn %l0, [%i3 + STACK_BIAS + (0*GREGSIZE)] ! save locals stn %l1, [%i3 + STACK_BIAS + (1*GREGSIZE)] stn %l2, [%i3 + STACK_BIAS + (2*GREGSIZE)] stn %l3, [%i3 + STACK_BIAS + (3*GREGSIZE)] stn %l4, [%i3 + STACK_BIAS + (4*GREGSIZE)] stn %l5, [%i3 + STACK_BIAS + (5*GREGSIZE)] stn %l6, [%i3 + STACK_BIAS + (6*GREGSIZE)] stn %l7, [%i3 + STACK_BIAS + (7*GREGSIZE)] stn %o0, [%i3 + STACK_BIAS + (8*GREGSIZE)] ! save ins using outs stn %o1, [%i3 + STACK_BIAS + (9*GREGSIZE)] stn %o2, [%i3 + STACK_BIAS + (10*GREGSIZE)] stn %o3, [%i3 + STACK_BIAS + (11*GREGSIZE)] stn %o4, [%i3 + STACK_BIAS + (12*GREGSIZE)] stn %o5, [%i3 + STACK_BIAS + (13*GREGSIZE)] stn %i4, [%i3 + STACK_BIAS + (14*GREGSIZE)] ! fp, %i7 copied using %i4 stn %i5, [%i3 + STACK_BIAS + (15*GREGSIZE)] stn %g0, [%i2 + ((8+6)*GREGSIZE)] ! clear fp in save area ! load saved pil for return ldub [%i0 + T_PIL], %i0 ret restore SET_SIZE(intr_passivate) #endif /* lint */ #if defined(lint) /* * intr_get_time() is a resource for interrupt handlers to determine how * much time has been spent handling the current interrupt. Such a function * is needed because higher level interrupts can arrive during the * processing of an interrupt, thus making direct comparisons of %tick by * the handler inaccurate. intr_get_time() only returns time spent in the * current interrupt handler. * * The caller must be calling from an interrupt handler running at a pil * below or at lock level. Timings are not provided for high-level * interrupts. * * The first time intr_get_time() is called while handling an interrupt, * it returns the time since the interrupt handler was invoked. Subsequent * calls will return the time since the prior call to intr_get_time(). Time * is returned as ticks, adjusted for any clock divisor due to power * management. Use tick2ns() to convert ticks to nsec. Warning: ticks may * not be the same across CPUs. * * Theory Of Intrstat[][]: * * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two * uint64_ts per pil. * * intrstat[pil][0] is a cumulative count of the number of ticks spent * handling all interrupts at the specified pil on this CPU. It is * exported via kstats to the user. * * intrstat[pil][1] is always a count of ticks less than or equal to the * value in [0]. The difference between [1] and [0] is the value returned * by a call to intr_get_time(). At the start of interrupt processing, * [0] and [1] will be equal (or nearly so). As the interrupt consumes * time, [0] will increase, but [1] will remain the same. A call to * intr_get_time() will return the difference, then update [1] to be the * same as [0]. Future calls will return the time since the last call. * Finally, when the interrupt completes, [1] is updated to the same as [0]. * * Implementation: * * intr_get_time() works much like a higher level interrupt arriving. It * "checkpoints" the timing information by incrementing intrstat[pil][0] * to include elapsed running time, and by setting t_intr_start to %tick. * It then sets the return value to intrstat[pil][0] - intrstat[pil][1], * and updates intrstat[pil][1] to be the same as the new value of * intrstat[pil][0]. * * In the normal handling of interrupts, after an interrupt handler returns * and the code in intr_thread() updates intrstat[pil][0], it then sets * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1], * the timings are reset, i.e. intr_get_time() will return [0] - [1] which * is 0. * * Whenever interrupts arrive on a CPU which is handling a lower pil * interrupt, they update the lower pil's [0] to show time spent in the * handler that they've interrupted. This results in a growing discrepancy * between [0] and [1], which is returned the next time intr_get_time() is * called. Time spent in the higher-pil interrupt will not be returned in * the next intr_get_time() call from the original interrupt, because * the higher-pil interrupt's time is accumulated in intrstat[higherpil][]. */ /*ARGSUSED*/ uint64_t intr_get_time(void) { return 0; } #else /* lint */ ENTRY_NP(intr_get_time) #ifdef DEBUG ! ! Lots of asserts, but just check panic_quiesce first. ! Don't bother with lots of tests if we're just ignoring them. ! sethi %hi(panic_quiesce), %o0 ld [%o0 + %lo(panic_quiesce)], %o0 brnz,pn %o0, 2f nop ! ! ASSERT(%pil <= LOCK_LEVEL) ! rdpr %pil, %o1 cmp %o1, LOCK_LEVEL ble,pt %xcc, 0f sethi %hi(intr_get_time_high_pil), %o0 ! delay call panic or %o0, %lo(intr_get_time_high_pil), %o0 0: ! ! ASSERT((t_flags & T_INTR_THREAD) != 0 && t_pil > 0) ! lduh [THREAD_REG + T_FLAGS], %o2 andcc %o2, T_INTR_THREAD, %g0 bz,pn %xcc, 1f ldub [THREAD_REG + T_PIL], %o1 ! delay brnz,pt %o1, 0f 1: sethi %hi(intr_get_time_not_intr), %o0 call panic or %o0, %lo(intr_get_time_not_intr), %o0 0: ! ! ASSERT(t_intr_start != 0) ! ldx [THREAD_REG + T_INTR_START], %o1 brnz,pt %o1, 2f sethi %hi(intr_get_time_no_start_time), %o0 ! delay call panic or %o0, %lo(intr_get_time_no_start_time), %o0 2: #endif /* DEBUG */ ! ! %o0 = elapsed time and return value ! %o1 = pil ! %o2 = scratch ! %o3 = scratch ! %o4 = scratch ! %o5 = cpu ! wrpr %g0, PIL_MAX, %pil ! make this easy -- block normal intrs ldn [THREAD_REG + T_CPU], %o5 ldub [THREAD_REG + T_PIL], %o1 ldx [THREAD_REG + T_INTR_START], %o3 ! %o3 = t_intr_start ! ! Calculate elapsed time since t_intr_start. Update t_intr_start, ! get delta, and multiply by cpu_divisor if necessary. ! RD_TICK_NO_SUSPEND_CHECK(%o2, %o0) stx %o2, [THREAD_REG + T_INTR_START] sub %o2, %o3, %o0 lduh [%o5 + CPU_DIVISOR], %o4 cmp %o4, 1 bg,a,pn %xcc, 1f mulx %o0, %o4, %o0 ! multiply interval by clock divisor iff > 1 1: ! Update intracct[] lduh [%o5 + CPU_MSTATE], %o4 sllx %o4, 3, %o4 add %o4, CPU_INTRACCT, %o4 ldx [%o5 + %o4], %o2 add %o2, %o0, %o2 stx %o2, [%o5 + %o4] ! ! Increment cpu_m.intrstat[pil][0]. Calculate elapsed time since ! cpu_m.intrstat[pil][1], which is either when the interrupt was ! first entered, or the last time intr_get_time() was invoked. Then ! update cpu_m.intrstat[pil][1] to match [0]. ! sllx %o1, 4, %o3 add %o3, CPU_MCPU, %o3 add %o3, MCPU_INTRSTAT, %o3 add %o3, %o5, %o3 ! %o3 = cpu_m.intrstat[pil][0] ldx [%o3], %o2 add %o2, %o0, %o2 ! %o2 = new value for intrstat stx %o2, [%o3] ldx [%o3 + 8], %o4 ! %o4 = cpu_m.intrstat[pil][1] sub %o2, %o4, %o0 ! %o0 is elapsed time since %o4 stx %o2, [%o3 + 8] ! make [1] match [0], resetting time ld [%o5 + CPU_BASE_SPL], %o2 ! restore %pil to the greater cmp %o2, %o1 ! of either our pil %o1 or movl %xcc, %o1, %o2 ! cpu_base_spl. retl wrpr %g0, %o2, %pil SET_SIZE(intr_get_time) #ifdef DEBUG intr_get_time_high_pil: .asciz "intr_get_time(): %pil > LOCK_LEVEL" intr_get_time_not_intr: .asciz "intr_get_time(): not called from an interrupt thread" intr_get_time_no_start_time: .asciz "intr_get_time(): t_intr_start == 0" #endif /* DEBUG */ #endif /* lint */