xref: /titanic_50/usr/src/uts/i86pc/os/intr.c (revision d2d5cf7c5d909b74a88d499283e24750a9a52c5d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/cpuvar.h>
30 #include <sys/regset.h>
31 #include <sys/psw.h>
32 #include <sys/types.h>
33 #include <sys/thread.h>
34 #include <sys/systm.h>
35 #include <sys/segments.h>
36 #include <sys/pcb.h>
37 #include <sys/trap.h>
38 #include <sys/ftrace.h>
39 #include <sys/traptrace.h>
40 #include <sys/clock.h>
41 #include <sys/panic.h>
42 #include <sys/disp.h>
43 #include <vm/seg_kp.h>
44 #include <sys/stack.h>
45 #include <sys/sysmacros.h>
46 #include <sys/cmn_err.h>
47 #include <sys/kstat.h>
48 #include <sys/smp_impldefs.h>
49 #include <sys/pool_pset.h>
50 #include <sys/zone.h>
51 #include <sys/bitmap.h>
52 #include <sys/archsystm.h>
53 #include <sys/machsystm.h>
54 #include <sys/ontrap.h>
55 #include <sys/x86_archext.h>
56 #include <sys/promif.h>
57 #include <vm/hat_i86.h>
58 #if defined(__xpv)
59 #include <sys/hypervisor.h>
60 #endif
61 
62 
63 #if defined(__xpv) && defined(DEBUG)
64 
65 /*
66  * This panic message is intended as an aid to interrupt debugging.
67  *
68  * The associated assertion tests the condition of enabling
69  * events when events are already enabled.  The implication
70  * being that whatever code the programmer thought was
71  * protected by having events disabled until the second
72  * enable happened really wasn't protected at all ..
73  */
74 
75 int stistipanic = 1;	/* controls the debug panic check */
76 const char *stistimsg = "stisti";
77 ulong_t laststi[NCPU];
78 
79 /*
80  * This variable tracks the last place events were disabled on each cpu
81  * it assists in debugging when asserts that interupts are enabled trip.
82  */
83 ulong_t lastcli[NCPU];
84 
85 #endif
86 
87 /*
88  * Set cpu's base SPL level to the highest active interrupt level
89  */
90 void
91 set_base_spl(void)
92 {
93 	struct cpu *cpu = CPU;
94 	uint16_t active = (uint16_t)cpu->cpu_intr_actv;
95 
96 	cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);
97 }
98 
99 /*
100  * Do all the work necessary to set up the cpu and thread structures
101  * to dispatch a high-level interrupt.
102  *
103  * Returns 0 if we're -not- already on the high-level interrupt stack,
104  * (and *must* switch to it), non-zero if we are already on that stack.
105  *
106  * Called with interrupts masked.
107  * The 'pil' is already set to the appropriate level for rp->r_trapno.
108  */
109 static int
110 hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
111 {
112 	struct machcpu *mcpu = &cpu->cpu_m;
113 	uint_t mask;
114 	hrtime_t intrtime;
115 	hrtime_t now = tsc_read();
116 
117 	ASSERT(pil > LOCK_LEVEL);
118 
119 	if (pil == CBE_HIGH_PIL) {
120 		cpu->cpu_profile_pil = oldpil;
121 		if (USERMODE(rp->r_cs)) {
122 			cpu->cpu_profile_pc = 0;
123 			cpu->cpu_profile_upc = rp->r_pc;
124 		} else {
125 			cpu->cpu_profile_pc = rp->r_pc;
126 			cpu->cpu_profile_upc = 0;
127 		}
128 	}
129 
130 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
131 	if (mask != 0) {
132 		int nestpil;
133 
134 		/*
135 		 * We have interrupted another high-level interrupt.
136 		 * Load starting timestamp, compute interval, update
137 		 * cumulative counter.
138 		 */
139 		nestpil = bsrw_insn((uint16_t)mask);
140 		ASSERT(nestpil < pil);
141 		intrtime = now -
142 		    mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
143 		mcpu->intrstat[nestpil][0] += intrtime;
144 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
145 		/*
146 		 * Another high-level interrupt is active below this one, so
147 		 * there is no need to check for an interrupt thread.  That
148 		 * will be done by the lowest priority high-level interrupt
149 		 * active.
150 		 */
151 	} else {
152 		kthread_t *t = cpu->cpu_thread;
153 
154 		/*
155 		 * See if we are interrupting a low-level interrupt thread.
156 		 * If so, account for its time slice only if its time stamp
157 		 * is non-zero.
158 		 */
159 		if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
160 			intrtime = now - t->t_intr_start;
161 			mcpu->intrstat[t->t_pil][0] += intrtime;
162 			cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
163 			t->t_intr_start = 0;
164 		}
165 	}
166 
167 	/*
168 	 * Store starting timestamp in CPU structure for this PIL.
169 	 */
170 	mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
171 
172 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
173 
174 	if (pil == 15) {
175 		/*
176 		 * To support reentrant level 15 interrupts, we maintain a
177 		 * recursion count in the top half of cpu_intr_actv.  Only
178 		 * when this count hits zero do we clear the PIL 15 bit from
179 		 * the lower half of cpu_intr_actv.
180 		 */
181 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
182 		(*refcntp)++;
183 	}
184 
185 	mask = cpu->cpu_intr_actv;
186 
187 	cpu->cpu_intr_actv |= (1 << pil);
188 
189 	return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
190 }
191 
192 /*
193  * Does most of the work of returning from a high level interrupt.
194  *
195  * Returns 0 if there are no more high level interrupts (in which
196  * case we must switch back to the interrupted thread stack) or
197  * non-zero if there are more (in which case we should stay on it).
198  *
199  * Called with interrupts masked
200  */
201 static int
202 hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
203 {
204 	struct machcpu *mcpu = &cpu->cpu_m;
205 	uint_t mask;
206 	hrtime_t intrtime;
207 	hrtime_t now = tsc_read();
208 
209 	ASSERT(mcpu->mcpu_pri == pil);
210 
211 	cpu->cpu_stats.sys.intr[pil - 1]++;
212 
213 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
214 
215 	if (pil == 15) {
216 		/*
217 		 * To support reentrant level 15 interrupts, we maintain a
218 		 * recursion count in the top half of cpu_intr_actv.  Only
219 		 * when this count hits zero do we clear the PIL 15 bit from
220 		 * the lower half of cpu_intr_actv.
221 		 */
222 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
223 
224 		ASSERT(*refcntp > 0);
225 
226 		if (--(*refcntp) == 0)
227 			cpu->cpu_intr_actv &= ~(1 << pil);
228 	} else {
229 		cpu->cpu_intr_actv &= ~(1 << pil);
230 	}
231 
232 	ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
233 
234 	intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
235 	mcpu->intrstat[pil][0] += intrtime;
236 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
237 
238 	/*
239 	 * Check for lower-pil nested high-level interrupt beneath
240 	 * current one.  If so, place a starting timestamp in its
241 	 * pil_high_start entry.
242 	 */
243 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
244 	if (mask != 0) {
245 		int nestpil;
246 
247 		/*
248 		 * find PIL of nested interrupt
249 		 */
250 		nestpil = bsrw_insn((uint16_t)mask);
251 		ASSERT(nestpil < pil);
252 		mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
253 		/*
254 		 * (Another high-level interrupt is active below this one,
255 		 * so there is no need to check for an interrupt
256 		 * thread.  That will be done by the lowest priority
257 		 * high-level interrupt active.)
258 		 */
259 	} else {
260 		/*
261 		 * Check to see if there is a low-level interrupt active.
262 		 * If so, place a starting timestamp in the thread
263 		 * structure.
264 		 */
265 		kthread_t *t = cpu->cpu_thread;
266 
267 		if (t->t_flag & T_INTR_THREAD)
268 			t->t_intr_start = now;
269 	}
270 
271 	mcpu->mcpu_pri = oldpil;
272 	(void) (*setlvlx)(oldpil, vecnum);
273 
274 	return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
275 }
276 
277 /*
278  * Set up the cpu, thread and interrupt thread structures for
279  * executing an interrupt thread.  The new stack pointer of the
280  * interrupt thread (which *must* be switched to) is returned.
281  */
282 static caddr_t
283 intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
284 {
285 	struct machcpu *mcpu = &cpu->cpu_m;
286 	kthread_t *t, *volatile it;
287 	hrtime_t now = tsc_read();
288 
289 	ASSERT(pil > 0);
290 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
291 	cpu->cpu_intr_actv |= (1 << pil);
292 
293 	/*
294 	 * Get set to run an interrupt thread.
295 	 * There should always be an interrupt thread, since we
296 	 * allocate one for each level on each CPU.
297 	 *
298 	 * t_intr_start could be zero due to cpu_intr_swtch_enter.
299 	 */
300 	t = cpu->cpu_thread;
301 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
302 		hrtime_t intrtime = now - t->t_intr_start;
303 		mcpu->intrstat[t->t_pil][0] += intrtime;
304 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
305 		t->t_intr_start = 0;
306 	}
307 
308 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
309 
310 	t->t_sp = (uintptr_t)stackptr;	/* mark stack in curthread for resume */
311 
312 	/*
313 	 * unlink the interrupt thread off the cpu
314 	 *
315 	 * Note that the code in kcpc_overflow_intr -relies- on the
316 	 * ordering of events here - in particular that t->t_lwp of
317 	 * the interrupt thread is set to the pinned thread *before*
318 	 * curthread is changed.
319 	 */
320 	it = cpu->cpu_intr_thread;
321 	cpu->cpu_intr_thread = it->t_link;
322 	it->t_intr = t;
323 	it->t_lwp = t->t_lwp;
324 
325 	/*
326 	 * (threads on the interrupt thread free list could have state
327 	 * preset to TS_ONPROC, but it helps in debugging if
328 	 * they're TS_FREE.)
329 	 */
330 	it->t_state = TS_ONPROC;
331 
332 	cpu->cpu_thread = it;		/* new curthread on this cpu */
333 	it->t_pil = (uchar_t)pil;
334 	it->t_pri = intr_pri + (pri_t)pil;
335 	it->t_intr_start = now;
336 
337 	return (it->t_stk);
338 }
339 
340 
341 #ifdef DEBUG
342 int intr_thread_cnt;
343 #endif
344 
345 /*
346  * Called with interrupts disabled
347  */
348 static void
349 intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
350 {
351 	struct machcpu *mcpu = &cpu->cpu_m;
352 	kthread_t *t;
353 	kthread_t *it = cpu->cpu_thread;	/* curthread */
354 	uint_t pil, basespl;
355 	hrtime_t intrtime;
356 	hrtime_t now = tsc_read();
357 
358 	pil = it->t_pil;
359 	cpu->cpu_stats.sys.intr[pil - 1]++;
360 
361 	ASSERT(it->t_intr_start != 0);
362 	intrtime = now - it->t_intr_start;
363 	mcpu->intrstat[pil][0] += intrtime;
364 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
365 
366 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
367 	cpu->cpu_intr_actv &= ~(1 << pil);
368 
369 	/*
370 	 * If there is still an interrupted thread underneath this one
371 	 * then the interrupt was never blocked and the return is
372 	 * fairly simple.  Otherwise it isn't.
373 	 */
374 	if ((t = it->t_intr) == NULL) {
375 		/*
376 		 * The interrupted thread is no longer pinned underneath
377 		 * the interrupt thread.  This means the interrupt must
378 		 * have blocked, and the interrupted thread has been
379 		 * unpinned, and has probably been running around the
380 		 * system for a while.
381 		 *
382 		 * Since there is no longer a thread under this one, put
383 		 * this interrupt thread back on the CPU's free list and
384 		 * resume the idle thread which will dispatch the next
385 		 * thread to run.
386 		 */
387 #ifdef DEBUG
388 		intr_thread_cnt++;
389 #endif
390 		cpu->cpu_stats.sys.intrblk++;
391 		/*
392 		 * Set CPU's base SPL based on active interrupts bitmask
393 		 */
394 		set_base_spl();
395 		basespl = cpu->cpu_base_spl;
396 		mcpu->mcpu_pri = basespl;
397 		(*setlvlx)(basespl, vec);
398 		(void) splhigh();
399 		sti();
400 		it->t_state = TS_FREE;
401 		/*
402 		 * Return interrupt thread to pool
403 		 */
404 		it->t_link = cpu->cpu_intr_thread;
405 		cpu->cpu_intr_thread = it;
406 		swtch();
407 		panic("intr_thread_epilog: swtch returned");
408 		/*NOTREACHED*/
409 	}
410 
411 	/*
412 	 * Return interrupt thread to the pool
413 	 */
414 	it->t_link = cpu->cpu_intr_thread;
415 	cpu->cpu_intr_thread = it;
416 	it->t_state = TS_FREE;
417 
418 	basespl = cpu->cpu_base_spl;
419 	pil = MAX(oldpil, basespl);
420 	mcpu->mcpu_pri = pil;
421 	(*setlvlx)(pil, vec);
422 	t->t_intr_start = now;
423 	cpu->cpu_thread = t;
424 }
425 
426 /*
427  * intr_get_time() is a resource for interrupt handlers to determine how
428  * much time has been spent handling the current interrupt. Such a function
429  * is needed because higher level interrupts can arrive during the
430  * processing of an interrupt.  intr_get_time() only returns time spent in the
431  * current interrupt handler.
432  *
433  * The caller must be calling from an interrupt handler running at a pil
434  * below or at lock level. Timings are not provided for high-level
435  * interrupts.
436  *
437  * The first time intr_get_time() is called while handling an interrupt,
438  * it returns the time since the interrupt handler was invoked. Subsequent
439  * calls will return the time since the prior call to intr_get_time(). Time
440  * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
441  *
442  * Theory Of Intrstat[][]:
443  *
444  * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
445  * uint64_ts per pil.
446  *
447  * intrstat[pil][0] is a cumulative count of the number of ticks spent
448  * handling all interrupts at the specified pil on this CPU. It is
449  * exported via kstats to the user.
450  *
451  * intrstat[pil][1] is always a count of ticks less than or equal to the
452  * value in [0]. The difference between [1] and [0] is the value returned
453  * by a call to intr_get_time(). At the start of interrupt processing,
454  * [0] and [1] will be equal (or nearly so). As the interrupt consumes
455  * time, [0] will increase, but [1] will remain the same. A call to
456  * intr_get_time() will return the difference, then update [1] to be the
457  * same as [0]. Future calls will return the time since the last call.
458  * Finally, when the interrupt completes, [1] is updated to the same as [0].
459  *
460  * Implementation:
461  *
462  * intr_get_time() works much like a higher level interrupt arriving. It
463  * "checkpoints" the timing information by incrementing intrstat[pil][0]
464  * to include elapsed running time, and by setting t_intr_start to rdtsc.
465  * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
466  * and updates intrstat[pil][1] to be the same as the new value of
467  * intrstat[pil][0].
468  *
469  * In the normal handling of interrupts, after an interrupt handler returns
470  * and the code in intr_thread() updates intrstat[pil][0], it then sets
471  * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
472  * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
473  * is 0.
474  *
475  * Whenever interrupts arrive on a CPU which is handling a lower pil
476  * interrupt, they update the lower pil's [0] to show time spent in the
477  * handler that they've interrupted. This results in a growing discrepancy
478  * between [0] and [1], which is returned the next time intr_get_time() is
479  * called. Time spent in the higher-pil interrupt will not be returned in
480  * the next intr_get_time() call from the original interrupt, because
481  * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
482  */
483 uint64_t
484 intr_get_time(void)
485 {
486 	struct cpu *cpu;
487 	struct machcpu *mcpu;
488 	kthread_t *t;
489 	uint64_t time, delta, ret;
490 	uint_t pil;
491 
492 	cli();
493 	cpu = CPU;
494 	mcpu = &cpu->cpu_m;
495 	t = cpu->cpu_thread;
496 	pil = t->t_pil;
497 	ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
498 	ASSERT(t->t_flag & T_INTR_THREAD);
499 	ASSERT(pil != 0);
500 	ASSERT(t->t_intr_start != 0);
501 
502 	time = tsc_read();
503 	delta = time - t->t_intr_start;
504 	t->t_intr_start = time;
505 
506 	time = mcpu->intrstat[pil][0] + delta;
507 	ret = time - mcpu->intrstat[pil][1];
508 	mcpu->intrstat[pil][0] = time;
509 	mcpu->intrstat[pil][1] = time;
510 	cpu->cpu_intracct[cpu->cpu_mstate] += delta;
511 
512 	sti();
513 	return (ret);
514 }
515 
516 static caddr_t
517 dosoftint_prolog(
518 	struct cpu *cpu,
519 	caddr_t stackptr,
520 	uint32_t st_pending,
521 	uint_t oldpil)
522 {
523 	kthread_t *t, *volatile it;
524 	struct machcpu *mcpu = &cpu->cpu_m;
525 	uint_t pil;
526 	hrtime_t now;
527 
528 top:
529 	ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
530 
531 	pil = bsrw_insn((uint16_t)st_pending);
532 	if (pil <= oldpil || pil <= cpu->cpu_base_spl)
533 		return (0);
534 
535 	/*
536 	 * XX64	Sigh.
537 	 *
538 	 * This is a transliteration of the i386 assembler code for
539 	 * soft interrupts.  One question is "why does this need
540 	 * to be atomic?"  One possible race is -other- processors
541 	 * posting soft interrupts to us in set_pending() i.e. the
542 	 * CPU might get preempted just after the address computation,
543 	 * but just before the atomic transaction, so another CPU would
544 	 * actually set the original CPU's st_pending bit.  However,
545 	 * it looks like it would be simpler to disable preemption there.
546 	 * Are there other races for which preemption control doesn't work?
547 	 *
548 	 * The i386 assembler version -also- checks to see if the bit
549 	 * being cleared was actually set; if it wasn't, it rechecks
550 	 * for more.  This seems a bit strange, as the only code that
551 	 * ever clears the bit is -this- code running with interrupts
552 	 * disabled on -this- CPU.  This code would probably be cheaper:
553 	 *
554 	 * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
555 	 *   ~(1 << pil));
556 	 *
557 	 * and t->t_preempt--/++ around set_pending() even cheaper,
558 	 * but at this point, correctness is critical, so we slavishly
559 	 * emulate the i386 port.
560 	 */
561 	if (atomic_btr32((uint32_t *)
562 	    &mcpu->mcpu_softinfo.st_pending, pil) == 0) {
563 		st_pending = mcpu->mcpu_softinfo.st_pending;
564 		goto top;
565 	}
566 
567 	mcpu->mcpu_pri = pil;
568 	(*setspl)(pil);
569 
570 	now = tsc_read();
571 
572 	/*
573 	 * Get set to run interrupt thread.
574 	 * There should always be an interrupt thread since we
575 	 * allocate one for each level on the CPU.
576 	 */
577 	it = cpu->cpu_intr_thread;
578 	cpu->cpu_intr_thread = it->t_link;
579 
580 	/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
581 	t = cpu->cpu_thread;
582 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
583 		hrtime_t intrtime = now - t->t_intr_start;
584 		mcpu->intrstat[pil][0] += intrtime;
585 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
586 		t->t_intr_start = 0;
587 	}
588 
589 	/*
590 	 * Note that the code in kcpc_overflow_intr -relies- on the
591 	 * ordering of events here - in particular that t->t_lwp of
592 	 * the interrupt thread is set to the pinned thread *before*
593 	 * curthread is changed.
594 	 */
595 	it->t_lwp = t->t_lwp;
596 	it->t_state = TS_ONPROC;
597 
598 	/*
599 	 * Push interrupted thread onto list from new thread.
600 	 * Set the new thread as the current one.
601 	 * Set interrupted thread's T_SP because if it is the idle thread,
602 	 * resume() may use that stack between threads.
603 	 */
604 
605 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
606 	t->t_sp = (uintptr_t)stackptr;
607 
608 	it->t_intr = t;
609 	cpu->cpu_thread = it;
610 
611 	/*
612 	 * Set bit for this pil in CPU's interrupt active bitmask.
613 	 */
614 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
615 	cpu->cpu_intr_actv |= (1 << pil);
616 
617 	/*
618 	 * Initialize thread priority level from intr_pri
619 	 */
620 	it->t_pil = (uchar_t)pil;
621 	it->t_pri = (pri_t)pil + intr_pri;
622 	it->t_intr_start = now;
623 
624 	return (it->t_stk);
625 }
626 
627 static void
628 dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
629 {
630 	struct machcpu *mcpu = &cpu->cpu_m;
631 	kthread_t *t, *it;
632 	uint_t pil, basespl;
633 	hrtime_t intrtime;
634 	hrtime_t now = tsc_read();
635 
636 	it = cpu->cpu_thread;
637 	pil = it->t_pil;
638 
639 	cpu->cpu_stats.sys.intr[pil - 1]++;
640 
641 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
642 	cpu->cpu_intr_actv &= ~(1 << pil);
643 	intrtime = now - it->t_intr_start;
644 	mcpu->intrstat[pil][0] += intrtime;
645 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
646 
647 	/*
648 	 * If there is still an interrupted thread underneath this one
649 	 * then the interrupt was never blocked and the return is
650 	 * fairly simple.  Otherwise it isn't.
651 	 */
652 	if ((t = it->t_intr) == NULL) {
653 		/*
654 		 * Put thread back on the interrupt thread list.
655 		 * This was an interrupt thread, so set CPU's base SPL.
656 		 */
657 		set_base_spl();
658 		it->t_state = TS_FREE;
659 		it->t_link = cpu->cpu_intr_thread;
660 		cpu->cpu_intr_thread = it;
661 		(void) splhigh();
662 		sti();
663 		swtch();
664 		/*NOTREACHED*/
665 		panic("dosoftint_epilog: swtch returned");
666 	}
667 	it->t_link = cpu->cpu_intr_thread;
668 	cpu->cpu_intr_thread = it;
669 	it->t_state = TS_FREE;
670 	cpu->cpu_thread = t;
671 	if (t->t_flag & T_INTR_THREAD)
672 		t->t_intr_start = now;
673 	basespl = cpu->cpu_base_spl;
674 	pil = MAX(oldpil, basespl);
675 	mcpu->mcpu_pri = pil;
676 	(*setspl)(pil);
677 }
678 
679 
680 /*
681  * Make the interrupted thread 'to' be runnable.
682  *
683  * Since t->t_sp has already been saved, t->t_pc is all
684  * that needs to be set in this function.
685  *
686  * Returns the interrupt level of the interrupt thread.
687  */
688 int
689 intr_passivate(
690 	kthread_t *it,		/* interrupt thread */
691 	kthread_t *t)		/* interrupted thread */
692 {
693 	extern void _sys_rtt();
694 
695 	ASSERT(it->t_flag & T_INTR_THREAD);
696 	ASSERT(SA(t->t_sp) == t->t_sp);
697 
698 	t->t_pc = (uintptr_t)_sys_rtt;
699 	return (it->t_pil);
700 }
701 
702 /*
703  * Create interrupt kstats for this CPU.
704  */
705 void
706 cpu_create_intrstat(cpu_t *cp)
707 {
708 	int		i;
709 	kstat_t		*intr_ksp;
710 	kstat_named_t	*knp;
711 	char		name[KSTAT_STRLEN];
712 	zoneid_t	zoneid;
713 
714 	ASSERT(MUTEX_HELD(&cpu_lock));
715 
716 	if (pool_pset_enabled())
717 		zoneid = GLOBAL_ZONEID;
718 	else
719 		zoneid = ALL_ZONES;
720 
721 	intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
722 	    KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid);
723 
724 	/*
725 	 * Initialize each PIL's named kstat
726 	 */
727 	if (intr_ksp != NULL) {
728 		intr_ksp->ks_update = cpu_kstat_intrstat_update;
729 		knp = (kstat_named_t *)intr_ksp->ks_data;
730 		intr_ksp->ks_private = cp;
731 		for (i = 0; i < PIL_MAX; i++) {
732 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
733 			    i + 1);
734 			kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
735 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
736 			    i + 1);
737 			kstat_named_init(&knp[(i * 2) + 1], name,
738 			    KSTAT_DATA_UINT64);
739 		}
740 		kstat_install(intr_ksp);
741 	}
742 }
743 
744 /*
745  * Delete interrupt kstats for this CPU.
746  */
747 void
748 cpu_delete_intrstat(cpu_t *cp)
749 {
750 	kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
751 }
752 
753 /*
754  * Convert interrupt statistics from CPU ticks to nanoseconds and
755  * update kstat.
756  */
757 int
758 cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
759 {
760 	kstat_named_t	*knp = ksp->ks_data;
761 	cpu_t		*cpup = (cpu_t *)ksp->ks_private;
762 	int		i;
763 	hrtime_t	hrt;
764 
765 	if (rw == KSTAT_WRITE)
766 		return (EACCES);
767 
768 	for (i = 0; i < PIL_MAX; i++) {
769 		hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
770 		scalehrtimef(&hrt);
771 		knp[i * 2].value.ui64 = (uint64_t)hrt;
772 		knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
773 	}
774 
775 	return (0);
776 }
777 
778 /*
779  * An interrupt thread is ending a time slice, so compute the interval it
780  * ran for and update the statistic for its PIL.
781  */
782 void
783 cpu_intr_swtch_enter(kthread_id_t t)
784 {
785 	uint64_t	interval;
786 	uint64_t	start;
787 	cpu_t		*cpu;
788 
789 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
790 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
791 
792 	/*
793 	 * We could be here with a zero timestamp. This could happen if:
794 	 * an interrupt thread which no longer has a pinned thread underneath
795 	 * it (i.e. it blocked at some point in its past) has finished running
796 	 * its handler. intr_thread() updated the interrupt statistic for its
797 	 * PIL and zeroed its timestamp. Since there was no pinned thread to
798 	 * return to, swtch() gets called and we end up here.
799 	 *
800 	 * Note that we use atomic ops below (cas64 and atomic_add_64), which
801 	 * we don't use in the functions above, because we're not called
802 	 * with interrupts blocked, but the epilog/prolog functions are.
803 	 */
804 	if (t->t_intr_start) {
805 		do {
806 			start = t->t_intr_start;
807 			interval = tsc_read() - start;
808 		} while (cas64(&t->t_intr_start, start, 0) != start);
809 		cpu = CPU;
810 		cpu->cpu_m.intrstat[t->t_pil][0] += interval;
811 
812 		atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
813 		    interval);
814 	} else
815 		ASSERT(t->t_intr == NULL);
816 }
817 
818 /*
819  * An interrupt thread is returning from swtch(). Place a starting timestamp
820  * in its thread structure.
821  */
822 void
823 cpu_intr_swtch_exit(kthread_id_t t)
824 {
825 	uint64_t ts;
826 
827 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
828 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
829 
830 	do {
831 		ts = t->t_intr_start;
832 	} while (cas64(&t->t_intr_start, ts, tsc_read()) != ts);
833 }
834 
835 /*
836  * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
837  */
838 /*ARGSUSED*/
839 static void
840 dispatch_hilevel(uint_t vector, uint_t arg2)
841 {
842 	sti();
843 	av_dispatch_autovect(vector);
844 	cli();
845 }
846 
847 /*
848  * Dispatch a soft interrupt
849  */
850 /*ARGSUSED*/
851 static void
852 dispatch_softint(uint_t oldpil, uint_t arg2)
853 {
854 	struct cpu *cpu = CPU;
855 
856 	sti();
857 	av_dispatch_softvect((int)cpu->cpu_thread->t_pil);
858 	cli();
859 
860 	/*
861 	 * Must run softint_epilog() on the interrupt thread stack, since
862 	 * there may not be a return from it if the interrupt thread blocked.
863 	 */
864 	dosoftint_epilog(cpu, oldpil);
865 }
866 
867 /*
868  * Dispatch a normal interrupt
869  */
870 static void
871 dispatch_hardint(uint_t vector, uint_t oldipl)
872 {
873 	struct cpu *cpu = CPU;
874 
875 	sti();
876 	av_dispatch_autovect(vector);
877 	cli();
878 
879 	/*
880 	 * Must run intr_thread_epilog() on the interrupt thread stack, since
881 	 * there may not be a return from it if the interrupt thread blocked.
882 	 */
883 	intr_thread_epilog(cpu, vector, oldipl);
884 }
885 
886 /*
887  * Deliver any softints the current interrupt priority allows.
888  * Called with interrupts disabled.
889  */
890 void
891 dosoftint(struct regs *regs)
892 {
893 	struct cpu *cpu = CPU;
894 	int oldipl;
895 	caddr_t newsp;
896 
897 	while (cpu->cpu_softinfo.st_pending) {
898 		oldipl = cpu->cpu_pri;
899 		newsp = dosoftint_prolog(cpu, (caddr_t)regs,
900 		    cpu->cpu_softinfo.st_pending, oldipl);
901 		/*
902 		 * If returned stack pointer is NULL, priority is too high
903 		 * to run any of the pending softints now.
904 		 * Break out and they will be run later.
905 		 */
906 		if (newsp == NULL)
907 			break;
908 		switch_sp_and_call(newsp, dispatch_softint, oldipl, 0);
909 	}
910 }
911 
912 /*
913  * Interrupt service routine, called with interrupts disabled.
914  */
915 /*ARGSUSED*/
916 void
917 do_interrupt(struct regs *rp, trap_trace_rec_t *ttp)
918 {
919 	struct cpu *cpu = CPU;
920 	int newipl, oldipl = cpu->cpu_pri;
921 	uint_t vector;
922 	caddr_t newsp;
923 
924 #ifdef TRAPTRACE
925 	ttp->ttr_marker = TT_INTERRUPT;
926 	ttp->ttr_ipl = 0xff;
927 	ttp->ttr_pri = oldipl;
928 	ttp->ttr_spl = cpu->cpu_base_spl;
929 	ttp->ttr_vector = 0xff;
930 #endif	/* TRAPTRACE */
931 
932 #if !defined(__xpv)
933 	/*
934 	 * Handle any pending TLB flushing
935 	 */
936 	tlb_service();
937 #endif
938 
939 	/*
940 	 * If it's a softint go do it now.
941 	 */
942 	if (rp->r_trapno == T_SOFTINT) {
943 		dosoftint(rp);
944 		ASSERT(!interrupts_enabled());
945 		return;
946 	}
947 
948 	/*
949 	 * Raise the interrupt priority.
950 	 */
951 	newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno);
952 #ifdef TRAPTRACE
953 	ttp->ttr_ipl = newipl;
954 #endif	/* TRAPTRACE */
955 
956 	/*
957 	 * Bail if it is a spurious interrupt
958 	 */
959 	if (newipl == -1)
960 		return;
961 	cpu->cpu_pri = newipl;
962 	vector = rp->r_trapno;
963 #ifdef TRAPTRACE
964 	ttp->ttr_vector = vector;
965 #endif	/* TRAPTRACE */
966 	if (newipl > LOCK_LEVEL) {
967 		/*
968 		 * High priority interrupts run on this cpu's interrupt stack.
969 		 */
970 		if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {
971 			newsp = cpu->cpu_intr_stack;
972 			switch_sp_and_call(newsp, dispatch_hilevel, vector, 0);
973 		} else { /* already on the interrupt stack */
974 			dispatch_hilevel(vector, 0);
975 		}
976 		(void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);
977 	} else {
978 		/*
979 		 * Run this interrupt in a separate thread.
980 		 */
981 		newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);
982 		switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl);
983 	}
984 
985 	/*
986 	 * Deliver any pending soft interrupts.
987 	 */
988 	if (cpu->cpu_softinfo.st_pending)
989 		dosoftint(rp);
990 }
991 
992 /*
993  * Common tasks always done by _sys_rtt, called with interrupts disabled.
994  * Returns 1 if returning to userland, 0 if returning to system mode.
995  */
996 int
997 sys_rtt_common(struct regs *rp)
998 {
999 	kthread_t *tp;
1000 	extern void mutex_exit_critical_start();
1001 	extern long mutex_exit_critical_size;
1002 	extern void mutex_owner_running_critical_start();
1003 	extern long mutex_owner_running_critical_size;
1004 
1005 loop:
1006 
1007 	/*
1008 	 * Check if returning to user
1009 	 */
1010 	tp = CPU->cpu_thread;
1011 	if (USERMODE(rp->r_cs)) {
1012 		/*
1013 		 * Check if AST pending.
1014 		 */
1015 		if (tp->t_astflag) {
1016 			/*
1017 			 * Let trap() handle the AST
1018 			 */
1019 			sti();
1020 			rp->r_trapno = T_AST;
1021 			trap(rp, (caddr_t)0, CPU->cpu_id);
1022 			cli();
1023 			goto loop;
1024 		}
1025 
1026 #if defined(__amd64)
1027 		/*
1028 		 * We are done if segment registers do not need updating.
1029 		 */
1030 		if (tp->t_lwp->lwp_pcb.pcb_rupdate == 0)
1031 			return (1);
1032 
1033 		if (update_sregs(rp, tp->t_lwp)) {
1034 			/*
1035 			 * 1 or more of the selectors is bad.
1036 			 * Deliver a SIGSEGV.
1037 			 */
1038 			proc_t *p = ttoproc(tp);
1039 
1040 			sti();
1041 			mutex_enter(&p->p_lock);
1042 			tp->t_lwp->lwp_cursig = SIGSEGV;
1043 			mutex_exit(&p->p_lock);
1044 			psig();
1045 			tp->t_sig_check = 1;
1046 			cli();
1047 		}
1048 		tp->t_lwp->lwp_pcb.pcb_rupdate = 0;
1049 
1050 #endif	/* __amd64 */
1051 		return (1);
1052 	}
1053 
1054 	/*
1055 	 * Here if we are returning to supervisor mode.
1056 	 * Check for a kernel preemption request.
1057 	 */
1058 	if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
1059 
1060 		/*
1061 		 * Do nothing if already in kpreempt
1062 		 */
1063 		if (!tp->t_preempt_lk) {
1064 			tp->t_preempt_lk = 1;
1065 			sti();
1066 			kpreempt(1); /* asynchronous kpreempt call */
1067 			cli();
1068 			tp->t_preempt_lk = 0;
1069 		}
1070 	}
1071 
1072 	/*
1073 	 * If we interrupted the mutex_exit() critical region we must
1074 	 * reset the PC back to the beginning to prevent missed wakeups
1075 	 * See the comments in mutex_exit() for details.
1076 	 */
1077 	if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
1078 	    mutex_exit_critical_size) {
1079 		rp->r_pc = (greg_t)mutex_exit_critical_start;
1080 	}
1081 
1082 	/*
1083 	 * If we interrupted the mutex_owner_running() critical region we
1084 	 * must reset the PC back to the beginning to prevent dereferencing
1085 	 * of a freed thread pointer. See the comments in mutex_owner_running
1086 	 * for details.
1087 	 */
1088 	if ((uintptr_t)rp->r_pc -
1089 	    (uintptr_t)mutex_owner_running_critical_start <
1090 	    mutex_owner_running_critical_size) {
1091 		rp->r_pc = (greg_t)mutex_owner_running_critical_start;
1092 	}
1093 
1094 	return (0);
1095 }
1096 
1097 void
1098 send_dirint(int cpuid, int int_level)
1099 {
1100 	(*send_dirintf)(cpuid, int_level);
1101 }
1102 
1103 /*
1104  * do_splx routine, takes new ipl to set
1105  * returns the old ipl.
1106  * We are careful not to set priority lower than CPU->cpu_base_pri,
1107  * even though it seems we're raising the priority, it could be set
1108  * higher at any time by an interrupt routine, so we must block interrupts
1109  * and look at CPU->cpu_base_pri
1110  */
1111 int
1112 do_splx(int newpri)
1113 {
1114 	ulong_t	flag;
1115 	cpu_t	*cpu;
1116 	int	curpri, basepri;
1117 
1118 	flag = intr_clear();
1119 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1120 	curpri = cpu->cpu_m.mcpu_pri;
1121 	basepri = cpu->cpu_base_spl;
1122 	if (newpri < basepri)
1123 		newpri = basepri;
1124 	cpu->cpu_m.mcpu_pri = newpri;
1125 	(*setspl)(newpri);
1126 	/*
1127 	 * If we are going to reenable interrupts see if new priority level
1128 	 * allows pending softint delivery.
1129 	 */
1130 	if ((flag & PS_IE) &&
1131 	    bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > newpri)
1132 		fakesoftint();
1133 	ASSERT(!interrupts_enabled());
1134 	intr_restore(flag);
1135 	return (curpri);
1136 }
1137 
1138 /*
1139  * Common spl raise routine, takes new ipl to set
1140  * returns the old ipl, will not lower ipl.
1141  */
1142 int
1143 splr(int newpri)
1144 {
1145 	ulong_t	flag;
1146 	cpu_t	*cpu;
1147 	int	curpri, basepri;
1148 
1149 	flag = intr_clear();
1150 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1151 	curpri = cpu->cpu_m.mcpu_pri;
1152 	/*
1153 	 * Only do something if new priority is larger
1154 	 */
1155 	if (newpri > curpri) {
1156 		basepri = cpu->cpu_base_spl;
1157 		if (newpri < basepri)
1158 			newpri = basepri;
1159 		cpu->cpu_m.mcpu_pri = newpri;
1160 		(*setspl)(newpri);
1161 		/*
1162 		 * See if new priority level allows pending softint delivery
1163 		 */
1164 		if ((flag & PS_IE) &&
1165 		    bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > newpri)
1166 			fakesoftint();
1167 	}
1168 	intr_restore(flag);
1169 	return (curpri);
1170 }
1171 
1172 int
1173 getpil(void)
1174 {
1175 	return (CPU->cpu_m.mcpu_pri);
1176 }
1177 
1178 int
1179 interrupts_enabled(void)
1180 {
1181 	ulong_t	flag;
1182 
1183 	flag = getflags();
1184 	return ((flag & PS_IE) == PS_IE);
1185 }
1186 
1187 #ifdef DEBUG
1188 void
1189 assert_ints_enabled(void)
1190 {
1191 	ASSERT(!interrupts_unleashed || interrupts_enabled());
1192 }
1193 #endif	/* DEBUG */
1194