xref: /illumos-gate/usr/src/uts/i86pc/os/intr.c (revision 2eeda98682960c32b47af773fc6db343bb6b588f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2018 Joyent, Inc.  All rights reserverd.
25  */
26 
27 /*
28  * To understand the present state of interrupt handling on i86pc, we must
29  * first consider the history of interrupt controllers and our way of handling
30  * interrupts.
31  *
32  * History of Interrupt Controllers on i86pc
33  * -----------------------------------------
34  *
35  *    Intel 8259 and 8259A
36  *
37  * The first interrupt controller that attained widespread use on i86pc was
38  * the Intel 8259(A) Programmable Interrupt Controller that first saw use with
39  * the 8086. It took up to 8 interrupt sources and combined them into one
40  * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs.
41  * With the switch to the 8259A, level mode interrupts became possible. For a
42  * long time on i86pc the 8259A was the only way to handle interrupts and it
43  * had its own set of quirks. The 8259A and its corresponding interval timer
44  * the 8254 are programmed using outb and inb instructions.
45  *
46  *    Intel Advanced Programmable Interrupt Controller (APIC)
47  *
48  * Starting around the time of the introduction of the P6 family
49  * microarchitecture (i686) Intel introduced a new interrupt controller.
50  * Instead of having the series of slaved 8259A devices, Intel opted to outfit
51  * each processor with a Local APIC (lapic) and to outfit the system with at
52  * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics
53  * initially communicated over a dedicated bus, but this has since been
54  * replaced. Each physical core and even hyperthread currently contains its
55  * own local apic, which is not shared. There are a few exceptions for
56  * hyperthreads, but that does not usually concern us.
57  *
58  * Instead of talking directly to 8259 for status, sending End Of Interrupt
59  * (EOI), etc. a microprocessor now communicates directly to the lapic. This
60  * also allows for each microprocessor to be able to have independent controls.
61  * The programming method is different from the 8259. Consumers map the lapic
62  * registers into uncacheable memory to read and manipulate the state.
63  *
64  * The number of addressable interrupt vectors was increased to 256. However
65  * vectors 0-31 are reserved for the processor exception handling, leaving the
66  * remaining vectors for general use. In addition to hardware generated
67  * interrupts, the lapic provides a way for generating inter-processor
68  * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
69  *
70  * AMD ended up implementing the Intel APIC architecture in lieu of their work
71  * with Cyrix.
72  *
73  *    Intel x2apic
74  *
75  * The x2apic is an extension to the lapic which started showing up around the
76  * same time as the Sandy Bridge chipsets. It provides a new programming mode
77  * as well as new features. The goal of the x2apic is to solve a few problems
78  * with the previous generation of lapic and the x2apic is backwards compatible
79  * with the previous programming and model. The only downsides to using the
80  * backwards compatibility is that you are not able to take advantage of the new
81  * x2apic features.
82  *
83  *    o The APIC ID is increased from an 8-bit value to a 32-bit value. This
84  *    increases the maximum number of addressable physical processors beyond
85  *    256. This new ID is assembled in a similar manner as the information that
86  *    is obtainable by the extended cpuid topology leaves.
87  *
88  *    o A new means of generating IPIs was introduced.
89  *
90  *    o Instead of memory mapping the registers, the x2apic only allows for
91  *    programming it through a series of wrmsrs. This has important semantic
92  *    side effects. Recall that the registers were previously all mapped to
93  *    uncachable memory which meant that all operations to the local apic were
94  *    serializing instructions. With the switch to using wrmsrs this has been
95  *    relaxed and these operations can no longer be assumed to be serializing
96  *    instructions.
97  *
98  * Note for the rest of this we are only going to concern ourselves with the
99  * apic and x2apic which practically all of i86pc has been using now for
100  * quite some time.
101  *
102  * Interrupt Priority Levels
103  * -------------------------
104  *
105  * On i86pc systems there are a total of fifteen interrupt priority levels
106  * (ipls) which range from 1-15. Level 0 is for normal processing and
107  * non-interrupt processing. To manipulate these values the family of spl
108  * functions (which date back to UNIX on the PDP-11) are used. Specifically,
109  * splr() to raise the priority level and splx() to lower it. One should not
110  * generally call setspl() directly.
111  *
112  * Both i86pc and the supported SPARC platforms honor the same conventions for
113  * the meaning behind these IPLs. The most important IPL is the platform's
114  * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not
115  * sleep on any synchronization object. The only allowed synchronization
116  * primitive is a mutex that has been specifically initialized to be a spin
117  * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on
118  * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher.
119  * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls.
120  *
121  * Each interrupt that is registered in the system fires at a specific IPL.
122  * Generally most interrupts fire below LOCK_LEVEL.
123  *
124  * PSM Drivers
125  * -----------
126  *
127  * We currently have three sets of PSM (platform specific module) drivers
128  * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original
129  * driver that interacts with the 8259A and 8254. In general, it is not used
130  * anymore given the prevalence of the apic.
131  *
132  * The system prefers to use the apix driver over the pcplusmp driver. The apix
133  * driver requires HW support for an x2apic. If there is no x2apic HW, apix
134  * will not be used. In general we prefer using the apix driver over the
135  * pcplusmp driver because it gives us much more flexibility with respect to
136  * interrupts. In the apix driver each local apic has its own independent set
137  * of  interrupts, whereas the pcplusmp driver only has a single global set of
138  * interrupts. This is why pcplusmp only supports a finite number of interrupts
139  * per IPL -- generally 16, often less. The apix driver supports using either
140  * the x2apic or the local apic programing modes. The programming mode does not
141  * change the number of interrupts available, just the number of processors
142  * that we can address. For the apix driver, the x2apic mode is enabled if the
143  * system supports interrupt re-mapping, otherwise the module manages the
144  * x2apic in local mode.
145  *
146  * When there is no x2apic present, we default back to the pcplusmp PSM driver.
147  * In general, this is not problematic unless you have more than 256
148  * processors in the machine or you do not have enough interrupts available.
149  *
150  * Controlling Interrupt Generation on i86pc
151  * -----------------------------------------
152  *
153  * There are two different ways to manipulate which interrupts will be
154  * generated on i86pc. Each offers different degrees of control.
155  *
156  * The first is through the flags register (eflags and rflags on i386 and amd64
157  * respectively). The IF bit determines whether or not interrupts are enabled
158  * or disabled. This is manipulated in one of several ways. The most common way
159  * is through the cli and sti instructions. These clear the IF flag and set it,
160  * respectively, for the current processor. The other common way is through the
161  * use of the intr_clear and intr_restore functions.
162  *
163  * Assuming interrupts are not blocked by the IF flag, then the second form is
164  * through the Processor-Priority Register (PPR). The PPR is used to determine
165  * whether or not a pending interrupt should be delivered. If the ipl of the
166  * new interrupt is higher than the current value in the PPR, then the lapic
167  * will either deliver it immediately (if interrupts are not in progress) or it
168  * will deliver it once the current interrupt processing has issued an EOI. The
169  * highest unmasked interrupt will be the one delivered.
170  *
171  * The PPR register is based upon the max of the following two registers in the
172  * lapic, the TPR register (also known as CR8 on amd64) that can be used to
173  * mask interrupt levels, and the current vector. Because the pcplusmp module
174  * always sets TPR appropriately early in the do_interrupt path, we can usually
175  * just think that the PPR is the TPR. The pcplusmp module also issues an EOI
176  * once it has set the TPR, so higher priority interrupts can come in while
177  * we're servicing a lower priority interrupt.
178  *
179  * Handling Interrupts
180  * -------------------
181  *
182  * Interrupts can be broken down into three categories based on priority and
183  * source:
184  *
185  *   o High level interrupts
186  *   o Low level hardware interrupts
187  *   o Low level software interrupts
188  *
189  *   High Level Interrupts
190  *
191  * High level interrupts encompasses both hardware-sourced and software-sourced
192  * interrupts. Examples of high level hardware interrupts include the serial
193  * console. High level software-sourced interrupts are still delivered through
194  * the local apic through IPIs. This is primarily cross calls.
195  *
196  * When a high level interrupt comes in, we will raise the SPL and then pin the
197  * current lwp to the processor. We will use its lwp, but our own interrupt
198  * stack and process the high level interrupt in-situ. These handlers are
199  * designed to be very short in nature and cannot go to sleep, only block on a
200  * spin lock. If the interrupt has a lot of work to do, it must generate a
201  * low-priority software interrupt that will be processed later.
202  *
203  *   Low level hardware interrupts
204  *
205  * Low level hardware interrupts start off like their high-level cousins. The
206  * current CPU contains a number of kernel threads (kthread_t) that can be used
207  * to process low level interrupts. These are shared between both low level
208  * hardware and software interrupts. Note that while we run with our
209  * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a
210  * synchronization object. If we hit one and need to sleep, then the scheduler
211  * will instead create the rest of what we need.
212  *
213  *   Low level software interrupts
214  *
215  * Low level software interrupts are handled in a similar way as hardware
216  * interrupts, but the notification vector is different. Each CPU has a bitmask
217  * of pending software interrupts. We can notify a CPU to process software
218  * interrupts through a specific trap vector as well as through several
219  * checks that are performed throughout the code. These checks will look at
220  * processing software interrupts as we lower our spl.
221  *
222  * We attempt to process the highest pending software interrupt that we can
223  * which is greater than our current IPL. If none currently exist, then we move
224  * on. We process a software interrupt in a similar fashion to a hardware
225  * interrupt.
226  *
227  * Traditional Interrupt Flow
228  * --------------------------
229  *
230  * The following diagram tracks the flow of the traditional uppc and pcplusmp
231  * interrupt handlers. The apix driver has its own version of do_interrupt().
232  * We come into the interrupt handler with all interrupts masked by the IF
233  * flag. This is because we set up the handler using an interrupt-gate, which
234  * is defined architecturally to have cleared the IF flag for us.
235  *
236  * +--------------+    +----------------+    +-----------+
237  * | _interrupt() |--->| do_interrupt() |--->| *setlvl() |
238  * +--------------+    +----------------+    +-----------+
239  *                       |      |     |
240  *                       |      |     |
241  *              low-level|      |     | softint
242  *                HW int |      |     +---------------------------------------+
243  * +--------------+      |      |                                             |
244  * | intr_thread_ |<-----+      | hi-level int                                |
245  * | prolog()     |             |    +----------+                             |
246  * +--------------+             +--->| hilevel_ |      Not on intr stack      |
247  *       |                           | intr_    |-----------------+           |
248  *       |                           | prolog() |                 |           |
249  * +------------+                    +----------+                 |           |
250  * | switch_sp_ |                        | On intr                v           |
251  * | and_call() |                        | Stack          +------------+      |
252  * +------------+                        |                | switch_sp_ |      |
253  *       |                               v                | and_call() |      |
254  *       v                             +-----------+      +------------+      |
255  * +-----------+                       | dispatch_ |             |            |
256  * | dispatch_ |   +-------------------| hilevel() |<------------+            |
257  * | hardint() |   |                   +-----------+                          |
258  * +-----------+   |                                                          |
259  *       |         v                                                          |
260  *       |     +-----+  +----------------------+  +-----+  hi-level           |
261  *       +---->| sti |->| av_dispatch_autovect |->| cli |---------+           |
262  *             +-----+  +----------------------+  +-----+         |           |
263  *                                |                |              |           |
264  *                                v                |              |           |
265  *                         +----------+            |              |           |
266  *                         | for each |            |              |           |
267  *                         | handler  |            |              |           |
268  *                         |  *intr() |            |              v           |
269  * +--------------+        +----------+            |      +----------------+  |
270  * | intr_thread_ |                      low-level |      | hilevel_intr_  |  |
271  * | epilog()     |<-------------------------------+      | epilog()       |  |
272  * +--------------+                                       +----------------+  |
273  *   |       |                                                   |            |
274  *   |       +----------------------v      v---------------------+            |
275  *   |                           +------------+                               |
276  *   |   +---------------------->| *setlvlx() |                               |
277  *   |   |                       +------------+                               |
278  *   |   |                              |                                     |
279  *   |   |                              v                                     |
280  *   |   |      +--------+     +------------------+      +-------------+      |
281  *   |   |      | return |<----| softint pending? |----->| dosoftint() |<-----+
282  *   |   |      +--------+  no +------------------+ yes  +-------------+
283  *   |   |           ^                                      |     |
284  *   |   |           |  softint pil too low                 |     |
285  *   |   |           +--------------------------------------+     |
286  *   |   |                                                        v
287  *   |   |    +-----------+      +------------+          +-----------+
288  *   |   |    | dispatch_ |<-----| switch_sp_ |<---------| *setspl() |
289  *   |   |    | softint() |      | and_call() |          +-----------+
290  *   |   |    +-----------+      +------------+
291  *   |   |        |
292  *   |   |        v
293  *   |   |      +-----+  +----------------------+  +-----+  +------------+
294  *   |   |      | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ |
295  *   |   |      +-----+  +----------------------+  +-----+  | epilog()   |
296  *   |   |                                                  +------------+
297  *   |   |                                                    |     |
298  *   |   +----------------------------------------------------+     |
299  *   v                                                              |
300  * +-----------+                                                    |
301  * | interrupt |                                                    |
302  * | thread    |<---------------------------------------------------+
303  * | blocked   |
304  * +-----------+
305  *      |
306  *      v
307  *  +----------------+  +------------+  +-----------+  +-------+  +---------+
308  *  | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() |
309  *  +----------------+  +------------+  +-----------+  +-------+  +---------+
310  *
311  *    Calls made on Interrupt Stacks and Epilogue routines
312  *
313  * We use the switch_sp_and_call() assembly routine to switch our sp to the
314  * interrupt stacks and then call the appropriate dispatch function. In the
315  * case of interrupts which may block, softints and hardints, we always ensure
316  * that we are still on the interrupt thread when we call the epilog routine.
317  * This is not just important, it's necessary. If the interrupt thread blocked,
318  * we won't return from our switch_sp_and_call() function and instead we'll go
319  * through and set ourselves up to swtch() directly.
320  *
321  * New Interrupt Flow
322  * ------------------
323  *
324  * The apix module has its own interrupt path. This is done for various
325  * reasons. The first is that rather than having global interrupt vectors, we
326  * now have per-cpu vectors.
327  *
328  * The other substantial change is that the apix design does not use the TPR to
329  * mask interrupts below the current level. In fact, except for one special
330  * case, it does not use the TPR at all. Instead, it only uses the IF flag
331  * (cli/sti) to either block all interrupts or allow any interrupts to come in.
332  * The design is such that when interrupts are allowed to come in, if we are
333  * currently servicing a higher priority interupt, the new interrupt is treated
334  * as pending and serviced later. Specifically, in the pcplusmp module's
335  * apic_intr_enter() the code masks interrupts at or below the current
336  * IPL using the TPR before sending EOI, whereas the apix module's
337  * apix_intr_enter() simply sends EOI.
338  *
339  * The one special case where the apix code uses the TPR is when it calls
340  * through the apic_reg_ops function pointer apic_write_task_reg in
341  * apix_init_intr() to initially mask all levels and then finally to enable all
342  * levels.
343  *
344  * Recall that we come into the interrupt handler with all interrupts masked
345  * by the IF flag. This is because we set up the handler using an
346  * interrupt-gate which is defined architecturally to have cleared the IF flag
347  * for us.
348  *
349  * +--------------+    +---------------------+
350  * | _interrupt() |--->| apix_do_interrupt() |
351  * +--------------+    +---------------------+
352  *                               |
353  *                hard int? +----+--------+ softint?
354  *                          |             | (but no low-level looping)
355  *                   +-----------+        |
356  *                   | *setlvl() |        |
357  * +---------+       +-----------+        +----------------------------------+
358  * |apix_add_|    check IPL |                                                |
359  * |pending_ |<-------------+------+----------------------+                  |
360  * |hardint()|        low-level int|          hi-level int|                  |
361  * +---------+                     v                      v                  |
362  *     | check IPL       +-----------------+     +---------------+           |
363  *  +--+-----+           | apix_intr_      |     | apix_hilevel_ |           |
364  *  |        |           | thread_prolog() |     | intr_prolog() |           |
365  *  |      return        +-----------------+     +---------------+           |
366  *  |                         |                    | On intr                 |
367  *  |                   +------------+             | stack?  +------------+  |
368  *  |                   | switch_sp_ |             +---------| switch_sp_ |  |
369  *  |                   | and_call() |             |         | and_call() |  |
370  *  |                   +------------+             |         +------------+  |
371  *  |                         |                    |          |              |
372  *  |                   +----------------+     +----------------+            |
373  *  |                   | apix_dispatch_ |     | apix_dispatch_ |            |
374  *  |                   | lowlevel()     |     | hilevel()      |            |
375  *  |                   +----------------+     +----------------+            |
376  *  |                                |             |                         |
377  *  |                                v             v                         |
378  *  |                       +-------------------------+                      |
379  *  |                       |apix_dispatch_by_vector()|----+                 |
380  *  |                       +-------------------------+    |                 |
381  *  |               !XC_HI_PIL|         |         |        |                 |
382  *  |                       +---+   +-------+   +---+      |                 |
383  *  |                       |sti|   |*intr()|   |cli|      |                 |
384  *  |                       +---+   +-------+   +---+      |  hi-level?      |
385  *  |                          +---------------------------+----+            |
386  *  |                          v                low-level?      v            |
387  *  |                  +----------------+               +----------------+   |
388  *  |                  | apix_intr_     |               | apix_hilevel_  |   |
389  *  |                  | thread_epilog()|               | intr_epilog()  |   |
390  *  |                  +----------------+               +----------------+   |
391  *  |                          |                                |            |
392  *  |        v-----------------+--------------------------------+            |
393  *  |  +------------+                                                        |
394  *  |  | *setlvlx() |   +----------------------------------------------------+
395  *  |  +------------+   |
396  *  |      |            |            +--------------------------------+ low
397  *  v      v     v------+            v                                | level
398  * +------------------+      +------------------+      +-----------+  | pending?
399  * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_  |--+
400  * | hilevel()        |      | hardint()        |      | softint() |  |
401  * +------------------+      +------------------+      +-----------+    return
402  *     |                       |                         |
403  *     | while pending         | while pending           | while pending
404  *     | hi-level              | low-level               | softint
405  *     |                       |                         |
406  *  +---------------+        +-----------------+       +-----------------+
407  *  | apix_hilevel_ |        | apix_intr_      |       | apix_do_        |
408  *  | intr_prolog() |        | thread_prolog() |       | softint_prolog()|
409  *  +---------------+        +-----------------+       +-----------------+
410  *     | On intr                       |                      |
411  *     | stack? +------------+    +------------+        +------------+
412  *     +--------| switch_sp_ |    | switch_sp_ |        | switch_sp_ |
413  *     |        | and_call() |    | and_call() |        | and_call() |
414  *     |        +------------+    +------------+        +------------+
415  *     |           |                   |                      |
416  *  +------------------+   +------------------+   +------------------------+
417  *  | apix_dispatch_   |   | apix_dispatch_   |   | apix_dispatch_softint()|
418  *  | pending_hilevel()|   | pending_hardint()|   +------------------------+
419  *  +------------------+   +------------------+      |    |      |      |
420  *    |         |           |         |              |    |      |      |
421  *    | +----------------+  | +----------------+     |    |      |      |
422  *    | | apix_hilevel_  |  | | apix_intr_     |     |    |      |      |
423  *    | | intr_epilog()  |  | | thread_epilog()|     |    |      |      |
424  *    | +----------------+  | +----------------+     |    |      |      |
425  *    |         |           |       |                |    |      |      |
426  *    |   +------------+    |  +----------+   +------+    |      |      |
427  *    |   | *setlvlx() |    |  |*setlvlx()|   |           |      |      |
428  *    |   +------------+    |  +----------+   |   +----------+   |   +---------+
429  *    |                     |               +---+ |av_       | +---+ |apix_do_ |
430  * +---------------------------------+      |sti| |dispatch_ | |cli| |softint_ |
431  * | apix_dispatch_pending_autovect()|      +---+ |softvect()| +---+ |epilog() |
432  * +---------------------------------+            +----------+       +---------+
433  *  |!XC_HI_PIL  |       |         |                    |
434  * +---+  +-------+    +---+  +----------+          +-------+
435  * |sti|  |*intr()|    |cli|  |apix_post_|          |*intr()|
436  * +---+  +-------+    +---+  |hardint() |          +-------+
437  *                            +----------+
438  */
439 
440 #include <sys/cpuvar.h>
441 #include <sys/cpu_event.h>
442 #include <sys/regset.h>
443 #include <sys/psw.h>
444 #include <sys/types.h>
445 #include <sys/thread.h>
446 #include <sys/systm.h>
447 #include <sys/segments.h>
448 #include <sys/pcb.h>
449 #include <sys/trap.h>
450 #include <sys/ftrace.h>
451 #include <sys/traptrace.h>
452 #include <sys/clock.h>
453 #include <sys/panic.h>
454 #include <sys/disp.h>
455 #include <vm/seg_kp.h>
456 #include <sys/stack.h>
457 #include <sys/sysmacros.h>
458 #include <sys/cmn_err.h>
459 #include <sys/kstat.h>
460 #include <sys/smp_impldefs.h>
461 #include <sys/pool_pset.h>
462 #include <sys/zone.h>
463 #include <sys/bitmap.h>
464 #include <sys/archsystm.h>
465 #include <sys/machsystm.h>
466 #include <sys/ontrap.h>
467 #include <sys/x86_archext.h>
468 #include <sys/promif.h>
469 #include <vm/hat_i86.h>
470 #if defined(__xpv)
471 #include <sys/hypervisor.h>
472 #endif
473 
474 #if defined(__amd64) && !defined(__xpv)
475 /* If this fails, then the padding numbers in machcpuvar.h are wrong. */
476 CTASSERT((offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, mcpu_pad)) <
477     MMU_PAGESIZE);
478 CTASSERT((offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, mcpu_kpti)) >=
479     MMU_PAGESIZE);
480 CTASSERT((offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, mcpu_kpti_dbg)) <
481     2 * MMU_PAGESIZE);
482 CTASSERT((offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, mcpu_pad2)) <
483     2 * MMU_PAGESIZE);
484 CTASSERT(((sizeof (struct kpti_frame)) & 0xF) == 0);
485 CTASSERT(((offsetof(cpu_t, cpu_m) +
486     offsetof(struct machcpu, mcpu_kpti_dbg)) & 0xF) == 0);
487 CTASSERT((offsetof(struct kpti_frame, kf_tr_rsp) & 0xF) == 0);
488 #endif
489 
490 #if defined(__xpv) && defined(DEBUG)
491 
492 /*
493  * This panic message is intended as an aid to interrupt debugging.
494  *
495  * The associated assertion tests the condition of enabling
496  * events when events are already enabled.  The implication
497  * being that whatever code the programmer thought was
498  * protected by having events disabled until the second
499  * enable happened really wasn't protected at all ..
500  */
501 
502 int stistipanic = 1;	/* controls the debug panic check */
503 const char *stistimsg = "stisti";
504 ulong_t laststi[NCPU];
505 
506 /*
507  * This variable tracks the last place events were disabled on each cpu
508  * it assists in debugging when asserts that interrupts are enabled trip.
509  */
510 ulong_t lastcli[NCPU];
511 
512 #endif
513 
514 void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
515 
516 void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *) = do_interrupt;
517 uintptr_t (*get_intr_handler)(int, short) = NULL;
518 
519 /*
520  * Set cpu's base SPL level to the highest active interrupt level
521  */
522 void
523 set_base_spl(void)
524 {
525 	struct cpu *cpu = CPU;
526 	uint16_t active = (uint16_t)cpu->cpu_intr_actv;
527 
528 	cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);
529 }
530 
531 /*
532  * Do all the work necessary to set up the cpu and thread structures
533  * to dispatch a high-level interrupt.
534  *
535  * Returns 0 if we're -not- already on the high-level interrupt stack,
536  * (and *must* switch to it), non-zero if we are already on that stack.
537  *
538  * Called with interrupts masked.
539  * The 'pil' is already set to the appropriate level for rp->r_trapno.
540  */
541 static int
542 hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
543 {
544 	struct machcpu *mcpu = &cpu->cpu_m;
545 	uint_t mask;
546 	hrtime_t intrtime;
547 	hrtime_t now = tsc_read();
548 
549 	ASSERT(pil > LOCK_LEVEL);
550 
551 	if (pil == CBE_HIGH_PIL) {
552 		cpu->cpu_profile_pil = oldpil;
553 		if (USERMODE(rp->r_cs)) {
554 			cpu->cpu_profile_pc = 0;
555 			cpu->cpu_profile_upc = rp->r_pc;
556 			cpu->cpu_cpcprofile_pc = 0;
557 			cpu->cpu_cpcprofile_upc = rp->r_pc;
558 		} else {
559 			cpu->cpu_profile_pc = rp->r_pc;
560 			cpu->cpu_profile_upc = 0;
561 			cpu->cpu_cpcprofile_pc = rp->r_pc;
562 			cpu->cpu_cpcprofile_upc = 0;
563 		}
564 	}
565 
566 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
567 	if (mask != 0) {
568 		int nestpil;
569 
570 		/*
571 		 * We have interrupted another high-level interrupt.
572 		 * Load starting timestamp, compute interval, update
573 		 * cumulative counter.
574 		 */
575 		nestpil = bsrw_insn((uint16_t)mask);
576 		ASSERT(nestpil < pil);
577 		intrtime = now -
578 		    mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
579 		mcpu->intrstat[nestpil][0] += intrtime;
580 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
581 		/*
582 		 * Another high-level interrupt is active below this one, so
583 		 * there is no need to check for an interrupt thread.  That
584 		 * will be done by the lowest priority high-level interrupt
585 		 * active.
586 		 */
587 	} else {
588 		kthread_t *t = cpu->cpu_thread;
589 
590 		/*
591 		 * See if we are interrupting a low-level interrupt thread.
592 		 * If so, account for its time slice only if its time stamp
593 		 * is non-zero.
594 		 */
595 		if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
596 			intrtime = now - t->t_intr_start;
597 			mcpu->intrstat[t->t_pil][0] += intrtime;
598 			cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
599 			t->t_intr_start = 0;
600 		}
601 	}
602 
603 	/*
604 	 * Store starting timestamp in CPU structure for this PIL.
605 	 */
606 	mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
607 
608 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
609 
610 	if (pil == 15) {
611 		/*
612 		 * To support reentrant level 15 interrupts, we maintain a
613 		 * recursion count in the top half of cpu_intr_actv.  Only
614 		 * when this count hits zero do we clear the PIL 15 bit from
615 		 * the lower half of cpu_intr_actv.
616 		 */
617 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
618 		(*refcntp)++;
619 	}
620 
621 	mask = cpu->cpu_intr_actv;
622 
623 	cpu->cpu_intr_actv |= (1 << pil);
624 
625 	return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
626 }
627 
628 /*
629  * Does most of the work of returning from a high level interrupt.
630  *
631  * Returns 0 if there are no more high level interrupts (in which
632  * case we must switch back to the interrupted thread stack) or
633  * non-zero if there are more (in which case we should stay on it).
634  *
635  * Called with interrupts masked
636  */
637 static int
638 hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
639 {
640 	struct machcpu *mcpu = &cpu->cpu_m;
641 	uint_t mask;
642 	hrtime_t intrtime;
643 	hrtime_t now = tsc_read();
644 
645 	ASSERT(mcpu->mcpu_pri == pil);
646 
647 	cpu->cpu_stats.sys.intr[pil - 1]++;
648 
649 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
650 
651 	if (pil == 15) {
652 		/*
653 		 * To support reentrant level 15 interrupts, we maintain a
654 		 * recursion count in the top half of cpu_intr_actv.  Only
655 		 * when this count hits zero do we clear the PIL 15 bit from
656 		 * the lower half of cpu_intr_actv.
657 		 */
658 		uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
659 
660 		ASSERT(*refcntp > 0);
661 
662 		if (--(*refcntp) == 0)
663 			cpu->cpu_intr_actv &= ~(1 << pil);
664 	} else {
665 		cpu->cpu_intr_actv &= ~(1 << pil);
666 	}
667 
668 	ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
669 
670 	intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
671 	mcpu->intrstat[pil][0] += intrtime;
672 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
673 
674 	/*
675 	 * Check for lower-pil nested high-level interrupt beneath
676 	 * current one.  If so, place a starting timestamp in its
677 	 * pil_high_start entry.
678 	 */
679 	mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
680 	if (mask != 0) {
681 		int nestpil;
682 
683 		/*
684 		 * find PIL of nested interrupt
685 		 */
686 		nestpil = bsrw_insn((uint16_t)mask);
687 		ASSERT(nestpil < pil);
688 		mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
689 		/*
690 		 * (Another high-level interrupt is active below this one,
691 		 * so there is no need to check for an interrupt
692 		 * thread.  That will be done by the lowest priority
693 		 * high-level interrupt active.)
694 		 */
695 	} else {
696 		/*
697 		 * Check to see if there is a low-level interrupt active.
698 		 * If so, place a starting timestamp in the thread
699 		 * structure.
700 		 */
701 		kthread_t *t = cpu->cpu_thread;
702 
703 		if (t->t_flag & T_INTR_THREAD)
704 			t->t_intr_start = now;
705 	}
706 
707 	mcpu->mcpu_pri = oldpil;
708 	(void) (*setlvlx)(oldpil, vecnum);
709 
710 	return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
711 }
712 
713 /*
714  * Set up the cpu, thread and interrupt thread structures for
715  * executing an interrupt thread.  The new stack pointer of the
716  * interrupt thread (which *must* be switched to) is returned.
717  */
718 static caddr_t
719 intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
720 {
721 	struct machcpu *mcpu = &cpu->cpu_m;
722 	kthread_t *t, *volatile it;
723 	hrtime_t now = tsc_read();
724 
725 	ASSERT(pil > 0);
726 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
727 	cpu->cpu_intr_actv |= (1 << pil);
728 
729 	/*
730 	 * Get set to run an interrupt thread.
731 	 * There should always be an interrupt thread, since we
732 	 * allocate one for each level on each CPU.
733 	 *
734 	 * t_intr_start could be zero due to cpu_intr_swtch_enter.
735 	 */
736 	t = cpu->cpu_thread;
737 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
738 		hrtime_t intrtime = now - t->t_intr_start;
739 		mcpu->intrstat[t->t_pil][0] += intrtime;
740 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
741 		t->t_intr_start = 0;
742 	}
743 
744 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
745 
746 	t->t_sp = (uintptr_t)stackptr;	/* mark stack in curthread for resume */
747 
748 	/*
749 	 * unlink the interrupt thread off the cpu
750 	 *
751 	 * Note that the code in kcpc_overflow_intr -relies- on the
752 	 * ordering of events here - in particular that t->t_lwp of
753 	 * the interrupt thread is set to the pinned thread *before*
754 	 * curthread is changed.
755 	 */
756 	it = cpu->cpu_intr_thread;
757 	cpu->cpu_intr_thread = it->t_link;
758 	it->t_intr = t;
759 	it->t_lwp = t->t_lwp;
760 
761 	/*
762 	 * (threads on the interrupt thread free list could have state
763 	 * preset to TS_ONPROC, but it helps in debugging if
764 	 * they're TS_FREE.)
765 	 */
766 	it->t_state = TS_ONPROC;
767 
768 	cpu->cpu_thread = it;		/* new curthread on this cpu */
769 	it->t_pil = (uchar_t)pil;
770 	it->t_pri = intr_pri + (pri_t)pil;
771 	it->t_intr_start = now;
772 
773 	return (it->t_stk);
774 }
775 
776 
777 #ifdef DEBUG
778 int intr_thread_cnt;
779 #endif
780 
781 /*
782  * Called with interrupts disabled
783  */
784 static void
785 intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
786 {
787 	struct machcpu *mcpu = &cpu->cpu_m;
788 	kthread_t *t;
789 	kthread_t *it = cpu->cpu_thread;	/* curthread */
790 	uint_t pil, basespl;
791 	hrtime_t intrtime;
792 	hrtime_t now = tsc_read();
793 
794 	pil = it->t_pil;
795 	cpu->cpu_stats.sys.intr[pil - 1]++;
796 
797 	ASSERT(it->t_intr_start != 0);
798 	intrtime = now - it->t_intr_start;
799 	mcpu->intrstat[pil][0] += intrtime;
800 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
801 
802 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
803 	cpu->cpu_intr_actv &= ~(1 << pil);
804 
805 	/*
806 	 * If there is still an interrupted thread underneath this one
807 	 * then the interrupt was never blocked and the return is
808 	 * fairly simple.  Otherwise it isn't.
809 	 */
810 	if ((t = it->t_intr) == NULL) {
811 		/*
812 		 * The interrupted thread is no longer pinned underneath
813 		 * the interrupt thread.  This means the interrupt must
814 		 * have blocked, and the interrupted thread has been
815 		 * unpinned, and has probably been running around the
816 		 * system for a while.
817 		 *
818 		 * Since there is no longer a thread under this one, put
819 		 * this interrupt thread back on the CPU's free list and
820 		 * resume the idle thread which will dispatch the next
821 		 * thread to run.
822 		 */
823 #ifdef DEBUG
824 		intr_thread_cnt++;
825 #endif
826 		cpu->cpu_stats.sys.intrblk++;
827 		/*
828 		 * Set CPU's base SPL based on active interrupts bitmask
829 		 */
830 		set_base_spl();
831 		basespl = cpu->cpu_base_spl;
832 		mcpu->mcpu_pri = basespl;
833 		(*setlvlx)(basespl, vec);
834 		(void) splhigh();
835 		sti();
836 		it->t_state = TS_FREE;
837 		/*
838 		 * Return interrupt thread to pool
839 		 */
840 		it->t_link = cpu->cpu_intr_thread;
841 		cpu->cpu_intr_thread = it;
842 		swtch();
843 		panic("intr_thread_epilog: swtch returned");
844 		/*NOTREACHED*/
845 	}
846 
847 	/*
848 	 * Return interrupt thread to the pool
849 	 */
850 	it->t_link = cpu->cpu_intr_thread;
851 	cpu->cpu_intr_thread = it;
852 	it->t_state = TS_FREE;
853 
854 	basespl = cpu->cpu_base_spl;
855 	pil = MAX(oldpil, basespl);
856 	mcpu->mcpu_pri = pil;
857 	(*setlvlx)(pil, vec);
858 	t->t_intr_start = now;
859 	cpu->cpu_thread = t;
860 }
861 
862 /*
863  * intr_get_time() is a resource for interrupt handlers to determine how
864  * much time has been spent handling the current interrupt. Such a function
865  * is needed because higher level interrupts can arrive during the
866  * processing of an interrupt.  intr_get_time() only returns time spent in the
867  * current interrupt handler.
868  *
869  * The caller must be calling from an interrupt handler running at a pil
870  * below or at lock level. Timings are not provided for high-level
871  * interrupts.
872  *
873  * The first time intr_get_time() is called while handling an interrupt,
874  * it returns the time since the interrupt handler was invoked. Subsequent
875  * calls will return the time since the prior call to intr_get_time(). Time
876  * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
877  *
878  * Theory Of Intrstat[][]:
879  *
880  * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
881  * uint64_ts per pil.
882  *
883  * intrstat[pil][0] is a cumulative count of the number of ticks spent
884  * handling all interrupts at the specified pil on this CPU. It is
885  * exported via kstats to the user.
886  *
887  * intrstat[pil][1] is always a count of ticks less than or equal to the
888  * value in [0]. The difference between [1] and [0] is the value returned
889  * by a call to intr_get_time(). At the start of interrupt processing,
890  * [0] and [1] will be equal (or nearly so). As the interrupt consumes
891  * time, [0] will increase, but [1] will remain the same. A call to
892  * intr_get_time() will return the difference, then update [1] to be the
893  * same as [0]. Future calls will return the time since the last call.
894  * Finally, when the interrupt completes, [1] is updated to the same as [0].
895  *
896  * Implementation:
897  *
898  * intr_get_time() works much like a higher level interrupt arriving. It
899  * "checkpoints" the timing information by incrementing intrstat[pil][0]
900  * to include elapsed running time, and by setting t_intr_start to rdtsc.
901  * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
902  * and updates intrstat[pil][1] to be the same as the new value of
903  * intrstat[pil][0].
904  *
905  * In the normal handling of interrupts, after an interrupt handler returns
906  * and the code in intr_thread() updates intrstat[pil][0], it then sets
907  * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
908  * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
909  * is 0.
910  *
911  * Whenever interrupts arrive on a CPU which is handling a lower pil
912  * interrupt, they update the lower pil's [0] to show time spent in the
913  * handler that they've interrupted. This results in a growing discrepancy
914  * between [0] and [1], which is returned the next time intr_get_time() is
915  * called. Time spent in the higher-pil interrupt will not be returned in
916  * the next intr_get_time() call from the original interrupt, because
917  * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
918  */
919 uint64_t
920 intr_get_time(void)
921 {
922 	struct cpu *cpu;
923 	struct machcpu *mcpu;
924 	kthread_t *t;
925 	uint64_t time, delta, ret;
926 	uint_t pil;
927 
928 	cli();
929 	cpu = CPU;
930 	mcpu = &cpu->cpu_m;
931 	t = cpu->cpu_thread;
932 	pil = t->t_pil;
933 	ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
934 	ASSERT(t->t_flag & T_INTR_THREAD);
935 	ASSERT(pil != 0);
936 	ASSERT(t->t_intr_start != 0);
937 
938 	time = tsc_read();
939 	delta = time - t->t_intr_start;
940 	t->t_intr_start = time;
941 
942 	time = mcpu->intrstat[pil][0] + delta;
943 	ret = time - mcpu->intrstat[pil][1];
944 	mcpu->intrstat[pil][0] = time;
945 	mcpu->intrstat[pil][1] = time;
946 	cpu->cpu_intracct[cpu->cpu_mstate] += delta;
947 
948 	sti();
949 	return (ret);
950 }
951 
952 static caddr_t
953 dosoftint_prolog(
954 	struct cpu *cpu,
955 	caddr_t stackptr,
956 	uint32_t st_pending,
957 	uint_t oldpil)
958 {
959 	kthread_t *t, *volatile it;
960 	struct machcpu *mcpu = &cpu->cpu_m;
961 	uint_t pil;
962 	hrtime_t now;
963 
964 top:
965 	ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
966 
967 	pil = bsrw_insn((uint16_t)st_pending);
968 	if (pil <= oldpil || pil <= cpu->cpu_base_spl)
969 		return (0);
970 
971 	/*
972 	 * XX64	Sigh.
973 	 *
974 	 * This is a transliteration of the i386 assembler code for
975 	 * soft interrupts.  One question is "why does this need
976 	 * to be atomic?"  One possible race is -other- processors
977 	 * posting soft interrupts to us in set_pending() i.e. the
978 	 * CPU might get preempted just after the address computation,
979 	 * but just before the atomic transaction, so another CPU would
980 	 * actually set the original CPU's st_pending bit.  However,
981 	 * it looks like it would be simpler to disable preemption there.
982 	 * Are there other races for which preemption control doesn't work?
983 	 *
984 	 * The i386 assembler version -also- checks to see if the bit
985 	 * being cleared was actually set; if it wasn't, it rechecks
986 	 * for more.  This seems a bit strange, as the only code that
987 	 * ever clears the bit is -this- code running with interrupts
988 	 * disabled on -this- CPU.  This code would probably be cheaper:
989 	 *
990 	 * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
991 	 *   ~(1 << pil));
992 	 *
993 	 * and t->t_preempt--/++ around set_pending() even cheaper,
994 	 * but at this point, correctness is critical, so we slavishly
995 	 * emulate the i386 port.
996 	 */
997 	if (atomic_btr32((uint32_t *)
998 	    &mcpu->mcpu_softinfo.st_pending, pil) == 0) {
999 		st_pending = mcpu->mcpu_softinfo.st_pending;
1000 		goto top;
1001 	}
1002 
1003 	mcpu->mcpu_pri = pil;
1004 	(*setspl)(pil);
1005 
1006 	now = tsc_read();
1007 
1008 	/*
1009 	 * Get set to run interrupt thread.
1010 	 * There should always be an interrupt thread since we
1011 	 * allocate one for each level on the CPU.
1012 	 */
1013 	it = cpu->cpu_intr_thread;
1014 	cpu->cpu_intr_thread = it->t_link;
1015 
1016 	/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
1017 	t = cpu->cpu_thread;
1018 	if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
1019 		hrtime_t intrtime = now - t->t_intr_start;
1020 		mcpu->intrstat[pil][0] += intrtime;
1021 		cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1022 		t->t_intr_start = 0;
1023 	}
1024 
1025 	/*
1026 	 * Note that the code in kcpc_overflow_intr -relies- on the
1027 	 * ordering of events here - in particular that t->t_lwp of
1028 	 * the interrupt thread is set to the pinned thread *before*
1029 	 * curthread is changed.
1030 	 */
1031 	it->t_lwp = t->t_lwp;
1032 	it->t_state = TS_ONPROC;
1033 
1034 	/*
1035 	 * Push interrupted thread onto list from new thread.
1036 	 * Set the new thread as the current one.
1037 	 * Set interrupted thread's T_SP because if it is the idle thread,
1038 	 * resume() may use that stack between threads.
1039 	 */
1040 
1041 	ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
1042 	t->t_sp = (uintptr_t)stackptr;
1043 
1044 	it->t_intr = t;
1045 	cpu->cpu_thread = it;
1046 
1047 	/*
1048 	 * Set bit for this pil in CPU's interrupt active bitmask.
1049 	 */
1050 	ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
1051 	cpu->cpu_intr_actv |= (1 << pil);
1052 
1053 	/*
1054 	 * Initialize thread priority level from intr_pri
1055 	 */
1056 	it->t_pil = (uchar_t)pil;
1057 	it->t_pri = (pri_t)pil + intr_pri;
1058 	it->t_intr_start = now;
1059 
1060 	return (it->t_stk);
1061 }
1062 
1063 static void
1064 dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
1065 {
1066 	struct machcpu *mcpu = &cpu->cpu_m;
1067 	kthread_t *t, *it;
1068 	uint_t pil, basespl;
1069 	hrtime_t intrtime;
1070 	hrtime_t now = tsc_read();
1071 
1072 	it = cpu->cpu_thread;
1073 	pil = it->t_pil;
1074 
1075 	cpu->cpu_stats.sys.intr[pil - 1]++;
1076 
1077 	ASSERT(cpu->cpu_intr_actv & (1 << pil));
1078 	cpu->cpu_intr_actv &= ~(1 << pil);
1079 	intrtime = now - it->t_intr_start;
1080 	mcpu->intrstat[pil][0] += intrtime;
1081 	cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1082 
1083 	/*
1084 	 * If there is still an interrupted thread underneath this one
1085 	 * then the interrupt was never blocked and the return is
1086 	 * fairly simple.  Otherwise it isn't.
1087 	 */
1088 	if ((t = it->t_intr) == NULL) {
1089 		/*
1090 		 * Put thread back on the interrupt thread list.
1091 		 * This was an interrupt thread, so set CPU's base SPL.
1092 		 */
1093 		set_base_spl();
1094 		it->t_state = TS_FREE;
1095 		it->t_link = cpu->cpu_intr_thread;
1096 		cpu->cpu_intr_thread = it;
1097 		(void) splhigh();
1098 		sti();
1099 		swtch();
1100 		/*NOTREACHED*/
1101 		panic("dosoftint_epilog: swtch returned");
1102 	}
1103 	it->t_link = cpu->cpu_intr_thread;
1104 	cpu->cpu_intr_thread = it;
1105 	it->t_state = TS_FREE;
1106 	cpu->cpu_thread = t;
1107 	if (t->t_flag & T_INTR_THREAD)
1108 		t->t_intr_start = now;
1109 	basespl = cpu->cpu_base_spl;
1110 	pil = MAX(oldpil, basespl);
1111 	mcpu->mcpu_pri = pil;
1112 	(*setspl)(pil);
1113 }
1114 
1115 
1116 /*
1117  * Make the interrupted thread 'to' be runnable.
1118  *
1119  * Since t->t_sp has already been saved, t->t_pc is all
1120  * that needs to be set in this function.
1121  *
1122  * Returns the interrupt level of the interrupt thread.
1123  */
1124 int
1125 intr_passivate(
1126 	kthread_t *it,		/* interrupt thread */
1127 	kthread_t *t)		/* interrupted thread */
1128 {
1129 	extern void _sys_rtt();
1130 
1131 	ASSERT(it->t_flag & T_INTR_THREAD);
1132 	ASSERT(SA(t->t_sp) == t->t_sp);
1133 
1134 	t->t_pc = (uintptr_t)_sys_rtt;
1135 	return (it->t_pil);
1136 }
1137 
1138 /*
1139  * Create interrupt kstats for this CPU.
1140  */
1141 void
1142 cpu_create_intrstat(cpu_t *cp)
1143 {
1144 	int		i;
1145 	kstat_t		*intr_ksp;
1146 	kstat_named_t	*knp;
1147 	char		name[KSTAT_STRLEN];
1148 	zoneid_t	zoneid;
1149 
1150 	ASSERT(MUTEX_HELD(&cpu_lock));
1151 
1152 	if (pool_pset_enabled())
1153 		zoneid = GLOBAL_ZONEID;
1154 	else
1155 		zoneid = ALL_ZONES;
1156 
1157 	intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
1158 	    KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid);
1159 
1160 	/*
1161 	 * Initialize each PIL's named kstat
1162 	 */
1163 	if (intr_ksp != NULL) {
1164 		intr_ksp->ks_update = cpu_kstat_intrstat_update;
1165 		knp = (kstat_named_t *)intr_ksp->ks_data;
1166 		intr_ksp->ks_private = cp;
1167 		for (i = 0; i < PIL_MAX; i++) {
1168 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
1169 			    i + 1);
1170 			kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
1171 			(void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
1172 			    i + 1);
1173 			kstat_named_init(&knp[(i * 2) + 1], name,
1174 			    KSTAT_DATA_UINT64);
1175 		}
1176 		kstat_install(intr_ksp);
1177 	}
1178 }
1179 
1180 /*
1181  * Delete interrupt kstats for this CPU.
1182  */
1183 void
1184 cpu_delete_intrstat(cpu_t *cp)
1185 {
1186 	kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
1187 }
1188 
1189 /*
1190  * Convert interrupt statistics from CPU ticks to nanoseconds and
1191  * update kstat.
1192  */
1193 int
1194 cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
1195 {
1196 	kstat_named_t	*knp = ksp->ks_data;
1197 	cpu_t		*cpup = (cpu_t *)ksp->ks_private;
1198 	int		i;
1199 	hrtime_t	hrt;
1200 
1201 	if (rw == KSTAT_WRITE)
1202 		return (EACCES);
1203 
1204 	for (i = 0; i < PIL_MAX; i++) {
1205 		hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
1206 		scalehrtimef(&hrt);
1207 		knp[i * 2].value.ui64 = (uint64_t)hrt;
1208 		knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
1209 	}
1210 
1211 	return (0);
1212 }
1213 
1214 /*
1215  * An interrupt thread is ending a time slice, so compute the interval it
1216  * ran for and update the statistic for its PIL.
1217  */
1218 void
1219 cpu_intr_swtch_enter(kthread_id_t t)
1220 {
1221 	uint64_t	interval;
1222 	uint64_t	start;
1223 	cpu_t		*cpu;
1224 
1225 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1226 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1227 
1228 	/*
1229 	 * We could be here with a zero timestamp. This could happen if:
1230 	 * an interrupt thread which no longer has a pinned thread underneath
1231 	 * it (i.e. it blocked at some point in its past) has finished running
1232 	 * its handler. intr_thread() updated the interrupt statistic for its
1233 	 * PIL and zeroed its timestamp. Since there was no pinned thread to
1234 	 * return to, swtch() gets called and we end up here.
1235 	 *
1236 	 * Note that we use atomic ops below (atomic_cas_64 and
1237 	 * atomic_add_64), which we don't use in the functions above,
1238 	 * because we're not called with interrupts blocked, but the
1239 	 * epilog/prolog functions are.
1240 	 */
1241 	if (t->t_intr_start) {
1242 		do {
1243 			start = t->t_intr_start;
1244 			interval = tsc_read() - start;
1245 		} while (atomic_cas_64(&t->t_intr_start, start, 0) != start);
1246 		cpu = CPU;
1247 		cpu->cpu_m.intrstat[t->t_pil][0] += interval;
1248 
1249 		atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
1250 		    interval);
1251 	} else
1252 		ASSERT(t->t_intr == NULL);
1253 }
1254 
1255 /*
1256  * An interrupt thread is returning from swtch(). Place a starting timestamp
1257  * in its thread structure.
1258  */
1259 void
1260 cpu_intr_swtch_exit(kthread_id_t t)
1261 {
1262 	uint64_t ts;
1263 
1264 	ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1265 	ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1266 
1267 	do {
1268 		ts = t->t_intr_start;
1269 	} while (atomic_cas_64(&t->t_intr_start, ts, tsc_read()) != ts);
1270 }
1271 
1272 /*
1273  * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
1274  */
1275 /*ARGSUSED*/
1276 static void
1277 dispatch_hilevel(uint_t vector, uint_t arg2)
1278 {
1279 	sti();
1280 	av_dispatch_autovect(vector);
1281 	cli();
1282 }
1283 
1284 /*
1285  * Dispatch a soft interrupt
1286  */
1287 /*ARGSUSED*/
1288 static void
1289 dispatch_softint(uint_t oldpil, uint_t arg2)
1290 {
1291 	struct cpu *cpu = CPU;
1292 
1293 	sti();
1294 	av_dispatch_softvect((int)cpu->cpu_thread->t_pil);
1295 	cli();
1296 
1297 	/*
1298 	 * Must run softint_epilog() on the interrupt thread stack, since
1299 	 * there may not be a return from it if the interrupt thread blocked.
1300 	 */
1301 	dosoftint_epilog(cpu, oldpil);
1302 }
1303 
1304 /*
1305  * Dispatch a normal interrupt
1306  */
1307 static void
1308 dispatch_hardint(uint_t vector, uint_t oldipl)
1309 {
1310 	struct cpu *cpu = CPU;
1311 
1312 	sti();
1313 	av_dispatch_autovect(vector);
1314 	cli();
1315 
1316 	/*
1317 	 * Must run intr_thread_epilog() on the interrupt thread stack, since
1318 	 * there may not be a return from it if the interrupt thread blocked.
1319 	 */
1320 	intr_thread_epilog(cpu, vector, oldipl);
1321 }
1322 
1323 /*
1324  * Deliver any softints the current interrupt priority allows.
1325  * Called with interrupts disabled.
1326  */
1327 void
1328 dosoftint(struct regs *regs)
1329 {
1330 	struct cpu *cpu = CPU;
1331 	int oldipl;
1332 	caddr_t newsp;
1333 
1334 	while (cpu->cpu_softinfo.st_pending) {
1335 		oldipl = cpu->cpu_pri;
1336 		newsp = dosoftint_prolog(cpu, (caddr_t)regs,
1337 		    cpu->cpu_softinfo.st_pending, oldipl);
1338 		/*
1339 		 * If returned stack pointer is NULL, priority is too high
1340 		 * to run any of the pending softints now.
1341 		 * Break out and they will be run later.
1342 		 */
1343 		if (newsp == NULL)
1344 			break;
1345 		switch_sp_and_call(newsp, dispatch_softint, oldipl, 0);
1346 	}
1347 }
1348 
1349 /*
1350  * Interrupt service routine, called with interrupts disabled.
1351  */
1352 /*ARGSUSED*/
1353 void
1354 do_interrupt(struct regs *rp, trap_trace_rec_t *ttp)
1355 {
1356 	struct cpu *cpu = CPU;
1357 	int newipl, oldipl = cpu->cpu_pri;
1358 	uint_t vector;
1359 	caddr_t newsp;
1360 
1361 #ifdef TRAPTRACE
1362 	ttp->ttr_marker = TT_INTERRUPT;
1363 	ttp->ttr_ipl = 0xff;
1364 	ttp->ttr_pri = oldipl;
1365 	ttp->ttr_spl = cpu->cpu_base_spl;
1366 	ttp->ttr_vector = 0xff;
1367 #endif	/* TRAPTRACE */
1368 
1369 	cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR);
1370 
1371 	++*(uint16_t *)&cpu->cpu_m.mcpu_istamp;
1372 
1373 	/*
1374 	 * If it's a softint go do it now.
1375 	 */
1376 	if (rp->r_trapno == T_SOFTINT) {
1377 		dosoftint(rp);
1378 		ASSERT(!interrupts_enabled());
1379 		return;
1380 	}
1381 
1382 	/*
1383 	 * Raise the interrupt priority.
1384 	 */
1385 	newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno);
1386 #ifdef TRAPTRACE
1387 	ttp->ttr_ipl = newipl;
1388 #endif	/* TRAPTRACE */
1389 
1390 	/*
1391 	 * Bail if it is a spurious interrupt
1392 	 */
1393 	if (newipl == -1)
1394 		return;
1395 	cpu->cpu_pri = newipl;
1396 	vector = rp->r_trapno;
1397 #ifdef TRAPTRACE
1398 	ttp->ttr_vector = vector;
1399 #endif	/* TRAPTRACE */
1400 	if (newipl > LOCK_LEVEL) {
1401 		/*
1402 		 * High priority interrupts run on this cpu's interrupt stack.
1403 		 */
1404 		if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {
1405 			newsp = cpu->cpu_intr_stack;
1406 			switch_sp_and_call(newsp, dispatch_hilevel, vector, 0);
1407 		} else { /* already on the interrupt stack */
1408 			dispatch_hilevel(vector, 0);
1409 		}
1410 		(void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);
1411 	} else {
1412 		/*
1413 		 * Run this interrupt in a separate thread.
1414 		 */
1415 		newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);
1416 		switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl);
1417 	}
1418 
1419 #if !defined(__xpv)
1420 	/*
1421 	 * Deliver any pending soft interrupts.
1422 	 */
1423 	if (cpu->cpu_softinfo.st_pending)
1424 		dosoftint(rp);
1425 #endif	/* !__xpv */
1426 }
1427 
1428 
1429 /*
1430  * Common tasks always done by _sys_rtt, called with interrupts disabled.
1431  * Returns 1 if returning to userland, 0 if returning to system mode.
1432  */
1433 int
1434 sys_rtt_common(struct regs *rp)
1435 {
1436 	kthread_t *tp;
1437 	extern void mutex_exit_critical_start();
1438 	extern long mutex_exit_critical_size;
1439 	extern void mutex_owner_running_critical_start();
1440 	extern long mutex_owner_running_critical_size;
1441 
1442 loop:
1443 
1444 	/*
1445 	 * Check if returning to user
1446 	 */
1447 	tp = CPU->cpu_thread;
1448 	if (USERMODE(rp->r_cs)) {
1449 		pcb_t *pcb;
1450 
1451 		/*
1452 		 * Check if AST pending.
1453 		 */
1454 		if (tp->t_astflag) {
1455 			/*
1456 			 * Let trap() handle the AST
1457 			 */
1458 			sti();
1459 			rp->r_trapno = T_AST;
1460 			trap(rp, (caddr_t)0, CPU->cpu_id);
1461 			cli();
1462 			goto loop;
1463 		}
1464 
1465 		pcb = &tp->t_lwp->lwp_pcb;
1466 
1467 		/*
1468 		 * Check to see if we need to initialize the FPU for this
1469 		 * thread. This should be an uncommon occurrence, but may happen
1470 		 * in the case where the system creates an lwp through an
1471 		 * abnormal path such as the agent lwp. Make sure that we still
1472 		 * happen to have the FPU in a good state.
1473 		 */
1474 		if ((pcb->pcb_fpu.fpu_flags & FPU_EN) == 0) {
1475 			kpreempt_disable();
1476 			fp_seed();
1477 			kpreempt_enable();
1478 			PCB_SET_UPDATE_FPU(pcb);
1479 		}
1480 
1481 		/*
1482 		 * We are done if segment registers do not need updating.
1483 		 */
1484 		if (!PCB_NEED_UPDATE(pcb))
1485 			return (1);
1486 
1487 		if (PCB_NEED_UPDATE_SEGS(pcb) && update_sregs(rp, tp->t_lwp)) {
1488 			/*
1489 			 * 1 or more of the selectors is bad.
1490 			 * Deliver a SIGSEGV.
1491 			 */
1492 			proc_t *p = ttoproc(tp);
1493 
1494 			sti();
1495 			mutex_enter(&p->p_lock);
1496 			tp->t_lwp->lwp_cursig = SIGSEGV;
1497 			mutex_exit(&p->p_lock);
1498 			psig();
1499 			tp->t_sig_check = 1;
1500 			cli();
1501 		}
1502 		PCB_CLEAR_UPDATE_SEGS(pcb);
1503 
1504 		if (PCB_NEED_UPDATE_FPU(pcb)) {
1505 			fprestore_ctxt(&pcb->pcb_fpu);
1506 		}
1507 		PCB_CLEAR_UPDATE_FPU(pcb);
1508 
1509 		ASSERT0(PCB_NEED_UPDATE(pcb));
1510 
1511 		return (1);
1512 	}
1513 
1514 #if !defined(__xpv)
1515 	/*
1516 	 * Assert that we're not trying to return into the syscall return
1517 	 * trampolines. Things will go baaaaad if we try to do that.
1518 	 *
1519 	 * Note that none of these run with interrupts on, so this should
1520 	 * never happen (even in the sysexit case the STI doesn't take effect
1521 	 * until after sysexit finishes).
1522 	 */
1523 	extern void tr_sysc_ret_start();
1524 	extern void tr_sysc_ret_end();
1525 	ASSERT(!(rp->r_pc >= (uintptr_t)tr_sysc_ret_start &&
1526 	    rp->r_pc <= (uintptr_t)tr_sysc_ret_end));
1527 #endif
1528 
1529 	/*
1530 	 * Here if we are returning to supervisor mode.
1531 	 * Check for a kernel preemption request.
1532 	 */
1533 	if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
1534 
1535 		/*
1536 		 * Do nothing if already in kpreempt
1537 		 */
1538 		if (!tp->t_preempt_lk) {
1539 			tp->t_preempt_lk = 1;
1540 			sti();
1541 			kpreempt(1); /* asynchronous kpreempt call */
1542 			cli();
1543 			tp->t_preempt_lk = 0;
1544 		}
1545 	}
1546 
1547 	/*
1548 	 * If we interrupted the mutex_exit() critical region we must
1549 	 * reset the PC back to the beginning to prevent missed wakeups
1550 	 * See the comments in mutex_exit() for details.
1551 	 */
1552 	if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
1553 	    mutex_exit_critical_size) {
1554 		rp->r_pc = (greg_t)mutex_exit_critical_start;
1555 	}
1556 
1557 	/*
1558 	 * If we interrupted the mutex_owner_running() critical region we
1559 	 * must reset the PC back to the beginning to prevent dereferencing
1560 	 * of a freed thread pointer. See the comments in mutex_owner_running
1561 	 * for details.
1562 	 */
1563 	if ((uintptr_t)rp->r_pc -
1564 	    (uintptr_t)mutex_owner_running_critical_start <
1565 	    mutex_owner_running_critical_size) {
1566 		rp->r_pc = (greg_t)mutex_owner_running_critical_start;
1567 	}
1568 
1569 	return (0);
1570 }
1571 
1572 void
1573 send_dirint(int cpuid, int int_level)
1574 {
1575 	(*send_dirintf)(cpuid, int_level);
1576 }
1577 
1578 #define	IS_FAKE_SOFTINT(flag, newpri)		\
1579 	(((flag) & PS_IE) &&				\
1580 	    (((*get_pending_spl)() > (newpri)) ||	\
1581 	    bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri)))
1582 
1583 /*
1584  * do_splx routine, takes new ipl to set
1585  * returns the old ipl.
1586  * We are careful not to set priority lower than CPU->cpu_base_pri,
1587  * even though it seems we're raising the priority, it could be set
1588  * higher at any time by an interrupt routine, so we must block interrupts
1589  * and look at CPU->cpu_base_pri
1590  */
1591 int
1592 do_splx(int newpri)
1593 {
1594 	ulong_t	flag;
1595 	cpu_t	*cpu;
1596 	int	curpri, basepri;
1597 
1598 	flag = intr_clear();
1599 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1600 	curpri = cpu->cpu_m.mcpu_pri;
1601 	basepri = cpu->cpu_base_spl;
1602 	if (newpri < basepri)
1603 		newpri = basepri;
1604 	cpu->cpu_m.mcpu_pri = newpri;
1605 	(*setspl)(newpri);
1606 	/*
1607 	 * If we are going to reenable interrupts see if new priority level
1608 	 * allows pending softint delivery.
1609 	 */
1610 	if (IS_FAKE_SOFTINT(flag, newpri))
1611 		fakesoftint();
1612 	ASSERT(!interrupts_enabled());
1613 	intr_restore(flag);
1614 	return (curpri);
1615 }
1616 
1617 /*
1618  * Common spl raise routine, takes new ipl to set
1619  * returns the old ipl, will not lower ipl.
1620  */
1621 int
1622 splr(int newpri)
1623 {
1624 	ulong_t	flag;
1625 	cpu_t	*cpu;
1626 	int	curpri, basepri;
1627 
1628 	flag = intr_clear();
1629 	cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1630 	curpri = cpu->cpu_m.mcpu_pri;
1631 	/*
1632 	 * Only do something if new priority is larger
1633 	 */
1634 	if (newpri > curpri) {
1635 		basepri = cpu->cpu_base_spl;
1636 		if (newpri < basepri)
1637 			newpri = basepri;
1638 		cpu->cpu_m.mcpu_pri = newpri;
1639 		(*setspl)(newpri);
1640 		/*
1641 		 * See if new priority level allows pending softint delivery
1642 		 */
1643 		if (IS_FAKE_SOFTINT(flag, newpri))
1644 			fakesoftint();
1645 	}
1646 	intr_restore(flag);
1647 	return (curpri);
1648 }
1649 
1650 int
1651 getpil(void)
1652 {
1653 	return (CPU->cpu_m.mcpu_pri);
1654 }
1655 
1656 int
1657 spl_xcall(void)
1658 {
1659 	return (splr(ipltospl(XCALL_PIL)));
1660 }
1661 
1662 int
1663 interrupts_enabled(void)
1664 {
1665 	ulong_t	flag;
1666 
1667 	flag = getflags();
1668 	return ((flag & PS_IE) == PS_IE);
1669 }
1670 
1671 #ifdef DEBUG
1672 void
1673 assert_ints_enabled(void)
1674 {
1675 	ASSERT(!interrupts_unleashed || interrupts_enabled());
1676 }
1677 #endif	/* DEBUG */
1678