xref: /illumos-gate/usr/src/uts/sun4/io/trapstat.c (revision bea83d026ee1bd1b2a2419e1d0232f107a5d7d9b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/systm.h>
29 #include <sys/conf.h>
30 #include <sys/stat.h>
31 #include <sys/ddi.h>
32 #include <sys/sunddi.h>
33 #include <sys/modctl.h>
34 #include <sys/cpu_module.h>
35 #include <vm/hat_sfmmu.h>
36 #include <vm/seg_kmem.h>
37 #include <vm/seg_kpm.h>
38 #include <vm/vm_dep.h>
39 #include <sys/machsystm.h>
40 #include <sys/machasi.h>
41 #include <sys/sysmacros.h>
42 #include <sys/callb.h>
43 #include <sys/archsystm.h>
44 #include <sys/trapstat.h>
45 #ifdef sun4v
46 #include <sys/hypervisor_api.h>
47 #endif
48 #ifndef sun4v
49 #include <sys/pghw.h>
50 #endif
51 
52 /* BEGIN CSTYLED */
53 /*
54  * trapstat:  Trap Statistics through Dynamic Trap Table Interposition
55  * -------------------------------------------------------------------
56  *
57  * Motivation and Overview
58  *
59  * Despite being a fundamental indicator of system behavior, there has
60  * historically been very little insight provided into the frequency and cost
61  * of machine-specific traps.  The lack of insight has been especially acute
62  * on UltraSPARC microprocessors:  because these microprocessors handle TLB
63  * misses as software traps, the frequency and duration of traps play a
64  * decisive role in the performance of the memory system.  As applications have
65  * increasingly outstripped TLB reach, this has become increasingly true.
66  *
67  * Part of the difficulty of observing trap behavior is that the trap handlers
68  * are so frequently called (e.g. millions of times per second) that any
69  * permanently enabled instrumentation would induce an unacceptable performance
70  * degradation.  Thus, it is a constraint on any trap observability
71  * infrastructure that it have no probe effect when not explicitly enabled.
72  *
73  * The basic idea, then, is to create an interposing trap table in which each
74  * entry increments a per-trap, in-memory counter and then jumps to the actual,
75  * underlying trap table entry.  To enable trapstat, we atomically write to the
76  * trap base address (%tba) register to point to our interposing trap table.
77  * (Note that per-CPU statistics fall out by creating a different trap table
78  * for each CPU.)
79  *
80  * Implementation Details
81  *
82  * While the idea is straight-forward, a nuance of SPARC V9 slightly
83  * complicates the implementation.  Unlike its predecessors, SPARC V9 supports
84  * the notion of nested traps.  The trap level is kept in the TL register:
85  * during normal operation it is 0; when a trap is taken, the TL register is
86  * incremented by 1.  To aid system software, SPARC V9 breaks the trap table
87  * into two halves:  the lower half contains the trap handlers for traps taken
88  * when TL is 0; the upper half contains the trap handlers for traps taken
89  * when TL is greater than 0.  Each half is further subdivided into two
90  * subsequent halves:  the lower half contains the trap handlers for traps
91  * other than those induced by the trap instruction (Tcc variants); the upper
92  * half contains the trap handlers for traps induced by the trap instruction.
93  * This gives a total of four ranges, with each range containing 256 traps:
94  *
95  *       +--------------------------------+- 3ff
96  *       |                                |   .
97  *       |     Trap instruction, TL>0     |   .
98  *       |                                |   .
99  *       |- - - - - - - - - - - - - - - - +- 300
100  *       |- - - - - - - - - - - - - - - - +- 2ff
101  *       |                                |   .
102  *       |   Non-trap instruction, TL>0   |   .
103  *       |                                |   .
104  *       |- - - - - - - - - - - - - - - - +- 200
105  *       |- - - - - - - - - - - - - - - - +- 1ff
106  *       |                                |   .
107  *       |     Trap instruction, TL=0     |   .
108  *       |                                |   .
109  *       |- - - - - - - - - - - - - - - - +- 100
110  *       |- - - - - - - - - - - - - - - - +- 0ff
111  *       |                                |   .
112  *       |   Non-trap instruction, TL=0   |   .
113  *       |                                |   .
114  *       +--------------------------------+- 000
115  *
116  *
117  * Solaris, however, doesn't have reason to support trap instructions when
118  * TL>0 (only privileged code may execute at TL>0; not supporting this only
119  * constrains our own implementation).  The trap table actually looks like:
120  *
121  *       +--------------------------------+- 2ff
122  *       |                                |   .
123  *       |   Non-trap instruction, TL>0   |   .
124  *       |                                |   .
125  *       |- - - - - - - - - - - - - - - - +- 200
126  *       |- - - - - - - - - - - - - - - - +- 1ff
127  *       |                                |   .
128  *       |     Trap instruction, TL=0     |   .
129  *       |                                |   .
130  *       |- - - - - - - - - - - - - - - - +- 100
131  *       |- - - - - - - - - - - - - - - - +- 0ff
132  *       |                                |   .
133  *       |   Non-trap instruction, TL=0   |   .
134  *       |                                |   .
135  *       +--------------------------------+- 000
136  *
137  * Putatively to aid system software, SPARC V9 has the notion of multiple
138  * sets of global registers.  UltraSPARC defines four sets of global
139  * registers:
140  *
141  *    Normal Globals
142  *    Alternate Globals (AGs)
143  *    MMU Globals (MGs)
144  *    Interrupt Globals (IGs)
145  *
146  * The set of globals in use is controlled by bits in PSTATE; when TL is 0
147  * (and PSTATE has not been otherwise explicitly modified), the Normal Globals
148  * are in use.  When a trap is issued, PSTATE is modified to point to a set of
149  * globals corresponding to the trap type.  Most traps correspond to the
150  * Alternate Globals, with a minority corresponding to the MMU Globals, and
151  * only the interrupt-vector trap (vector 0x60) corresponding to the Interrupt
152  * Globals.  (The complete mapping can be found in the UltraSPARC I&II User's
153  * Manual.)
154  *
155  * Note that the sets of globals are per trap _type_, not per trap _level_.
156  * Thus, when executing a TL>0 trap handler, one may not have registers
157  * available (for example, both trap-instruction traps and spill traps execute
158  * on the alternate globals; if a trap-instruction trap induces a window spill,
159  * the window spill handler has no available globals).  For trapstat, this is
160  * problematic:  a register is required to transfer control from one arbitrary
161  * location (in the interposing trap table) to another (in the actual trap
162  * table).
163  *
164  * We solve this problem by exploiting the trap table's location at the bottom
165  * of valid kernel memory (i.e. at KERNELBASE).  We locate the interposing trap
166  * tables just below KERNELBASE -- thereby allowing us to use a branch-always
167  * instruction (ba) instead of a jump instruction (jmp) to transfer control
168  * from the TL>0 entries in the interposing trap table to the TL>0 entries in
169  * the actual trap table.  (N.B. while this allows trap table interposition to
170  * work, it necessarily limits trapstat to only recording information about
171  * TL=0 traps -- there is no way to increment a counter without using a
172  * register.)  Diagrammatically:
173  *
174  *  Actual trap table:
175  *
176  *       +--------------------------------+- 2ff
177  *       |                                |   .
178  *       |   Non-trap instruction, TL>0   |   .   <-----------------------+
179  *       |                                |   .   <-----------------------|-+
180  *       |- - - - - - - - - - - - - - - - +- 200  <-----------------------|-|-+
181  *       |- - - - - - - - - - - - - - - - +- 1ff                          | | |
182  *       |                                |   .                           | | |
183  *       |     Trap instruction, TL=0     |   .   <-----------------+     | | |
184  *       |                                |   .   <-----------------|-+   | | |
185  *       |- - - - - - - - - - - - - - - - +- 100  <-----------------|-|-+ | | |
186  *       |- - - - - - - - - - - - - - - - +- 0ff                    | | | | | |
187  *       |                                |   .                     | | | | | |
188  *       |   Non-trap instruction, TL=0   |   .   <-----------+     | | | | | |
189  *       |                                |   .   <-----------|-+   | | | | | |
190  *       +--------------------------------+- 000  <-----------|-|-+ | | | | | |
191  *        KERNELBASE                                          | | | | | | | | |
192  *                                                            | | | | | | | | |
193  *                                                            | | | | | | | | |
194  *  Interposing trap table:                                   | | | | | | | | |
195  *                                                            | | | | | | | | |
196  *       +--------------------------------+- 2ff              | | | | | | | | |
197  *       |  ...                           |   .               | | | | | | | | |
198  *       |  ...                           |   .               | | | | | | | | |
199  *       |  ...                           |   .               | | | | | | | | |
200  *       |- - - - - - - - - - - - - - - - +- 203              | | | | | | | | |
201  *       |  ba,a                          |      -------------|-|-|-|-|-|-+ | |
202  *       |- - - - - - - - - - - - - - - - +- 202              | | | | | |   | |
203  *       |  ba,a                          |      -------------|-|-|-|-|-|---+ |
204  *       |- - - - - - - - - - - - - - - - +- 201              | | | | | |     |
205  *       |  ba,a                          |      -------------|-|-|-|-|-|-----+
206  *       |- - - - - - - - - - - - - - - - +- 200              | | | | | |
207  *       |  ...                           |   .               | | | | | |
208  *       |  ...                           |   .               | | | | | |
209  *       |  ...                           |   .               | | | | | |
210  *       |- - - - - - - - - - - - - - - - +- 103              | | | | | |
211  *       |  (Increment counter)           |                   | | | | | |
212  *       |  ba,a                          |      -------------------+ | |
213  *       |- - - - - - - - - - - - - - - - +- 102              | | |   | |
214  *       |  (Increment counter)           |                   | | |   | |
215  *       |  ba,a                          |      ---------------------+ |
216  *       |- - - - - - - - - - - - - - - - +- 101              | | |     |
217  *       |  (Increment counter)           |                   | | |     |
218  *       |  ba,a                          |      -----------------------+
219  *       |- - - - - - - - - - - - - - - - +- 100              | | |
220  *       |  ...                           |   .               | | |
221  *       |  ...                           |   .               | | |
222  *       |  ...                           |   .               | | |
223  *       |- - - - - - - - - - - - - - - - +- 003              | | |
224  *       |  (Increment counter)           |                   | | |
225  *       |  ba,a                          |      -------------+ | |
226  *       |- - - - - - - - - - - - - - - - +- 002                | |
227  *       |  (Increment counter)           |                     | |
228  *       |  ba,a                          |      ---------------+ |
229  *       |- - - - - - - - - - - - - - - - +- 001                  |
230  *       |  (Increment counter)           |                       |
231  *       |  ba,a                          |      -----------------+
232  *       +--------------------------------+- 000
233  *        KERNELBASE - tstat_total_size
234  *
235  * tstat_total_size is the number of pages required for each trap table.  It
236  * must be true that KERNELBASE - tstat_total_size is less than the maximum
237  * branch displacement; if each CPU were to consume a disjoint virtual range
238  * below KERNELBASE for its trap table, we could support at most
239  * (maximum_branch_displacement / tstat_total_size) CPUs.  The maximum branch
240  * displacement for Bicc variants is just under eight megabytes, and (because
241  * the %tba must be 32K aligned), tstat_total_size must be at least 32K; if
242  * each CPU were to consume a disjoint virtual range, we would have an
243  * unacceptably low upper bound of 256 CPUs.
244  *
245  * While there are tricks that one could use to address this constraint (e.g.,
246  * creating trampolines every maximum_branch_displacement bytes), we instead
247  * solve this by not permitting each CPU to consume a disjoint virtual range.
248  * Rather, we have each CPU's interposing trap table use the _same_ virtual
249  * range, but we back the trap tables with disjoint physical memory.  Normally,
250  * such one-to-many virtual-to-physical mappings are illegal; this is
251  * permissible here only because the pages for the interposing trap table are
252  * necessarily locked in the TLB.  (The CPUs thus never have the opportunity to
253  * discover that they have conflicting translations.)
254  *
255  * On CMT architectures in which CPUs can share MMUs, the above trick will not
256  * work: two CPUs that share an MMU cannot have the same virtual address map
257  * to disjoint physical pages.  On these architectures, any CPUs sharing the
258  * same MMU must consume a disjoint 32K virtual address range -- limiting the
259  * number of CPUs sharing an MMU on these architectures to 256 due to the
260  * branch displacement limitation described above.  On the sun4v architecture,
261  * there is a further limitation: a guest may not have more than eight locked
262  * TLB entries per MMU.  To allow operation under this restriction, the
263  * interposing trap table and the trap statistics are each accessed through
264  * a single 4M TLB entry.  This limits the footprint to two locked entries
265  * (one for the I-TLB and one for the D-TLB), but further restricts the number
266  * of CPUs to 128 per MMU.  However, support for more than 128 CPUs can easily
267  * be added via a hybrid scheme, where the same 4M virtual address is used
268  * on different MMUs.
269  *
270  * On sun4v architecture, we currently don't use hybrid scheme as it imposes
271  * additional restriction on live migration and transparent CPU replacement.
272  * Instead, we increase the number of supported CPUs by reducing the virtual
273  * address space requirements per CPU via shared interposing trap table as
274  * follows:
275  *
276  *                                          Offset (within 4MB page)
277  *       +------------------------------------+- 0x400000
278  *       |  CPU 507 trap statistics (8KB)     |   .
279  *       |- - - - - - - - - - - - - - - - - - +- 0x3fe000
280  *       |                                    |
281  *       |   ...                              |
282  *       |                                    |
283  *       |- - - - - - - - - - - - - - - - - - +- 0x00c000
284  *       |  CPU 1 trap statistics (8KB)       |   .
285  *       |- - - - - - - - - - - - - - - - - - +- 0x00a000
286  *       |  CPU 0 trap statistics (8KB)       |   .
287  *       |- - - - - - - - - - - - - - - - - - +- 0x008000
288  *       |  Shared trap handler continuation  |   .
289  *       |- - - - - - - - - - - - - - - - - - +- 0x006000
290  *       |  Non-trap instruction, TL>0        |   .
291  *       |- - - - - - - - - - - - - - - - - - +- 0x004000
292  *       |  Trap instruction, TL=0            |   .
293  *       |- - - - - - - - - - - - - - - - - - +- 0x002000
294  *       |  Non-trap instruction, TL=0        |   .
295  *       +------------------------------------+- 0x000000
296  *
297  * Note that each CPU has its own 8K space for its trap statistics but
298  * shares the same interposing trap handlers.  Interposing trap handlers
299  * use the CPU ID to determine the location of per CPU trap statistics
300  * area dynamically. This increases the interposing trap handler overhead,
301  * but is acceptable as it allows us to support up to 508 CPUs with one
302  * 4MB page on sun4v architecture. Support for additional CPUs can be
303  * added via hybrid scheme as mentioned earlier.
304  *
305  * TLB Statistics
306  *
307  * Because TLB misses are an important component of system performance, we wish
308  * to know much more about these traps than simply the number received.
309  * Specifically, we wish to know:
310  *
311  *  (a)	The amount of time spent executing the TLB miss handler
312  *  (b)	TLB misses versus TSB misses
313  *  (c) Kernel-level misses versus user-level misses
314  *  (d) Misses per pagesize
315  *
316  * TLB Statistics: Time Spent Executing
317  *
318  * To accurately determine the amount of time spent executing the TLB miss
319  * handler, one must get a timestamp on trap entry and trap exit, subtract the
320  * latter from the former, and add the result to an accumulating count.
321  * Consider flow of control during normal TLB miss processing (where "ldx
322  * [%g2], %g2" is an arbitrary TLB-missing instruction):
323  *
324  * + - - - - - - - -+
325  * :                :
326  * : ldx [%g2], %g2 :<-------------------------------------------------------+
327  * :                :              Return from trap:                         |
328  * + - - - - - - - -+                TL <- TL - 1 (0)                        |
329  *	  |                          %pc <- TSTATE[TL].TPC (address of load) |
330  *	  | TLB miss:                                                        |
331  *        |   TL <- TL + 1 (1)                                               |
332  *        |   %pc <- TLB-miss-trap-handler                                   |
333  *        |                                                                  |
334  *        v                                                                  |
335  * + - - - - - - - - - - - - - - - +                                         |
336  * :                               :                                         |
337  * : Lookup VA in TSB              :                                         |
338  * : If (hit)                      :                                         |
339  * :     Fill TLB                  :                                         |
340  * : Else                          :                                         |
341  * :     Lookup VA (hme hash table :                                         |
342  * :                or segkpm)     :                                         |
343  * :     Fill TLB                  :                                         |
344  * : Endif                         :                                         |
345  * : Issue "retry"  ---------------------------------------------------------+
346  * :                               :
347  * + - - - - - - - - - - - - - - - +
348  *  TLB-miss-trap-handler
349  *
350  *
351  * As the above diagram indicates, interposing on the trap table allows one
352  * only to determine a timestamp on trap _entry_:  when the TLB miss handler
353  * has completed filling the TLB, a "retry" will be issued, and control will
354  * transfer immediately back to the missing %pc.
355  *
356  * To obtain a timestamp on trap exit, we must then somehow interpose between
357  * the "retry" and the subsequent control transfer to the TLB-missing
358  * instruction.  To do this, we _push_ a trap level.  The basic idea is to
359  * spoof a TLB miss by raising TL, setting the %tpc to be within text
360  * controlled by trapstat (the "TLB return entry") and branching to the
361  * underlying TLB miss handler.  When the TLB miss handler issues its "retry",
362  * control will transfer not to the TLB-missing instruction, but rather to the
363  * TLB return entry.  This code can then obtain a timestamp, and issue its own
364  * "retry" -- thereby correctly returning to the TLB-missing instruction.
365  * Here is the above TLB miss flow control diagram modified to reflect
366  * trapstat's operation:
367  *
368  * + - - - - - - - -+
369  * :                :
370  * : ldx [%g2], %g2 :<-------------------------------------------------------+
371  * :                :             Return from trap:                          |
372  * + - - - - - - - -+               TL <- TL - 1 (0)                         |
373  *	  |                         %pc <- TSTATE[TL].TPC (address of load)  |
374  *	  | TLB miss:                                                        |
375  *        |   TL <- TL + 1 (1)                                               |
376  *        |   %pc <- TLB-miss-trap-handler (trapstat)                        |
377  *        |                                                                  |
378  *        v                                    TLB-return-entry (trapstat)   |
379  * + - - - - - - - - - - - - - - - - - - +    + - - - - - - - - - - - - - +  |
380  * :                                     :    :                           :  |
381  * : Record timestamp                    :    : Record timestamp          :  |
382  * : TL <- 2                             :    : Take timestamp difference :  |
383  * : TSTATE[1].TPC <- TLB-return-entry   :    : Add to running total      :  |
384  * : ba,a TLB-miss-trap-handler -----------+  : Issue "retry"  --------------+
385  * :                                     : |  :                           :
386  * + - - - - - - - - - - - - - - - - - - + |  + - - - - - - - - - - - - - +
387  *  TLB-miss-trap-handler	           |                  ^
388  *  (trapstat)                             |                  |
389  *                                         |                  |
390  *                                         |                  |
391  *                 +-----------------------+                  |
392  *                 |                                          |
393  *                 |                                          |
394  *                 v                                          |
395  * + - - - - - - - - - - - - - - - +                          |
396  * :                               :                          |
397  * : Lookup VA in TSB              :                          |
398  * : If (hit)                      :                          |
399  * :     Fill TLB                  :                          |
400  * : Else                          :                          |
401  * :     Lookup VA (hme hash table :                          |
402  * :                or segkpm)     :                          |
403  * :     Fill TLB                  :                          |
404  * : Endif                         :                          |
405  * : Issue "retry"  ------------------------------------------+
406  * :                               : Return from trap:
407  * + - - - - - - - - - - - - - - - +   TL <- TL - 1 (1)
408  *  TLB-miss-trap-handler              %pc <- TSTATE[TL].TPC (TLB-return-entry)
409  *
410  *
411  * A final subterfuge is required to complete our artifice:  if we miss in
412  * the TLB, the TSB _and_ the subsequent hash or segkpm lookup (that is, if
413  * there is no valid translation for the TLB-missing address), common system
414  * software will need to accurately determine the %tpc as part of its page
415  * fault handling. We therefore modify the kernel to check the %tpc in this
416  * case: if the %tpc falls within the VA range controlled by trapstat and
417  * the TL is 2, TL is simply lowered back to 1 (this check is implemented
418  * by the TSTAT_CHECK_TL1 macro).  Lowering TL to 1 has the effect of
419  * discarding the state pushed by trapstat.
420  *
421  * TLB Statistics: TLB Misses versus TSB Misses
422  *
423  * Distinguishing TLB misses from TSB misses requires further interposition
424  * on the TLB miss handler:  we cannot know a priori or a posteriori if a
425  * given VA will or has hit in the TSB.
426  *
427  * We achieve this distinction by adding a second TLB return entry almost
428  * identical to the first -- differing only in the address to which it
429  * stores its results.  We then modify the TLB miss handlers of the kernel
430  * such that they check the %tpc when they determine that a TLB miss has
431  * subsequently missed in the TSB:  if the %tpc lies within trapstat's VA
432  * range and TL is 2 (that is, if trapstat is running), the TLB miss handler
433  * _increments_ the %tpc by the size of the TLB return entry.  The ensuing
434  * "retry" will thus transfer control to the second TLB return entry, and
435  * the time spent in the handler will be accumulated in a memory location
436  * specific to TSB misses.
437  *
438  * N.B.:  To minimize the amount of knowledge the kernel must have of trapstat,
439  * we do not allow the kernel to hard-code the size of the TLB return entry.
440  * Rather, the actual tsbmiss handler executes a known instruction at the
441  * corresponding tsbmiss patch points (see the tstat_tsbmiss_patch_table) with
442  * the %tpc in %g7:  when trapstat is not running, these points contain the
443  * harmless TSTAT_TSBMISS_INSTR instruction ("add %g7, 0, %g7"). Before
444  * running, trapstat modifies the instructions at these patch points such
445  * that the simm13 equals the size of the TLB return entry.
446  *
447  * TLB Statistics: Kernel-level Misses versus User-level Misses
448  *
449  * Differentiating user-level misses from kernel-level misses employs a
450  * similar technique, but is simplified by the ability to distinguish a
451  * user-level miss from a kernel-level miss a priori by reading the context
452  * register:  we implement kernel-/user-level differentiation by again doubling
453  * the number of TLB return entries, and setting the %tpc to the appropriate
454  * TLB return entry in trapstat's TLB miss handler.  Together with the doubling
455  * of entries required for TLB-miss/TSB-miss differentiation, this yields a
456  * total of four TLB return entries:
457  *
458  *	Level		TSB hit?	Structure member
459  *	------------------------------------------------------------
460  *	Kernel		Yes		tstat_tlbret_t.ttlbr_ktlb
461  *	Kernel		No		tstat_tlbret_t.ttlbr_ktsb
462  *	User		Yes		tstat_tlbret_t.ttlbr_utlb
463  *	User		No		tstat_tlbret_t.ttlbr_utsb
464  *
465  * TLB Statistics: Misses per Pagesize
466  *
467  * As with the TLB-/TSB-miss differentiation, we have no way of determining
468  * pagesize a priori.  This is therefore implemented by mandating a new rule:
469  * whenever the kernel fills the TLB in its TLB miss handler, the TTE
470  * corresponding to the TLB-missing VA must be in %g5 when the handler
471  * executes its "retry".  This allows the TLB return entry to determine
472  * pagesize by simply looking at the pagesize field in the TTE stored in
473  * %g5.
474  *
475  * TLB Statistics: Probe Effect
476  *
477  * As one might imagine, gathering TLB statistics by pushing a trap level
478  * induces significant probe effect.  To account for this probe effect,
479  * trapstat attempts to observe it by executing a code sequence with a known
480  * number of TLB misses both before and after interposing on the trap table.
481  * This allows trapstat to determine a per-trap probe effect which can then be
482  * factored into the "%tim" fields of the trapstat command.
483  *
484  * Note that on sun4v platforms, TLB misses are normally handled by the
485  * hypervisor or the hardware TSB walker. Thus no fast MMU miss information
486  * is reported for normal operation. However, when trapstat is invoked
487  * with -t or -T option to collect detailed TLB statistics, kernel takes
488  * over TLB miss handling. This results in significantly more overhead
489  * and TLB statistics may not be as accurate as on sun4u platforms.
490  * On some processors, hypervisor or hardware may provide a low overhead
491  * interface to collect TSB hit statistics. This support is exposed via
492  * a well defined CPU module interface (cpu_trapstat_conf to enable this
493  * interface and cpu_trapstat_data to get detailed TSB hit statistics).
494  * In this scenario, TSB miss statistics is collected by intercepting the
495  * IMMU_miss and DMMU_miss traps using above mentioned trap interposition
496  * approach.
497  *
498  * Locking
499  *
500  * The implementation uses two locks:  tstat_lock (a local lock) and the global
501  * cpu_lock.  tstat_lock is used to assure trapstat's consistency in the
502  * presence of multithreaded /dev/trapstat consumers (while as of this writing
503  * the only consumer of /dev/trapstat is single threaded, it is obviously
504  * necessary to correctly support multithreaded access).  cpu_lock is held
505  * whenever CPUs are being manipulated directly, to prevent them from
506  * disappearing in the process.  Because trapstat's DR callback
507  * (trapstat_cpu_setup()) must grab tstat_lock and is called with cpu_lock
508  * held, the lock ordering is necessarily cpu_lock before tstat_lock.
509  *
510  */
511 /* END CSTYLED */
512 
513 static dev_info_t	*tstat_devi;	/* saved in xxattach() for xxinfo() */
514 static int		tstat_open;	/* set if driver is open */
515 static kmutex_t		tstat_lock;	/* serialize access */
516 static vmem_t		*tstat_arena;	/* arena for TLB-locked pages */
517 static tstat_percpu_t	*tstat_percpu;	/* per-CPU data */
518 static int		tstat_running;	/* set if trapstat is running */
519 static tstat_data_t	*tstat_buffer;	/* staging buffer for outgoing data */
520 static int		tstat_options;	/* bit-wise indication of options */
521 static int		*tstat_enabled;	/* map of enabled trap entries */
522 static int		tstat_tsbmiss_patched; /* tsbmiss patch flag */
523 static callb_id_t	tstat_cprcb;	/* CPR callback */
524 static char		*tstat_probe_area; /* VA range used for probe effect */
525 static caddr_t		tstat_probe_phys; /* physical to back above VA */
526 static hrtime_t		tstat_probe_time; /* time spent on probe effect */
527 static hrtime_t		tstat_probe_before[TSTAT_PROBE_NLAPS];
528 static hrtime_t		tstat_probe_after[TSTAT_PROBE_NLAPS];
529 static uint_t		tstat_pgszs;		/* # of kernel page sizes */
530 static uint_t		tstat_user_pgszs;	/* # of user page sizes */
531 
532 /*
533  * sizeof tstat_data_t + pgsz data for the kernel.  For simplicity's sake, when
534  * we collect data, we do it based upon szc, but when we report data back to
535  * userland, we have to do it based upon the userszc which may not match.
536  * So, these two variables are for internal use and exported use respectively.
537  */
538 static size_t		tstat_data_t_size;
539 static size_t		tstat_data_t_exported_size;
540 
541 #ifndef sun4v
542 
543 static size_t		tstat_data_pages;  /* number of pages of tstat data */
544 static size_t		tstat_data_size;   /* tstat data size in bytes */
545 static size_t		tstat_total_pages; /* #data pages + #instr pages */
546 static size_t		tstat_total_size;  /* tstat data size + instr size */
547 
548 #else /* sun4v */
549 
550 static caddr_t		tstat_va;	/* VA of memory reserved for TBA */
551 static pfn_t		tstat_pfn;	/* PFN of memory reserved for TBA */
552 static boolean_t	tstat_fast_tlbstat = B_FALSE;
553 static int		tstat_traptab_initialized;
554 
555 #endif /* sun4v */
556 
557 /*
558  * In the above block comment, see "TLB Statistics: TLB Misses versus
559  * TSB Misses" for an explanation of the tsbmiss patch points.
560  */
561 extern uint32_t		tsbmiss_trapstat_patch_point;
562 extern uint32_t		tsbmiss_trapstat_patch_point_kpm;
563 extern uint32_t		tsbmiss_trapstat_patch_point_kpm_small;
564 
565 /*
566  * Trapstat tsbmiss patch table
567  */
568 tstat_tsbmiss_patch_entry_t tstat_tsbmiss_patch_table[] = {
569 	{(uint32_t *)&tsbmiss_trapstat_patch_point, 0},
570 	{(uint32_t *)&tsbmiss_trapstat_patch_point_kpm, 0},
571 	{(uint32_t *)&tsbmiss_trapstat_patch_point_kpm_small, 0},
572 	{(uint32_t *)NULL, 0}
573 };
574 
575 /*
576  * We define some general SPARC-specific constants to allow more readable
577  * relocations.
578  */
579 #define	NOP	0x01000000
580 #define	HI22(v) ((uint32_t)(v) >> 10)
581 #define	LO10(v) ((uint32_t)(v) & 0x3ff)
582 #define	LO12(v) ((uint32_t)(v) & 0xfff)
583 #define	DISP22(from, to) \
584 	((((uintptr_t)(to) - (uintptr_t)(from)) >> 2) & 0x3fffff)
585 #define	ASI(asi)	((asi) << 5)
586 
587 /*
588  * The interposing trap table must be locked in the I-TLB, and any data
589  * referred to in the interposing trap handler must be locked in the D-TLB.
590  * This function locks these pages in the appropriate TLBs by creating TTEs
591  * from whole cloth, and manually loading them into the TLB.  This function is
592  * called from cross call context.
593  *
594  * On sun4v platforms, we use 4M page size mappings to minimize the number
595  * of locked down entries (i.e. permanent mappings). Each CPU uses a
596  * reserved portion of that 4M page for its TBA and data.
597  */
598 static void
599 trapstat_load_tlb(void)
600 {
601 #ifndef sun4v
602 	int i;
603 #else
604 	uint64_t ret;
605 #endif
606 	tte_t tte;
607 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
608 	caddr_t va = tcpu->tcpu_vabase;
609 
610 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
611 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
612 
613 #ifndef sun4v
614 	for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) {
615 		tte.tte_inthi = TTE_VALID_INT | TTE_SZ_INT(TTE8K) |
616 		    TTE_PFN_INTHI(tcpu->tcpu_pfn[i]);
617 		if (i < TSTAT_INSTR_PAGES) {
618 			tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) |
619 			    TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT;
620 			sfmmu_itlb_ld_kva(va, &tte);
621 		} else {
622 			tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) |
623 			    TTE_LCK_INT | TTE_CP_INT | TTE_CV_INT |
624 			    TTE_PRIV_INT | TTE_HWWR_INT;
625 			sfmmu_dtlb_ld_kva(va, &tte);
626 		}
627 	}
628 #else /* sun4v */
629 	tte.tte_inthi = TTE_VALID_INT | TTE_PFN_INTHI(tstat_pfn);
630 	tte.tte_intlo = TTE_PFN_INTLO(tstat_pfn) | TTE_CP_INT |
631 	    TTE_CV_INT | TTE_PRIV_INT | TTE_HWWR_INT |
632 	    TTE_SZ_INTLO(TTE4M);
633 	ret = hv_mmu_map_perm_addr(va, KCONTEXT, *(uint64_t *)&tte,
634 	    MAP_ITLB | MAP_DTLB);
635 
636 	if (ret != H_EOK)
637 		cmn_err(CE_PANIC, "trapstat: cannot map new TBA "
638 		    "for cpu %d  (error: 0x%lx)", CPU->cpu_id, ret);
639 #endif /* sun4v */
640 }
641 
642 /*
643  * As mentioned in the "TLB Statistics: TLB Misses versus TSB Misses" section
644  * of the block comment, TLB misses are differentiated from TSB misses in
645  * part by hot-patching the instructions at the tsbmiss patch points (see
646  * tstat_tsbmiss_patch_table). This routine is used both to initially patch
647  * the instructions, and to patch them back to their original values upon
648  * restoring the original trap table.
649  */
650 static void
651 trapstat_hotpatch()
652 {
653 	uint32_t instr;
654 	uint32_t simm13;
655 	tstat_tsbmiss_patch_entry_t *ep;
656 
657 	ASSERT(MUTEX_HELD(&tstat_lock));
658 
659 	if (!(tstat_options & TSTAT_OPT_TLBDATA))
660 		return;
661 
662 	if (!tstat_tsbmiss_patched) {
663 		/*
664 		 * We haven't patched the TSB paths; do so now.
665 		 */
666 		/*CONSTCOND*/
667 		ASSERT(offsetof(tstat_tlbret_t, ttlbr_ktsb) -
668 		    offsetof(tstat_tlbret_t, ttlbr_ktlb) ==
669 		    offsetof(tstat_tlbret_t, ttlbr_utsb) -
670 		    offsetof(tstat_tlbret_t, ttlbr_utlb));
671 
672 		simm13 = offsetof(tstat_tlbret_t, ttlbr_ktsb) -
673 		    offsetof(tstat_tlbret_t, ttlbr_ktlb);
674 
675 		for (ep = tstat_tsbmiss_patch_table; ep->tpe_addr; ep++) {
676 			ASSERT(ep->tpe_instr == 0);
677 			instr = ep->tpe_instr = *ep->tpe_addr;
678 
679 			/*
680 			 * Assert that the instruction we're about to patch is
681 			 * "add %g7, 0, %g7" (0x8e01e000).
682 			 */
683 			ASSERT(instr == TSTAT_TSBMISS_INSTR);
684 
685 			instr |= simm13;
686 			hot_patch_kernel_text((caddr_t)ep->tpe_addr,
687 			    instr, sizeof (instr));
688 		}
689 
690 		tstat_tsbmiss_patched = 1;
691 
692 	} else {
693 		/*
694 		 * Remove patches from the TSB paths.
695 		 */
696 		for (ep = tstat_tsbmiss_patch_table; ep->tpe_addr; ep++) {
697 			ASSERT(ep->tpe_instr == TSTAT_TSBMISS_INSTR);
698 			hot_patch_kernel_text((caddr_t)ep->tpe_addr,
699 			    ep->tpe_instr, sizeof (instr));
700 			ep->tpe_instr = 0;
701 		}
702 
703 		tstat_tsbmiss_patched = 0;
704 	}
705 }
706 
707 /*
708  * This is the routine executed to clock the performance of the trap table,
709  * executed both before and after interposing on the trap table to attempt to
710  * determine probe effect.  The probe effect is used to adjust the "%tim"
711  * fields of trapstat's -t and -T output; we only use TLB misses to clock the
712  * trap table.  We execute the inner loop (which is designed to exceed the
713  * TLB's reach) nlaps times, taking the best time as our time (thereby
714  * factoring out the effects of interrupts, cache misses or other perturbing
715  * events.
716  */
717 static hrtime_t
718 trapstat_probe_laps(int nlaps, hrtime_t *buf)
719 {
720 	int i, j = 0;
721 	hrtime_t ts, best = INT64_MAX;
722 
723 	while (nlaps--) {
724 		ts = rdtick();
725 
726 		for (i = 0; i < TSTAT_PROBE_SIZE; i += MMU_PAGESIZE)
727 			*((volatile char *)&tstat_probe_area[i]);
728 
729 		if ((ts = rdtick() - ts) < best)
730 			best = ts;
731 		buf[j++] = ts;
732 	}
733 
734 	return (best);
735 }
736 
737 /*
738  * This routine determines the probe effect by calling trapstat_probe_laps()
739  * both without and with the interposing trap table.  Note that this is
740  * called from a cross call on the desired CPU, and that it is called on
741  * every CPU (this is necessary because the probe effect may differ from
742  * one CPU to another).
743  */
744 static void
745 trapstat_probe()
746 {
747 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
748 	hrtime_t before, after;
749 
750 	if (!(tcpu->tcpu_flags & TSTAT_CPU_SELECTED))
751 		return;
752 
753 	if (tstat_probe_area == NULL || (tstat_options & TSTAT_OPT_NOGO))
754 		return;
755 
756 	/*
757 	 * We very much expect the %tba to be KERNELBASE; this is a
758 	 * precautionary measure to assure that trapstat doesn't melt the
759 	 * machine should the %tba point unexpectedly elsewhere.
760 	 */
761 	if (get_tba() != (caddr_t)KERNELBASE)
762 		return;
763 
764 	/*
765 	 * Preserve this CPU's data before destroying it by enabling the
766 	 * interposing trap table.  We can safely use tstat_buffer because
767 	 * the caller of the trapstat_probe() cross call is holding tstat_lock.
768 	 */
769 	bcopy(tcpu->tcpu_data, tstat_buffer, tstat_data_t_size);
770 
771 	tstat_probe_time = gethrtime();
772 
773 	before = trapstat_probe_laps(TSTAT_PROBE_NLAPS, tstat_probe_before);
774 	(void) set_tba(tcpu->tcpu_ibase);
775 
776 	after = trapstat_probe_laps(TSTAT_PROBE_NLAPS, tstat_probe_after);
777 	(void) set_tba((caddr_t)KERNELBASE);
778 
779 	tstat_probe_time = gethrtime() - tstat_probe_time;
780 
781 	bcopy(tstat_buffer, tcpu->tcpu_data, tstat_data_t_size);
782 	tcpu->tcpu_data->tdata_peffect = (after - before) / TSTAT_PROBE_NPAGES;
783 }
784 
785 static void
786 trapstat_probe_alloc()
787 {
788 	pfn_t pfn;
789 	caddr_t va;
790 	int i;
791 
792 	ASSERT(MUTEX_HELD(&tstat_lock));
793 	ASSERT(tstat_probe_area == NULL);
794 	ASSERT(tstat_probe_phys == NULL);
795 
796 	if (!(tstat_options & TSTAT_OPT_TLBDATA))
797 		return;
798 
799 	/*
800 	 * Grab some virtual from the heap arena.
801 	 */
802 	tstat_probe_area = vmem_alloc(heap_arena, TSTAT_PROBE_SIZE, VM_SLEEP);
803 	va = tstat_probe_area;
804 
805 	/*
806 	 * Grab a single physical page.
807 	 */
808 	tstat_probe_phys = vmem_alloc(tstat_arena, MMU_PAGESIZE, VM_SLEEP);
809 	pfn = hat_getpfnum(kas.a_hat, tstat_probe_phys);
810 
811 	/*
812 	 * Now set the translation for every page in our virtual range
813 	 * to be our allocated physical page.
814 	 */
815 	for (i = 0; i < TSTAT_PROBE_NPAGES; i++) {
816 		hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, PROT_READ,
817 		    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
818 		va += MMU_PAGESIZE;
819 	}
820 }
821 
822 static void
823 trapstat_probe_free()
824 {
825 	caddr_t va;
826 	int i;
827 
828 	ASSERT(MUTEX_HELD(&tstat_lock));
829 
830 	if ((va = tstat_probe_area) == NULL)
831 		return;
832 
833 	for (i = 0; i < TSTAT_PROBE_NPAGES; i++) {
834 		hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD_UNLOCK);
835 		va += MMU_PAGESIZE;
836 	}
837 
838 	vmem_free(tstat_arena, tstat_probe_phys, MMU_PAGESIZE);
839 	vmem_free(heap_arena, tstat_probe_area, TSTAT_PROBE_SIZE);
840 
841 	tstat_probe_phys = NULL;
842 	tstat_probe_area = NULL;
843 }
844 
845 /*
846  * This routine actually enables a CPU by setting its %tba to be the
847  * CPU's interposing trap table.  It is called out of cross call context.
848  */
849 static void
850 trapstat_enable()
851 {
852 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
853 
854 	if (!(tcpu->tcpu_flags & TSTAT_CPU_SELECTED))
855 		return;
856 
857 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
858 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
859 
860 	if (get_tba() != (caddr_t)KERNELBASE)
861 		return;
862 
863 	if (!(tstat_options & TSTAT_OPT_NOGO))
864 		(void) set_tba(tcpu->tcpu_ibase);
865 	tcpu->tcpu_flags |= TSTAT_CPU_ENABLED;
866 #ifdef sun4v
867 	if ((tstat_options & TSTAT_OPT_TLBDATA) &&
868 	    !(tstat_options & TSTAT_OPT_NOGO)) {
869 		if (tstat_fast_tlbstat) {
870 			/*
871 			 * Invoke processor specific interface to enable
872 			 * collection of TSB hit statistics.
873 			 */
874 			cpu_trapstat_conf(CPU_TSTATCONF_ENABLE);
875 		} else {
876 			/*
877 			 * Collect TLB miss statistics by taking over
878 			 * TLB miss handling from the hypervisor. This
879 			 * is done by telling the hypervisor that there
880 			 * is no TSB configured. Also set TSTAT_TLB_STATS
881 			 * flag so that no user TSB is configured during
882 			 * context switch time.
883 			 */
884 			cpu_t *cp = CPU;
885 
886 			cp->cpu_m.cpu_tstat_flags |= TSTAT_TLB_STATS;
887 			(void) hv_set_ctx0(NULL, NULL);
888 			(void) hv_set_ctxnon0(NULL, NULL);
889 		}
890 	}
891 #endif
892 }
893 
894 /*
895  * This routine disables a CPU (vis a vis trapstat) by setting its %tba to be
896  * the actual, underlying trap table.  It is called out of cross call context.
897  */
898 static void
899 trapstat_disable()
900 {
901 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
902 
903 	if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED))
904 		return;
905 
906 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
907 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
908 
909 	if (!(tstat_options & TSTAT_OPT_NOGO))
910 		(void) set_tba((caddr_t)KERNELBASE);
911 
912 	tcpu->tcpu_flags &= ~TSTAT_CPU_ENABLED;
913 
914 #ifdef sun4v
915 	if ((tstat_options & TSTAT_OPT_TLBDATA) &&
916 	    !(tstat_options & TSTAT_OPT_NOGO)) {
917 		if (tstat_fast_tlbstat) {
918 			/*
919 			 * Invoke processor specific interface to disable
920 			 * collection of TSB hit statistics on each processor.
921 			 */
922 			cpu_trapstat_conf(CPU_TSTATCONF_DISABLE);
923 		} else {
924 			/*
925 			 * As part of collecting TLB miss statistics, we took
926 			 * over TLB miss handling from the hypervisor by
927 			 * telling the hypervisor that NO TSB is configured.
928 			 * We need to restore that by communicating proper
929 			 * kernel/user TSB information so that TLB misses
930 			 * can be handled by the hypervisor or the hardware
931 			 * more efficiently.
932 			 *
933 			 * We restore kernel TSB information right away.
934 			 * However, to minimize any locking dependency, we
935 			 * don't restore user TSB information right away.
936 			 * Instead, we simply clear the TSTAT_TLB_STATS flag
937 			 * so that the user TSB information is automatically
938 			 * restored on next context switch.
939 			 *
940 			 * Note that the call to restore kernel TSB information
941 			 * will normally not fail, unless wrong information is
942 			 * passed here. In that scenario, system will still
943 			 * continue to function properly with the exception of
944 			 * kernel handling all the TLB misses.
945 			 */
946 			struct hv_tsb_block *hvbp = &ksfmmup->sfmmu_hvblock;
947 			cpu_t *cp = CPU;
948 
949 			cp->cpu_m.cpu_tstat_flags &= ~TSTAT_TLB_STATS;
950 			(void) hv_set_ctx0(hvbp->hv_tsb_info_cnt,
951 			    hvbp->hv_tsb_info_pa);
952 		}
953 	}
954 #endif
955 }
956 
957 /*
958  * We use %tick as the time base when recording the time spent executing
959  * the trap handler.  %tick, however, is not necessarily kept in sync
960  * across CPUs (indeed, different CPUs may have different %tick frequencies).
961  * We therefore cross call onto a CPU to get a snapshot of its data to
962  * copy out; this is the routine executed out of that cross call.
963  */
964 static void
965 trapstat_snapshot()
966 {
967 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
968 	tstat_data_t *data = tcpu->tcpu_data;
969 
970 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
971 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
972 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ENABLED);
973 
974 	data->tdata_snapts = gethrtime();
975 	data->tdata_snaptick = rdtick();
976 	bcopy(data, tstat_buffer, tstat_data_t_size);
977 #ifdef sun4v
978 	/*
979 	 * Invoke processor specific interface to collect TSB hit
980 	 * statistics on each processor.
981 	 */
982 	if ((tstat_options & TSTAT_OPT_TLBDATA) && tstat_fast_tlbstat)
983 		cpu_trapstat_data((void *) tstat_buffer->tdata_pgsz,
984 		    tstat_pgszs);
985 #endif
986 }
987 
988 /*
989  * The TSTAT_RETENT_* constants define offsets in the TLB return entry.
990  * They are used only in trapstat_tlbretent() (below) and #undef'd
991  * immediately afterwards.  Any change to "retent" in trapstat_tlbretent()
992  * will likely require changes to these constants.
993  */
994 
995 #ifndef sun4v
996 #define	TSTAT_RETENT_STATHI	1
997 #define	TSTAT_RETENT_STATLO	2
998 #define	TSTAT_RETENT_SHIFT	11
999 #define	TSTAT_RETENT_COUNT_LD	13
1000 #define	TSTAT_RETENT_COUNT_ST	15
1001 #define	TSTAT_RETENT_TMPTSHI	16
1002 #define	TSTAT_RETENT_TMPTSLO	17
1003 #define	TSTAT_RETENT_TIME_LD	19
1004 #define	TSTAT_RETENT_TIME_ST	21
1005 #else /* sun4v */
1006 #define	TSTAT_RETENT_TDATASHFT	2
1007 #define	TSTAT_RETENT_STATHI	4
1008 #define	TSTAT_RETENT_STATLO	6
1009 #define	TSTAT_RETENT_SHIFT	9
1010 #define	TSTAT_RETENT_COUNT_LD	11
1011 #define	TSTAT_RETENT_COUNT_ST	13
1012 #define	TSTAT_RETENT_TMPTSHI	14
1013 #define	TSTAT_RETENT_TMPTSLO	16
1014 #define	TSTAT_RETENT_TIME_LD	18
1015 #define	TSTAT_RETENT_TIME_ST	20
1016 #endif /* sun4v */
1017 
1018 static void
1019 trapstat_tlbretent(tstat_percpu_t *tcpu, tstat_tlbretent_t *ret,
1020     tstat_missdata_t *data)
1021 {
1022 	uint32_t *ent = ret->ttlbrent_instr, shift;
1023 	uintptr_t base;
1024 #ifndef sun4v
1025 	uintptr_t tmptick = TSTAT_DATA_OFFS(tcpu, tdata_tmptick);
1026 #else
1027 	uintptr_t tmptick = TSTAT_CPU0_DATA_OFFS(tcpu, tdata_tmptick);
1028 #endif
1029 
1030 	/*
1031 	 * This is the entry executed upon return from the TLB/TSB miss
1032 	 * handler (i.e. the code interpositioned between the "retry" and
1033 	 * the actual return to the TLB-missing instruction).  Detail on its
1034 	 * theory of operation can be found in the "TLB Statistics" section
1035 	 * of the block comment.  Note that we expect the TTE just loaded
1036 	 * into the TLB to be in %g5; all other globals are available as
1037 	 * scratch.  Finally, note that the page size information in sun4v is
1038 	 * located in the lower bits of the TTE -- requiring us to have a
1039 	 * different return entry on sun4v.
1040 	 */
1041 	static const uint32_t retent[TSTAT_TLBRET_NINSTR] = {
1042 #ifndef sun4v
1043 	    0x87410000,		/* rd    %tick, %g3			*/
1044 	    0x03000000, 	/* sethi %hi(stat), %g1			*/
1045 	    0x82106000,		/* or    %g1, %lo(stat), %g1		*/
1046 	    0x89297001,		/* sllx  %g5, 1, %g4			*/
1047 	    0x8931303e,		/* srlx  %g4, 62, %g4			*/
1048 	    0x8531702e,		/* srlx  %g5, 46, %g2			*/
1049 	    0x8408a004,		/* and   %g2, 4, %g2			*/
1050 	    0x88110002,		/* or    %g4, %g2, %g4			*/
1051 	    0x80a12005,		/* cmp   %g4, 5				*/
1052 	    0x34400002,		/* bg,a,pn %icc, +8			*/
1053 	    0x88102004,		/* mov   4, %g4				*/
1054 	    0x89292000,		/* sll   %g4, shift, %g4		*/
1055 	    0x82004004,		/* add   %g1, %g4, %g1			*/
1056 	    0xc4586000,		/* ldx   [%g1 + tmiss_count], %g2	*/
1057 	    0x8400a001,		/* add   %g2, 1, %g2			*/
1058 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_count]	*/
1059 	    0x0d000000, 	/* sethi %hi(tdata_tmptick), %g6	*/
1060 	    0xc459a000, 	/* ldx   [%g6 + %lo(tdata_tmptick)], %g2 */
1061 	    0x8620c002,		/* sub   %g3, %g2, %g3			*/
1062 	    0xc4586000,		/* ldx   [%g1 + tmiss_time], %g2	*/
1063 	    0x84008003,		/* add   %g2, %g3, %g2			*/
1064 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_time]	*/
1065 	    0x83f00000		/* retry				*/
1066 #else /* sun4v */
1067 	    0x82102008,		/* mov   SCRATCHPAD_CPUID, %g1 		*/
1068 	    0xced84400,		/* ldxa  [%g1]ASI_SCRATCHPAD, %g7	*/
1069 	    0x8f29f000,		/* sllx  %g7, TSTAT_DATA_SHIFT, %g7	*/
1070 	    0x87410000,		/* rd    %tick, %g3			*/
1071 	    0x03000000, 	/* sethi %hi(stat), %g1			*/
1072 	    0x82004007,		/* add   %g1, %g7, %g1			*/
1073 	    0x82106000,		/* or    %g1, %lo(stat), %g1		*/
1074 	    0x8929703d,		/* sllx  %g5, 61, %g4			*/
1075 	    0x8931303d,		/* srlx  %g4, 61, %g4			*/
1076 	    0x89292000,		/* sll   %g4, shift, %g4		*/
1077 	    0x82004004,		/* add   %g1, %g4, %g1			*/
1078 	    0xc4586000,		/* ldx   [%g1 + tmiss_count], %g2	*/
1079 	    0x8400a001,		/* add   %g2, 1, %g2			*/
1080 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_count]	*/
1081 	    0x0d000000, 	/* sethi %hi(tdata_tmptick), %g6	*/
1082 	    0x8c018007,		/* add   %g6, %g7, %g6			*/
1083 	    0xc459a000, 	/* ldx   [%g6 + %lo(tdata_tmptick)], %g2 */
1084 	    0x8620c002,		/* sub   %g3, %g2, %g3			*/
1085 	    0xc4586000,		/* ldx   [%g1 + tmiss_time], %g2	*/
1086 	    0x84008003,		/* add   %g2, %g3, %g2			*/
1087 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_time]	*/
1088 	    0x83f00000		/* retry				*/
1089 #endif /* sun4v */
1090 	};
1091 
1092 	ASSERT(MUTEX_HELD(&tstat_lock));
1093 	/*CONSTCOND*/
1094 	ASSERT(offsetof(tstat_missdata_t, tmiss_count) <= LO10(-1));
1095 	/*CONSTCOND*/
1096 	ASSERT(offsetof(tstat_missdata_t, tmiss_time) <= LO10(-1));
1097 	/*CONSTCOND*/
1098 	ASSERT(!((sizeof (tstat_pgszdata_t) - 1) & sizeof (tstat_pgszdata_t)));
1099 
1100 	for (shift = 1; (1 << shift) != sizeof (tstat_pgszdata_t); shift++)
1101 		continue;
1102 
1103 	base = (uintptr_t)tcpu->tcpu_ibase + TSTAT_INSTR_SIZE +
1104 	    ((uintptr_t)data - (uintptr_t)tcpu->tcpu_data);
1105 
1106 	bcopy(retent, ent, sizeof (retent));
1107 
1108 #if defined(sun4v)
1109 	ent[TSTAT_RETENT_TDATASHFT] |= LO10((uintptr_t)TSTAT_DATA_SHIFT);
1110 #endif
1111 	ent[TSTAT_RETENT_STATHI] |= HI22(base);
1112 	ent[TSTAT_RETENT_STATLO] |= LO10(base);
1113 	ent[TSTAT_RETENT_SHIFT] |= shift;
1114 	/* LINTED E_EXPR_NULL_EFFECT */
1115 	ent[TSTAT_RETENT_COUNT_LD] |= offsetof(tstat_missdata_t, tmiss_count);
1116 	/* LINTED E_EXPR_NULL_EFFECT */
1117 	ent[TSTAT_RETENT_COUNT_ST] |= offsetof(tstat_missdata_t, tmiss_count);
1118 	ent[TSTAT_RETENT_TMPTSHI] |= HI22(tmptick);
1119 	ent[TSTAT_RETENT_TMPTSLO] |= LO10(tmptick);
1120 	ent[TSTAT_RETENT_TIME_LD] |= offsetof(tstat_missdata_t, tmiss_time);
1121 	ent[TSTAT_RETENT_TIME_ST] |= offsetof(tstat_missdata_t, tmiss_time);
1122 }
1123 
1124 #if defined(sun4v)
1125 #undef TSTAT_RETENT_TDATASHFT
1126 #endif
1127 #undef TSTAT_RETENT_STATHI
1128 #undef TSTAT_RETENT_STATLO
1129 #undef TSTAT_RETENT_SHIFT
1130 #undef TSTAT_RETENT_COUNT_LD
1131 #undef TSTAT_RETENT_COUNT_ST
1132 #undef TSTAT_RETENT_TMPTSHI
1133 #undef TSTAT_RETENT_TMPTSLO
1134 #undef TSTAT_RETENT_TIME_LD
1135 #undef TSTAT_RETENT_TIME_ST
1136 
1137 /*
1138  * The TSTAT_TLBENT_* constants define offsets in the TLB entry.  They are
1139  * used only in trapstat_tlbent() (below) and #undef'd immediately afterwards.
1140  * Any change to "tlbent" in trapstat_tlbent() will likely require changes
1141  * to these constants.
1142  */
1143 
1144 #ifndef sun4v
1145 #define	TSTAT_TLBENT_STATHI	0
1146 #define	TSTAT_TLBENT_STATLO_LD	1
1147 #define	TSTAT_TLBENT_STATLO_ST	3
1148 #define	TSTAT_TLBENT_MMUASI	15
1149 #define	TSTAT_TLBENT_TPCHI	18
1150 #define	TSTAT_TLBENT_TPCLO_USER	19
1151 #define	TSTAT_TLBENT_TPCLO_KERN	21
1152 #define	TSTAT_TLBENT_TSHI	25
1153 #define	TSTAT_TLBENT_TSLO	27
1154 #define	TSTAT_TLBENT_BA		28
1155 #else /* sun4v */
1156 #define	TSTAT_TLBENT_TDATASHFT	2
1157 #define	TSTAT_TLBENT_STATHI	3
1158 #define	TSTAT_TLBENT_STATLO_LD	5
1159 #define	TSTAT_TLBENT_STATLO_ST	7
1160 #define	TSTAT_TLBENT_TAGTARGET	23
1161 #define	TSTAT_TLBENT_TPCHI	25
1162 #define	TSTAT_TLBENT_TPCLO_USER	26
1163 #define	TSTAT_TLBENT_TPCLO_KERN	28
1164 #define	TSTAT_TLBENT_TSHI	32
1165 #define	TSTAT_TLBENT_TSLO	35
1166 #define	TSTAT_TLBENT_BA		36
1167 #endif /* sun4v */
1168 
1169 static void
1170 trapstat_tlbent(tstat_percpu_t *tcpu, int entno)
1171 {
1172 	uint32_t *ent;
1173 	uintptr_t orig, va, baoffs;
1174 #ifndef sun4v
1175 	int itlb = entno == TSTAT_ENT_ITLBMISS;
1176 	uint32_t asi = itlb ? ASI(ASI_IMMU) : ASI(ASI_DMMU);
1177 #else
1178 	int itlb = (entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_ITLBMISS);
1179 	uint32_t tagtarget_off = itlb ? MMFSA_I_CTX : MMFSA_D_CTX;
1180 	uint32_t *tent;			/* MMU trap vector entry */
1181 	uintptr_t tentva;		/* MMU trap vector entry va */
1182 	static const uint32_t mmumiss[TSTAT_ENT_NINSTR] = {
1183 	    0x30800000,			/* ba,a addr */
1184 	    NOP, NOP, NOP, NOP, NOP, NOP, NOP
1185 	};
1186 #endif
1187 	int entoffs = entno << TSTAT_ENT_SHIFT;
1188 	uintptr_t tmptick, stat, tpc, utpc;
1189 	tstat_pgszdata_t *data = &tcpu->tcpu_data->tdata_pgsz[0];
1190 	tstat_tlbdata_t *udata, *kdata;
1191 	tstat_tlbret_t *ret;
1192 
1193 	/*
1194 	 * When trapstat is run with TLB statistics, this is the entry for
1195 	 * both I- and D-TLB misses; this code performs trap level pushing,
1196 	 * as described in the "TLB Statistics" section of the block comment.
1197 	 * This code is executing at TL 1; %tstate[0] contains the saved
1198 	 * state at the time of the TLB miss.  Pushing trap level 1 (and thus
1199 	 * raising TL to 2) requires us to fill in %tstate[1] with our %pstate,
1200 	 * %cwp and %asi.  We leave %tt unchanged, and we set %tpc and %tnpc to
1201 	 * the appropriate TLB return entry (based on the context of the miss).
1202 	 * Finally, we sample %tick, and stash it in the tdata_tmptick member
1203 	 * the per-CPU tstat_data structure.  tdata_tmptick will be used in
1204 	 * the TLB return entry to determine the amount of time spent in the
1205 	 * TLB miss handler.
1206 	 *
1207 	 * Note that on sun4v platforms, we must obtain the context information
1208 	 * from the MMU fault status area. (The base address of this MMU fault
1209 	 * status area is kept in the scratchpad register 0.)
1210 	 */
1211 	static const uint32_t tlbent[] = {
1212 #ifndef sun4v
1213 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
1214 	    0xc4586000,			/* ldx   [%g1 + %lo(stat)], %g2	*/
1215 	    0x8400a001,			/* add   %g2, 1, %g2		*/
1216 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(stat)]	*/
1217 	    0x85524000,			/* rdpr  %cwp, %g2		*/
1218 	    0x87518000,			/* rdpr  %pstate, %g3		*/
1219 	    0x8728f008,			/* sllx  %g3, 8, %g3		*/
1220 	    0x84108003,			/* or    %g2, %g3, %g2		*/
1221 	    0x8740c000,			/* rd    %asi, %g3		*/
1222 	    0x8728f018,			/* sllx  %g3, 24, %g3		*/
1223 	    0x84108003,			/* or    %g2, %g3, %g2		*/
1224 	    0x8350c000,			/* rdpr  %tt, %g1		*/
1225 	    0x8f902002,			/* wrpr  %g0, 2, %tl		*/
1226 	    0x85908000,			/* wrpr  %g2, %g0, %tstate	*/
1227 	    0x87904000,			/* wrpr  %g1, %g0, %tt		*/
1228 	    0xc2d80000,			/* ldxa  [%g0]ASI_MMU, %g1	*/
1229 	    0x83307030,			/* srlx  %g1, CTXSHIFT, %g1	*/
1230 	    0x02c04004,			/* brz,pn %g1, .+0x10		*/
1231 	    0x03000000, 		/* sethi %hi(new_tpc), %g1	*/
1232 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
1233 	    0x30800002,			/* ba,a  .+0x8			*/
1234 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
1235 	    0x81904000,			/* wrpr  %g1, %g0, %tpc		*/
1236 	    0x82006004,			/* add   %g1, 4, %g1		*/
1237 	    0x83904000,			/* wrpr  %g1, %g0, %tnpc	*/
1238 	    0x03000000, 		/* sethi %hi(tmptick), %g1	*/
1239 	    0x85410000,			/* rd    %tick, %g2		*/
1240 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(tmptick)] */
1241 	    0x30800000,			/* ba,a  addr			*/
1242 	    NOP, NOP, NOP
1243 #else /* sun4v */
1244 	    0x82102008,			/* mov SCRATCHPAD_CPUID, %g1	*/
1245 	    0xc8d84400,			/* ldxa [%g1]ASI_SCRATCHPAD, %g4 */
1246 	    0x89293000,			/* sllx %g4, TSTAT_DATA_SHIFT, %g4 */
1247 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
1248 	    0x82004004,			/* add %g1, %g4, %g1		*/
1249 	    0xc4586000,			/* ldx   [%g1 + %lo(stat)], %g2	*/
1250 	    0x8400a001,			/* add   %g2, 1, %g2		*/
1251 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(stat)]	*/
1252 	    0x85524000,			/* rdpr  %cwp, %g2		*/
1253 	    0x87518000,			/* rdpr  %pstate, %g3		*/
1254 	    0x8728f008,			/* sllx  %g3, 8, %g3		*/
1255 	    0x84108003,			/* or    %g2, %g3, %g2		*/
1256 	    0x8740c000,			/* rd    %asi, %g3		*/
1257 	    0x8728f018,			/* sllx  %g3, 24, %g3		*/
1258 	    0x83540000,			/* rdpr  %gl, %g1		*/
1259 	    0x83287028,			/* sllx  %g1, 40, %g1		*/
1260 	    0x86104003,			/* or    %g1, %g3, %g3		*/
1261 	    0x84108003,			/* or    %g2, %g3, %g2		*/
1262 	    0x8350c000,			/* rdpr  %tt, %g1		*/
1263 	    0x8f902002,			/* wrpr  %g0, 2, %tl		*/
1264 	    0x85908000,			/* wrpr  %g2, %g0, %tstate	*/
1265 	    0x87904000,			/* wrpr  %g1, %g0, %tt		*/
1266 	    0xc2d80400,			/* ldxa  [%g0]ASI_SCRATCHPAD, %g1 */
1267 	    0xc2586000,			/* ldx  [%g1 + MMFSA_?_CTX], %g1 */
1268 	    0x02c04004,			/* brz,pn %g1, .+0x10		*/
1269 	    0x03000000, 		/* sethi %hi(new_tpc), %g1	*/
1270 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
1271 	    0x30800002,			/* ba,a  .+0x8			*/
1272 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
1273 	    0x81904000,			/* wrpr  %g1, %g0, %tpc		*/
1274 	    0x82006004,			/* add   %g1, 4, %g1		*/
1275 	    0x83904000,			/* wrpr  %g1, %g0, %tnpc	*/
1276 	    0x03000000, 		/* sethi %hi(tmptick), %g1	*/
1277 	    0x82004004,			/* add %g1, %g4, %g1		*/
1278 	    0x85410000,			/* rd    %tick, %g2		*/
1279 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(tmptick)] */
1280 	    0x30800000			/* ba,a  addr			*/
1281 #endif /* sun4v */
1282 	};
1283 
1284 	ASSERT(MUTEX_HELD(&tstat_lock));
1285 #ifndef sun4v
1286 	ASSERT(entno == TSTAT_ENT_ITLBMISS || entno == TSTAT_ENT_DTLBMISS);
1287 
1288 	stat = TSTAT_DATA_OFFS(tcpu, tdata_traps) + entoffs;
1289 	tmptick = TSTAT_DATA_OFFS(tcpu, tdata_tmptick);
1290 #else /* sun4v */
1291 	ASSERT(entno == TSTAT_ENT_ITLBMISS || entno == TSTAT_ENT_DTLBMISS ||
1292 	    entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_DMMUMISS);
1293 
1294 	stat = TSTAT_CPU0_DATA_OFFS(tcpu, tdata_traps) + entoffs;
1295 	tmptick = TSTAT_CPU0_DATA_OFFS(tcpu, tdata_tmptick);
1296 #endif /* sun4v */
1297 
1298 	if (itlb) {
1299 		ret = &tcpu->tcpu_instr->tinst_itlbret;
1300 		udata = &data->tpgsz_user.tmode_itlb;
1301 		kdata = &data->tpgsz_kernel.tmode_itlb;
1302 		tpc = TSTAT_INSTR_OFFS(tcpu, tinst_itlbret.ttlbr_ktlb);
1303 	} else {
1304 		ret = &tcpu->tcpu_instr->tinst_dtlbret;
1305 		udata = &data->tpgsz_user.tmode_dtlb;
1306 		kdata = &data->tpgsz_kernel.tmode_dtlb;
1307 		tpc = TSTAT_INSTR_OFFS(tcpu, tinst_dtlbret.ttlbr_ktlb);
1308 	}
1309 
1310 	utpc = tpc + offsetof(tstat_tlbret_t, ttlbr_utlb) -
1311 	    offsetof(tstat_tlbret_t, ttlbr_ktlb);
1312 
1313 	ASSERT(HI22(tpc) == HI22(utpc));
1314 
1315 	ent = (uint32_t *)((uintptr_t)tcpu->tcpu_instr + entoffs);
1316 	orig = KERNELBASE + entoffs;
1317 	va = (uintptr_t)tcpu->tcpu_ibase + entoffs;
1318 	baoffs = TSTAT_TLBENT_BA * sizeof (uint32_t);
1319 
1320 #ifdef sun4v
1321 	/*
1322 	 * Because of lack of space, interposing tlbent trap handler
1323 	 * for TLB and MMU miss traps cannot be placed in-line. Instead,
1324 	 * we copy it to the space set aside for shared trap handlers
1325 	 * continuation in the interposing trap table and invoke it by
1326 	 * placing a branch in the trap table itself.
1327 	 */
1328 	tent = ent;		/* trap vector entry */
1329 	tentva = va;		/* trap vector entry va */
1330 
1331 	if (itlb) {
1332 		ent = (uint32_t *)((uintptr_t)
1333 		    &tcpu->tcpu_instr->tinst_immumiss);
1334 		va = TSTAT_INSTR_OFFS(tcpu, tinst_immumiss);
1335 	} else {
1336 		ent = (uint32_t *)((uintptr_t)
1337 		    &tcpu->tcpu_instr->tinst_dmmumiss);
1338 		va = TSTAT_INSTR_OFFS(tcpu, tinst_dmmumiss);
1339 	}
1340 	bcopy(mmumiss, tent, sizeof (mmumiss));
1341 	tent[0] |= DISP22(tentva, va);
1342 #endif /* sun4v */
1343 
1344 	bcopy(tlbent, ent, sizeof (tlbent));
1345 
1346 #if defined(sun4v)
1347 	ent[TSTAT_TLBENT_TDATASHFT] |= LO10((uintptr_t)TSTAT_DATA_SHIFT);
1348 #endif
1349 	ent[TSTAT_TLBENT_STATHI] |= HI22(stat);
1350 	ent[TSTAT_TLBENT_STATLO_LD] |= LO10(stat);
1351 	ent[TSTAT_TLBENT_STATLO_ST] |= LO10(stat);
1352 #ifndef sun4v
1353 	ent[TSTAT_TLBENT_MMUASI] |= asi;
1354 #else
1355 	ent[TSTAT_TLBENT_TAGTARGET] |= tagtarget_off;
1356 #endif
1357 	ent[TSTAT_TLBENT_TPCHI] |= HI22(tpc);
1358 	ent[TSTAT_TLBENT_TPCLO_USER] |= LO10(utpc);
1359 	ent[TSTAT_TLBENT_TPCLO_KERN] |= LO10(tpc);
1360 	ent[TSTAT_TLBENT_TSHI] |= HI22(tmptick);
1361 	ent[TSTAT_TLBENT_TSLO] |= LO10(tmptick);
1362 	ent[TSTAT_TLBENT_BA] |= DISP22(va + baoffs, orig);
1363 
1364 	/*
1365 	 * And now set up the TLB return entries.
1366 	 */
1367 	trapstat_tlbretent(tcpu, &ret->ttlbr_ktlb, &kdata->ttlb_tlb);
1368 	trapstat_tlbretent(tcpu, &ret->ttlbr_ktsb, &kdata->ttlb_tsb);
1369 	trapstat_tlbretent(tcpu, &ret->ttlbr_utlb, &udata->ttlb_tlb);
1370 	trapstat_tlbretent(tcpu, &ret->ttlbr_utsb, &udata->ttlb_tsb);
1371 }
1372 
1373 #if defined(sun4v)
1374 #undef TSTAT_TLBENT_TDATASHFT
1375 #endif
1376 #undef TSTAT_TLBENT_STATHI
1377 #undef TSTAT_TLBENT_STATLO_LD
1378 #undef TSTAT_TLBENT_STATLO_ST
1379 #ifndef sun4v
1380 #undef TSTAT_TLBENT_MMUASI
1381 #else
1382 #undef TSTAT_TLBENT_TAGTARGET
1383 #endif
1384 #undef TSTAT_TLBENT_TPCHI
1385 #undef TSTAT_TLBENT_TPCLO_USER
1386 #undef TSTAT_TLBENT_TPCLO_KERN
1387 #undef TSTAT_TLBENT_TSHI
1388 #undef TSTAT_TLBENT_TSLO
1389 #undef TSTAT_TLBENT_BA
1390 
1391 /*
1392  * The TSTAT_ENABLED_* constants define offsets in the enabled entry; the
1393  * TSTAT_DISABLED_BA constant defines an offset in the disabled entry.  Both
1394  * sets of constants are used only in trapstat_make_traptab() (below) and
1395  * #undef'd immediately afterwards.  Any change to "enabled" or "disabled"
1396  * in trapstat_make_traptab() will likely require changes to these constants.
1397  */
1398 #ifndef sun4v
1399 #define	TSTAT_ENABLED_STATHI	0
1400 #define	TSTAT_ENABLED_STATLO_LD	1
1401 #define	TSTAT_ENABLED_STATLO_ST 3
1402 #define	TSTAT_ENABLED_BA	4
1403 #define	TSTAT_DISABLED_BA	0
1404 
1405 static void
1406 trapstat_make_traptab(tstat_percpu_t *tcpu)
1407 {
1408 	uint32_t *ent;
1409 	uint64_t *stat;
1410 	uintptr_t orig, va, en_baoffs, dis_baoffs;
1411 	int nent;
1412 
1413 	/*
1414 	 * This is the entry in the interposing trap table for enabled trap
1415 	 * table entries.  It loads a counter, increments it and stores it
1416 	 * back before branching to the actual trap table entry.
1417 	 */
1418 	static const uint32_t enabled[TSTAT_ENT_NINSTR] = {
1419 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
1420 	    0xc4586000,			/* ldx   [%g1 + %lo(stat)], %g2	*/
1421 	    0x8400a001,			/* add   %g2, 1, %g2		*/
1422 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(stat)]	*/
1423 	    0x30800000,			/* ba,a addr			*/
1424 	    NOP, NOP, NOP
1425 	};
1426 
1427 	/*
1428 	 * This is the entry in the interposing trap table for disabled trap
1429 	 * table entries.  It simply branches to the actual, underlying trap
1430 	 * table entry.  As explained in the "Implementation Details" section
1431 	 * of the block comment, all TL>0 traps _must_ use the disabled entry;
1432 	 * additional entries may be explicitly disabled through the use
1433 	 * of TSTATIOC_ENTRY/TSTATIOC_NOENTRY.
1434 	 */
1435 	static const uint32_t disabled[TSTAT_ENT_NINSTR] = {
1436 	    0x30800000,			/* ba,a addr			*/
1437 	    NOP, NOP, NOP, NOP, NOP, NOP, NOP,
1438 	};
1439 
1440 	ASSERT(MUTEX_HELD(&tstat_lock));
1441 
1442 	ent = tcpu->tcpu_instr->tinst_traptab;
1443 	stat = (uint64_t *)TSTAT_DATA_OFFS(tcpu, tdata_traps);
1444 	orig = KERNELBASE;
1445 	va = (uintptr_t)tcpu->tcpu_ibase;
1446 	en_baoffs = TSTAT_ENABLED_BA * sizeof (uint32_t);
1447 	dis_baoffs = TSTAT_DISABLED_BA * sizeof (uint32_t);
1448 
1449 	for (nent = 0; nent < TSTAT_TOTAL_NENT; nent++) {
1450 		if (tstat_enabled[nent]) {
1451 			bcopy(enabled, ent, sizeof (enabled));
1452 			ent[TSTAT_ENABLED_STATHI] |= HI22((uintptr_t)stat);
1453 			ent[TSTAT_ENABLED_STATLO_LD] |= LO10((uintptr_t)stat);
1454 			ent[TSTAT_ENABLED_STATLO_ST] |= LO10((uintptr_t)stat);
1455 			ent[TSTAT_ENABLED_BA] |= DISP22(va + en_baoffs, orig);
1456 		} else {
1457 			bcopy(disabled, ent, sizeof (disabled));
1458 			ent[TSTAT_DISABLED_BA] |= DISP22(va + dis_baoffs, orig);
1459 		}
1460 
1461 		stat++;
1462 		orig += sizeof (enabled);
1463 		ent += sizeof (enabled) / sizeof (*ent);
1464 		va += sizeof (enabled);
1465 	}
1466 }
1467 
1468 #undef TSTAT_ENABLED_STATHI
1469 #undef TSTAT_ENABLED_STATLO_LD
1470 #undef TSTAT_ENABLED_STATLO_ST
1471 #undef TSTAT_ENABLED_BA
1472 #undef TSTAT_DISABLED_BA
1473 
1474 #else /* sun4v */
1475 
1476 #define	TSTAT_ENABLED_STATHI	0
1477 #define	TSTAT_ENABLED_STATLO	1
1478 #define	TSTAT_ENABLED_ADDRHI	2
1479 #define	TSTAT_ENABLED_ADDRLO	3
1480 #define	TSTAT_ENABLED_CONTBA	6
1481 #define	TSTAT_ENABLED_TDATASHFT	7
1482 #define	TSTAT_DISABLED_BA	0
1483 
1484 static void
1485 trapstat_make_traptab(tstat_percpu_t *tcpu)
1486 {
1487 	uint32_t *ent;
1488 	uint64_t *stat;
1489 	uintptr_t orig, va, en_baoffs, dis_baoffs;
1490 	uintptr_t tstat_cont_va;
1491 	int nent;
1492 
1493 	/*
1494 	 * This is the entry in the interposing trap table for enabled trap
1495 	 * table entries.  It loads a counter, increments it and stores it
1496 	 * back before branching to the actual trap table entry.
1497 	 *
1498 	 * All CPUs share the same interposing trap entry to count the
1499 	 * number of traps. Note that the trap counter is kept in per CPU
1500 	 * trap statistics area. Its address is obtained dynamically by
1501 	 * adding the offset of that CPU's trap statistics area from CPU 0
1502 	 * (i.e. cpu_id * TSTAT_DATA_SIZE) to the address of the CPU 0
1503 	 * trap counter already coded in the interposing trap entry itself.
1504 	 *
1505 	 * Since this interposing code sequence to count traps takes more
1506 	 * than 8 instructions, it's split in two parts as follows:
1507 	 *
1508 	 *   tstat_trapcnt:
1509 	 *	sethi %hi(stat), %g1
1510 	 *	or    %g1, %lo[stat), %g1	! %g1 = CPU0 trap counter addr
1511 	 *	sethi %hi(addr), %g2
1512 	 *	or    %g2, %lo(addr), %g2	! %g2 = real trap handler addr
1513 	 *	mov   ASI_SCRATCHPAD_CPUID, %g3
1514 	 *	ldxa [%g3]ASI_SCRATCHPAD, %g3	! %g3 = CPU ID
1515 	 *	ba tstat_trapcnt_cont		! branch to tstat_trapcnt_cont
1516 	 *	sllx %g3, TSTAT_DATA_SHIFT, %g3	! %g3 = CPU trapstat data offset
1517 	 *
1518 	 *   tstat_trapcnt_cont:
1519 	 *	ldx [%g1 + %g3], %g4		! get counter value
1520 	 *	add %g4, 1, %g4			! increment value
1521 	 *	jmp %g2				! jump to original trap handler
1522 	 *	stx %g4, [%g1 + %g3]		! store counter value
1523 	 *
1524 	 * First part, i.e. tstat_trapcnt, is per trap and is kept in-line in
1525 	 * the interposing trap table. However, the tstat_trapcnt_cont code
1526 	 * sequence is shared by all traps and is kept right after the
1527 	 * the interposing trap table.
1528 	 */
1529 	static const uint32_t enabled[TSTAT_ENT_NINSTR] = {
1530 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
1531 	    0x82106000,			/* or   %g1, %lo[stat), %g1	*/
1532 	    0x05000000, 		/* sethi %hi(addr), %g2		*/
1533 	    0x8410a000,			/* or   %g2, %lo(addr), %g2	*/
1534 	    0x86102008,			/* mov	ASI_SCRATCHPAD_CPUID, %g3 */
1535 	    0xc6d8c400,			/* ldxa [%g3]ASI_SCRATCHPAD, %g3 */
1536 	    0x10800000,			/* ba enabled_cont		*/
1537 	    0x8728f000			/* sllx %g3, TSTAT_DATA_SHIFT, %g3 */
1538 	};
1539 
1540 	static const uint32_t enabled_cont[TSTAT_ENT_NINSTR] = {
1541 	    0xc8584003, 		/* ldx [%g1 + %g3], %g4		*/
1542 	    0x88012001,			/* add %g4, 1, %g4		*/
1543 	    0x81c08000,			/* jmp %g2			*/
1544 	    0xc8704003,			/* stx %g4, [%g1 + %g3]		*/
1545 	    NOP, NOP, NOP, NOP
1546 	};
1547 
1548 	/*
1549 	 * This is the entry in the interposing trap table for disabled trap
1550 	 * table entries.  It simply branches to the actual, underlying trap
1551 	 * table entry.  As explained in the "Implementation Details" section
1552 	 * of the block comment, all TL>0 traps _must_ use the disabled entry;
1553 	 * additional entries may be explicitly disabled through the use
1554 	 * of TSTATIOC_ENTRY/TSTATIOC_NOENTRY.
1555 	 */
1556 	static const uint32_t disabled[TSTAT_ENT_NINSTR] = {
1557 	    0x30800000,			/* ba,a addr			*/
1558 	    NOP, NOP, NOP, NOP, NOP, NOP, NOP,
1559 	};
1560 
1561 	ASSERT(MUTEX_HELD(&tstat_lock));
1562 	ent = tcpu->tcpu_instr->tinst_traptab;
1563 	stat = (uint64_t *)TSTAT_CPU0_DATA_OFFS(tcpu, tdata_traps);
1564 	orig = KERNELBASE;
1565 	va = (uintptr_t)tcpu->tcpu_ibase;
1566 	en_baoffs = TSTAT_ENABLED_CONTBA * sizeof (uint32_t);
1567 	dis_baoffs = TSTAT_DISABLED_BA * sizeof (uint32_t);
1568 	tstat_cont_va = TSTAT_INSTR_OFFS(tcpu, tinst_trapcnt);
1569 
1570 	for (nent = 0; nent < TSTAT_TOTAL_NENT; nent++) {
1571 		if (tstat_enabled[nent]) {
1572 			bcopy(enabled, ent, sizeof (enabled));
1573 			ent[TSTAT_ENABLED_STATHI] |= HI22((uintptr_t)stat);
1574 			ent[TSTAT_ENABLED_STATLO] |= LO10((uintptr_t)stat);
1575 			ent[TSTAT_ENABLED_ADDRHI] |= HI22((uintptr_t)orig);
1576 			ent[TSTAT_ENABLED_ADDRLO] |= LO10((uintptr_t)orig);
1577 			ent[TSTAT_ENABLED_CONTBA] |=
1578 			    DISP22(va + en_baoffs, tstat_cont_va);
1579 			ent[TSTAT_ENABLED_TDATASHFT] |=
1580 			    LO10((uintptr_t)TSTAT_DATA_SHIFT);
1581 		} else {
1582 			bcopy(disabled, ent, sizeof (disabled));
1583 			ent[TSTAT_DISABLED_BA] |= DISP22(va + dis_baoffs, orig);
1584 		}
1585 
1586 		stat++;
1587 		orig += sizeof (enabled);
1588 		ent += sizeof (enabled) / sizeof (*ent);
1589 		va += sizeof (enabled);
1590 	}
1591 	bcopy(enabled_cont, (uint32_t *)tcpu->tcpu_instr->tinst_trapcnt,
1592 	    sizeof (enabled_cont));
1593 }
1594 
1595 #undef	TSTAT_ENABLED_TDATASHFT
1596 #undef	TSTAT_ENABLED_STATHI
1597 #undef	TSTAT_ENABLED_STATLO
1598 #undef	TSTAT_ENABLED_ADDRHI
1599 #undef	TSTAT_ENABLED_ADDRLO
1600 #undef	TSTAT_ENABLED_CONTBA
1601 #undef	TSTAT_DISABLED_BA
1602 
1603 #endif /* sun4v */
1604 
1605 #ifndef sun4v
1606 /*
1607  * See Section A.6 in SPARC v9 Manual.
1608  * max branch = 4*((2^21)-1) = 8388604
1609  */
1610 #define	MAX_BICC_BRANCH_DISPLACEMENT (4 * ((1 << 21) - 1))
1611 #endif
1612 
1613 static void
1614 trapstat_setup(processorid_t cpu)
1615 {
1616 	tstat_percpu_t *tcpu = &tstat_percpu[cpu];
1617 #ifndef sun4v
1618 	int i;
1619 	caddr_t va;
1620 	pfn_t *pfn;
1621 	cpu_t *cp;
1622 	uint_t strand_idx;
1623 	size_t tstat_offset;
1624 #endif
1625 
1626 	ASSERT(tcpu->tcpu_pfn == NULL);
1627 	ASSERT(tcpu->tcpu_instr == NULL);
1628 	ASSERT(tcpu->tcpu_data == NULL);
1629 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
1630 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED));
1631 	ASSERT(MUTEX_HELD(&cpu_lock));
1632 	ASSERT(MUTEX_HELD(&tstat_lock));
1633 
1634 #ifndef sun4v
1635 	/*
1636 	 * The lower fifteen bits of the %tba are always read as zero; we must
1637 	 * align our instruction base address appropriately.
1638 	 */
1639 	tstat_offset = tstat_total_size;
1640 
1641 	cp = cpu_get(cpu);
1642 	ASSERT(cp != NULL);
1643 	if ((strand_idx = cpu ^ pg_plat_hw_instance_id(cp, PGHW_IPIPE)) != 0) {
1644 		/*
1645 		 * On sun4u platforms with multiple CPUs sharing the MMU
1646 		 * (Olympus-C has 2 strands per core), each CPU uses a
1647 		 * disjoint trap table.  The indexing is based on the
1648 		 * strand id, which is obtained by XOR'ing the cpuid with
1649 		 * the coreid.
1650 		 */
1651 		tstat_offset += tstat_total_size * strand_idx;
1652 
1653 		/*
1654 		 * Offset must be less than the maximum PC-relative branch
1655 		 * displacement for Bicc variants.  See the Implementation
1656 		 * Details comment.
1657 		 */
1658 		ASSERT(tstat_offset <= MAX_BICC_BRANCH_DISPLACEMENT);
1659 	}
1660 
1661 	tcpu->tcpu_ibase = (caddr_t)((KERNELBASE - tstat_offset)
1662 	    & TSTAT_TBA_MASK);
1663 	tcpu->tcpu_dbase = tcpu->tcpu_ibase + TSTAT_INSTR_SIZE;
1664 	tcpu->tcpu_vabase = tcpu->tcpu_ibase;
1665 
1666 	tcpu->tcpu_pfn = vmem_alloc(tstat_arena, tstat_total_pages, VM_SLEEP);
1667 	bzero(tcpu->tcpu_pfn, tstat_total_pages);
1668 	pfn = tcpu->tcpu_pfn;
1669 
1670 	tcpu->tcpu_instr = vmem_alloc(tstat_arena, TSTAT_INSTR_SIZE, VM_SLEEP);
1671 
1672 	va = (caddr_t)tcpu->tcpu_instr;
1673 	for (i = 0; i < TSTAT_INSTR_PAGES; i++, va += MMU_PAGESIZE)
1674 		*pfn++ = hat_getpfnum(kas.a_hat, va);
1675 
1676 	/*
1677 	 * We must be sure that the pages that we will use to examine the data
1678 	 * have the same virtual color as the pages to which the data is being
1679 	 * recorded, hence the alignment and phase constraints on the
1680 	 * allocation.
1681 	 */
1682 	tcpu->tcpu_data = vmem_xalloc(tstat_arena, tstat_data_size,
1683 	    shm_alignment, (uintptr_t)tcpu->tcpu_dbase & (shm_alignment - 1),
1684 	    0, 0, NULL, VM_SLEEP);
1685 	bzero(tcpu->tcpu_data, tstat_data_size);
1686 	tcpu->tcpu_data->tdata_cpuid = cpu;
1687 
1688 	va = (caddr_t)tcpu->tcpu_data;
1689 	for (i = 0; i < tstat_data_pages; i++, va += MMU_PAGESIZE)
1690 		*pfn++ = hat_getpfnum(kas.a_hat, va);
1691 
1692 	/*
1693 	 * Now that we have all of the instruction and data pages allocated,
1694 	 * make the trap table from scratch.
1695 	 */
1696 	trapstat_make_traptab(tcpu);
1697 
1698 	if (tstat_options & TSTAT_OPT_TLBDATA) {
1699 		/*
1700 		 * TLB Statistics have been specified; set up the I- and D-TLB
1701 		 * entries and corresponding TLB return entries.
1702 		 */
1703 		trapstat_tlbent(tcpu, TSTAT_ENT_ITLBMISS);
1704 		trapstat_tlbent(tcpu, TSTAT_ENT_DTLBMISS);
1705 	}
1706 
1707 #else /* sun4v */
1708 
1709 	/*
1710 	 * The lower fifteen bits of the %tba are always read as zero; hence
1711 	 * it must be aligned at least on 512K boundary.
1712 	 */
1713 	tcpu->tcpu_vabase = (caddr_t)(KERNELBASE - MMU_PAGESIZE4M);
1714 	tcpu->tcpu_ibase = tcpu->tcpu_vabase;
1715 	tcpu->tcpu_dbase = tcpu->tcpu_ibase + TSTAT_INSTR_SIZE +
1716 	    cpu * TSTAT_DATA_SIZE;
1717 
1718 	tcpu->tcpu_pfn = &tstat_pfn;
1719 	tcpu->tcpu_instr = (tstat_instr_t *)tstat_va;
1720 	tcpu->tcpu_data = (tstat_data_t *)(tstat_va + TSTAT_INSTR_SIZE +
1721 	    cpu * TSTAT_DATA_SIZE);
1722 	bzero(tcpu->tcpu_data, TSTAT_DATA_SIZE);
1723 	tcpu->tcpu_data->tdata_cpuid = cpu;
1724 
1725 	/*
1726 	 * Now that we have all of the instruction and data pages allocated,
1727 	 * make the trap table from scratch. It should be done only once
1728 	 * as it is shared by all CPUs.
1729 	 */
1730 	if (!tstat_traptab_initialized)
1731 		trapstat_make_traptab(tcpu);
1732 
1733 	if (tstat_options & TSTAT_OPT_TLBDATA) {
1734 		/*
1735 		 * TLB Statistics have been specified; set up the I- and D-TLB
1736 		 * entries and corresponding TLB return entries.
1737 		 */
1738 		if (!tstat_traptab_initialized) {
1739 			if (tstat_fast_tlbstat) {
1740 				trapstat_tlbent(tcpu, TSTAT_ENT_IMMUMISS);
1741 				trapstat_tlbent(tcpu, TSTAT_ENT_DMMUMISS);
1742 			} else {
1743 				trapstat_tlbent(tcpu, TSTAT_ENT_ITLBMISS);
1744 				trapstat_tlbent(tcpu, TSTAT_ENT_DTLBMISS);
1745 			}
1746 		}
1747 	}
1748 	tstat_traptab_initialized = 1;
1749 #endif /* sun4v */
1750 
1751 	tcpu->tcpu_flags |= TSTAT_CPU_ALLOCATED;
1752 
1753 	/*
1754 	 * Finally, get the target CPU to load the locked pages into its TLBs.
1755 	 */
1756 	xc_one(cpu, (xcfunc_t *)trapstat_load_tlb, 0, 0);
1757 }
1758 
1759 static void
1760 trapstat_teardown(processorid_t cpu)
1761 {
1762 	tstat_percpu_t *tcpu = &tstat_percpu[cpu];
1763 #ifndef sun4v
1764 	int i;
1765 #endif
1766 	caddr_t va = tcpu->tcpu_vabase;
1767 
1768 	ASSERT(tcpu->tcpu_pfn != NULL);
1769 	ASSERT(tcpu->tcpu_instr != NULL);
1770 	ASSERT(tcpu->tcpu_data != NULL);
1771 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
1772 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
1773 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
1774 	ASSERT(MUTEX_HELD(&cpu_lock));
1775 	ASSERT(MUTEX_HELD(&tstat_lock));
1776 
1777 #ifndef sun4v
1778 	vmem_free(tstat_arena, tcpu->tcpu_pfn, tstat_total_pages);
1779 	vmem_free(tstat_arena, tcpu->tcpu_instr, TSTAT_INSTR_SIZE);
1780 	vmem_free(tstat_arena, tcpu->tcpu_data, tstat_data_size);
1781 
1782 	for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) {
1783 		xt_one(cpu, vtag_flushpage_tl1, (uint64_t)va,
1784 		    (uint64_t)ksfmmup);
1785 	}
1786 #else
1787 	xt_one(cpu, vtag_unmap_perm_tl1, (uint64_t)va, KCONTEXT);
1788 #endif
1789 
1790 	tcpu->tcpu_pfn = NULL;
1791 	tcpu->tcpu_instr = NULL;
1792 	tcpu->tcpu_data = NULL;
1793 	tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED;
1794 }
1795 
1796 static int
1797 trapstat_go()
1798 {
1799 	cpu_t *cp;
1800 
1801 	mutex_enter(&cpu_lock);
1802 	mutex_enter(&tstat_lock);
1803 
1804 	if (tstat_running) {
1805 		mutex_exit(&tstat_lock);
1806 		mutex_exit(&cpu_lock);
1807 		return (EBUSY);
1808 	}
1809 
1810 #ifdef sun4v
1811 	/*
1812 	 * Allocate large page to hold interposing tables.
1813 	 */
1814 	tstat_va = contig_mem_alloc(MMU_PAGESIZE4M);
1815 	tstat_pfn = va_to_pfn(tstat_va);
1816 	if (tstat_pfn == PFN_INVALID) {
1817 		mutex_exit(&tstat_lock);
1818 		mutex_exit(&cpu_lock);
1819 		return (EAGAIN);
1820 	}
1821 
1822 	/*
1823 	 * For detailed TLB statistics, invoke CPU specific interface
1824 	 * to see if it supports a low overhead interface to collect
1825 	 * TSB hit statistics. If so, make set tstat_fast_tlbstat flag
1826 	 * to reflect that.
1827 	 */
1828 	if (tstat_options & TSTAT_OPT_TLBDATA) {
1829 		int error;
1830 
1831 		tstat_fast_tlbstat = B_FALSE;
1832 		error = cpu_trapstat_conf(CPU_TSTATCONF_INIT);
1833 		if (error == 0)
1834 			tstat_fast_tlbstat = B_TRUE;
1835 		else if (error != ENOTSUP) {
1836 			contig_mem_free(tstat_va, MMU_PAGESIZE4M);
1837 			mutex_exit(&tstat_lock);
1838 			mutex_exit(&cpu_lock);
1839 			return (error);
1840 		}
1841 	}
1842 #endif /* sun4v */
1843 
1844 	/*
1845 	 * First, perform any necessary hot patching.
1846 	 */
1847 	trapstat_hotpatch();
1848 
1849 	/*
1850 	 * Allocate the resources we'll need to measure probe effect.
1851 	 */
1852 	trapstat_probe_alloc();
1853 
1854 
1855 	cp = cpu_list;
1856 	do {
1857 		if (!(tstat_percpu[cp->cpu_id].tcpu_flags & TSTAT_CPU_SELECTED))
1858 			continue;
1859 
1860 		trapstat_setup(cp->cpu_id);
1861 
1862 		/*
1863 		 * Note that due to trapstat_probe()'s use of global data,
1864 		 * we determine the probe effect on each CPU serially instead
1865 		 * of in parallel with an xc_all().
1866 		 */
1867 		xc_one(cp->cpu_id, (xcfunc_t *)trapstat_probe, 0, 0);
1868 	} while ((cp = cp->cpu_next) != cpu_list);
1869 
1870 	xc_all((xcfunc_t *)trapstat_enable, 0, 0);
1871 
1872 	trapstat_probe_free();
1873 	tstat_running = 1;
1874 	mutex_exit(&tstat_lock);
1875 	mutex_exit(&cpu_lock);
1876 
1877 	return (0);
1878 }
1879 
1880 static int
1881 trapstat_stop()
1882 {
1883 	int i;
1884 
1885 	mutex_enter(&cpu_lock);
1886 	mutex_enter(&tstat_lock);
1887 	if (!tstat_running) {
1888 		mutex_exit(&tstat_lock);
1889 		mutex_exit(&cpu_lock);
1890 		return (ENXIO);
1891 	}
1892 
1893 	xc_all((xcfunc_t *)trapstat_disable, 0, 0);
1894 
1895 	for (i = 0; i <= max_cpuid; i++) {
1896 		if (tstat_percpu[i].tcpu_flags & TSTAT_CPU_ALLOCATED)
1897 			trapstat_teardown(i);
1898 	}
1899 
1900 #ifdef sun4v
1901 	tstat_traptab_initialized = 0;
1902 	if (tstat_options & TSTAT_OPT_TLBDATA)
1903 		cpu_trapstat_conf(CPU_TSTATCONF_FINI);
1904 	contig_mem_free(tstat_va, MMU_PAGESIZE4M);
1905 #endif
1906 	trapstat_hotpatch();
1907 	tstat_running = 0;
1908 	mutex_exit(&tstat_lock);
1909 	mutex_exit(&cpu_lock);
1910 
1911 	return (0);
1912 }
1913 
1914 /*
1915  * This is trapstat's DR CPU configuration callback.  It's called (with
1916  * cpu_lock held) to unconfigure a newly powered-off CPU, or to configure a
1917  * powered-off CPU that is to be brought into the system.  We need only take
1918  * action in the unconfigure case:  because a powered-off CPU will have its
1919  * trap table restored to KERNELBASE if it is ever powered back on, we must
1920  * update the flags to reflect that trapstat is no longer enabled on the
1921  * powered-off CPU.  Note that this means that a TSTAT_CPU_ENABLED CPU that
1922  * is unconfigured/powered off and later powered back on/reconfigured will
1923  * _not_ be re-TSTAT_CPU_ENABLED.
1924  */
1925 static int
1926 trapstat_cpu_setup(cpu_setup_t what, processorid_t cpu)
1927 {
1928 	tstat_percpu_t *tcpu = &tstat_percpu[cpu];
1929 
1930 	ASSERT(MUTEX_HELD(&cpu_lock));
1931 	mutex_enter(&tstat_lock);
1932 
1933 	if (!tstat_running) {
1934 		mutex_exit(&tstat_lock);
1935 		return (0);
1936 	}
1937 
1938 	switch (what) {
1939 	case CPU_CONFIG:
1940 		ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
1941 		break;
1942 
1943 	case CPU_UNCONFIG:
1944 		if (tcpu->tcpu_flags & TSTAT_CPU_ENABLED) {
1945 			tcpu->tcpu_flags &= ~TSTAT_CPU_ENABLED;
1946 #ifdef	sun4v
1947 			/*
1948 			 * A power-off, causes the cpu mondo queues to be
1949 			 * unconfigured on sun4v. Since we can't teardown
1950 			 * trapstat's mappings on the cpu that is going away,
1951 			 * we simply mark it as not allocated. This will
1952 			 * prevent a teardown on a cpu with the same cpu id
1953 			 * that might have been added while trapstat is running.
1954 			 */
1955 			if (tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED) {
1956 				tcpu->tcpu_pfn = NULL;
1957 				tcpu->tcpu_instr = NULL;
1958 				tcpu->tcpu_data = NULL;
1959 				tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED;
1960 			}
1961 #endif
1962 		}
1963 		break;
1964 
1965 	default:
1966 		break;
1967 	}
1968 
1969 	mutex_exit(&tstat_lock);
1970 	return (0);
1971 }
1972 
1973 /*
1974  * This is called before a CPR suspend and after a CPR resume.  We don't have
1975  * anything to do before a suspend, but after a restart we must restore the
1976  * trap table to be our interposing trap table.  However, we don't actually
1977  * know whether or not the CPUs have been powered off -- this routine may be
1978  * called while restoring from a failed CPR suspend.  We thus run through each
1979  * TSTAT_CPU_ENABLED CPU, and explicitly destroy and reestablish its
1980  * interposing trap table.  This assures that our state is correct regardless
1981  * of whether or not the CPU has been newly powered on.
1982  */
1983 /*ARGSUSED*/
1984 static boolean_t
1985 trapstat_cpr(void *arg, int code)
1986 {
1987 	cpu_t *cp;
1988 
1989 	if (code == CB_CODE_CPR_CHKPT)
1990 		return (B_TRUE);
1991 
1992 	ASSERT(code == CB_CODE_CPR_RESUME);
1993 
1994 	mutex_enter(&cpu_lock);
1995 	mutex_enter(&tstat_lock);
1996 
1997 	if (!tstat_running) {
1998 		mutex_exit(&tstat_lock);
1999 		mutex_exit(&cpu_lock);
2000 		return (B_TRUE);
2001 	}
2002 
2003 	cp = cpu_list;
2004 	do {
2005 		tstat_percpu_t *tcpu = &tstat_percpu[cp->cpu_id];
2006 
2007 		if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED))
2008 			continue;
2009 
2010 		ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
2011 		ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
2012 
2013 		xc_one(cp->cpu_id, (xcfunc_t *)trapstat_disable, 0, 0);
2014 		ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
2015 
2016 		/*
2017 		 * Preserve this CPU's data in tstat_buffer and rip down its
2018 		 * interposing trap table.
2019 		 */
2020 		bcopy(tcpu->tcpu_data, tstat_buffer, tstat_data_t_size);
2021 		trapstat_teardown(cp->cpu_id);
2022 		ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED));
2023 
2024 		/*
2025 		 * Reestablish the interposing trap table and restore the old
2026 		 * data.
2027 		 */
2028 		trapstat_setup(cp->cpu_id);
2029 		ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
2030 		bcopy(tstat_buffer, tcpu->tcpu_data, tstat_data_t_size);
2031 
2032 		xc_one(cp->cpu_id, (xcfunc_t *)trapstat_enable, 0, 0);
2033 	} while ((cp = cp->cpu_next) != cpu_list);
2034 
2035 	mutex_exit(&tstat_lock);
2036 	mutex_exit(&cpu_lock);
2037 
2038 	return (B_TRUE);
2039 }
2040 
2041 /*ARGSUSED*/
2042 static int
2043 trapstat_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
2044 {
2045 	int i;
2046 
2047 	mutex_enter(&cpu_lock);
2048 	mutex_enter(&tstat_lock);
2049 	if (tstat_open != 0) {
2050 		mutex_exit(&tstat_lock);
2051 		mutex_exit(&cpu_lock);
2052 		return (EBUSY);
2053 	}
2054 
2055 	/*
2056 	 * Register this in open() rather than in attach() to prevent deadlock
2057 	 * with DR code. During attach, I/O device tree locks are grabbed
2058 	 * before trapstat_attach() is invoked - registering in attach
2059 	 * will result in the lock order: device tree lock, cpu_lock.
2060 	 * DR code however requires that cpu_lock be acquired before
2061 	 * device tree locks.
2062 	 */
2063 	ASSERT(!tstat_running);
2064 	register_cpu_setup_func((cpu_setup_func_t *)trapstat_cpu_setup, NULL);
2065 
2066 	/*
2067 	 * Clear all options.  And until specific CPUs are specified, we'll
2068 	 * mark all CPUs as selected.
2069 	 */
2070 	tstat_options = 0;
2071 
2072 	for (i = 0; i <= max_cpuid; i++)
2073 		tstat_percpu[i].tcpu_flags |= TSTAT_CPU_SELECTED;
2074 
2075 	/*
2076 	 * By default, all traps at TL=0 are enabled.  Traps at TL>0 must
2077 	 * be disabled.
2078 	 */
2079 	for (i = 0; i < TSTAT_TOTAL_NENT; i++)
2080 		tstat_enabled[i] = i < TSTAT_NENT ? 1 : 0;
2081 
2082 	tstat_open = 1;
2083 	mutex_exit(&tstat_lock);
2084 	mutex_exit(&cpu_lock);
2085 
2086 	return (0);
2087 }
2088 
2089 /*ARGSUSED*/
2090 static int
2091 trapstat_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
2092 {
2093 	(void) trapstat_stop();
2094 
2095 	ASSERT(!tstat_running);
2096 
2097 	mutex_enter(&cpu_lock);
2098 	unregister_cpu_setup_func((cpu_setup_func_t *)trapstat_cpu_setup, NULL);
2099 	mutex_exit(&cpu_lock);
2100 
2101 	tstat_open = 0;
2102 	return (DDI_SUCCESS);
2103 }
2104 
2105 static int
2106 trapstat_option(int option)
2107 {
2108 	mutex_enter(&tstat_lock);
2109 
2110 	if (tstat_running) {
2111 		mutex_exit(&tstat_lock);
2112 		return (EBUSY);
2113 	}
2114 
2115 	tstat_options |= option;
2116 	mutex_exit(&tstat_lock);
2117 
2118 	return (0);
2119 }
2120 
2121 /*ARGSUSED*/
2122 static int
2123 trapstat_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *crd, int *rval)
2124 {
2125 	int i, j, out;
2126 	size_t dsize;
2127 
2128 	switch (cmd) {
2129 	case TSTATIOC_GO:
2130 		return (trapstat_go());
2131 
2132 	case TSTATIOC_NOGO:
2133 		return (trapstat_option(TSTAT_OPT_NOGO));
2134 
2135 	case TSTATIOC_STOP:
2136 		return (trapstat_stop());
2137 
2138 	case TSTATIOC_CPU:
2139 		if (arg < 0 || arg > max_cpuid)
2140 			return (EINVAL);
2141 		/*FALLTHROUGH*/
2142 
2143 	case TSTATIOC_NOCPU:
2144 		mutex_enter(&tstat_lock);
2145 
2146 		if (tstat_running) {
2147 			mutex_exit(&tstat_lock);
2148 			return (EBUSY);
2149 		}
2150 
2151 		/*
2152 		 * If this is the first CPU to be specified (or if we are
2153 		 * being asked to explicitly de-select CPUs), disable all CPUs.
2154 		 */
2155 		if (!(tstat_options & TSTAT_OPT_CPU) || cmd == TSTATIOC_NOCPU) {
2156 			tstat_options |= TSTAT_OPT_CPU;
2157 
2158 			for (i = 0; i <= max_cpuid; i++) {
2159 				tstat_percpu_t *tcpu = &tstat_percpu[i];
2160 
2161 				ASSERT(cmd == TSTATIOC_NOCPU ||
2162 				    (tcpu->tcpu_flags & TSTAT_CPU_SELECTED));
2163 				tcpu->tcpu_flags &= ~TSTAT_CPU_SELECTED;
2164 			}
2165 		}
2166 
2167 		if (cmd == TSTATIOC_CPU)
2168 			tstat_percpu[arg].tcpu_flags |= TSTAT_CPU_SELECTED;
2169 
2170 		mutex_exit(&tstat_lock);
2171 
2172 		return (0);
2173 
2174 	case TSTATIOC_ENTRY:
2175 		mutex_enter(&tstat_lock);
2176 
2177 		if (tstat_running) {
2178 			mutex_exit(&tstat_lock);
2179 			return (EBUSY);
2180 		}
2181 
2182 		if (arg >= TSTAT_NENT || arg < 0) {
2183 			mutex_exit(&tstat_lock);
2184 			return (EINVAL);
2185 		}
2186 
2187 		if (!(tstat_options & TSTAT_OPT_ENTRY)) {
2188 			/*
2189 			 * If this is the first entry that we are explicitly
2190 			 * enabling, explicitly disable every TL=0 entry.
2191 			 */
2192 			for (i = 0; i < TSTAT_NENT; i++)
2193 				tstat_enabled[i] = 0;
2194 
2195 			tstat_options |= TSTAT_OPT_ENTRY;
2196 		}
2197 
2198 		tstat_enabled[arg] = 1;
2199 		mutex_exit(&tstat_lock);
2200 		return (0);
2201 
2202 	case TSTATIOC_NOENTRY:
2203 		mutex_enter(&tstat_lock);
2204 
2205 		if (tstat_running) {
2206 			mutex_exit(&tstat_lock);
2207 			return (EBUSY);
2208 		}
2209 
2210 		for (i = 0; i < TSTAT_NENT; i++)
2211 			tstat_enabled[i] = 0;
2212 
2213 		mutex_exit(&tstat_lock);
2214 		return (0);
2215 
2216 	case TSTATIOC_READ:
2217 		mutex_enter(&tstat_lock);
2218 
2219 		if (tstat_options & TSTAT_OPT_TLBDATA) {
2220 			dsize = tstat_data_t_exported_size;
2221 		} else {
2222 			dsize = sizeof (tstat_data_t);
2223 		}
2224 
2225 		for (i = 0, out = 0; i <= max_cpuid; i++) {
2226 			tstat_percpu_t *tcpu = &tstat_percpu[i];
2227 
2228 			if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED))
2229 				continue;
2230 
2231 			ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
2232 			ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
2233 
2234 			tstat_buffer->tdata_cpuid = -1;
2235 			xc_one(i, (xcfunc_t *)trapstat_snapshot, 0, 0);
2236 
2237 			if (tstat_buffer->tdata_cpuid == -1) {
2238 				/*
2239 				 * This CPU is not currently responding to
2240 				 * cross calls; we have caught it while it is
2241 				 * being unconfigured.  We'll drop tstat_lock
2242 				 * and pick up and drop cpu_lock.  By the
2243 				 * time we acquire cpu_lock, the DR operation
2244 				 * will appear consistent and we can assert
2245 				 * that trapstat_cpu_setup() has cleared
2246 				 * TSTAT_CPU_ENABLED.
2247 				 */
2248 				mutex_exit(&tstat_lock);
2249 				mutex_enter(&cpu_lock);
2250 				mutex_exit(&cpu_lock);
2251 				mutex_enter(&tstat_lock);
2252 				ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
2253 				continue;
2254 			}
2255 
2256 			/*
2257 			 * Need to compensate for the difference between page
2258 			 * sizes exported to users and page sizes available
2259 			 * within the kernel.
2260 			 */
2261 			if ((tstat_options & TSTAT_OPT_TLBDATA) &&
2262 			    (tstat_pgszs != tstat_user_pgszs)) {
2263 				tstat_pgszdata_t *tp;
2264 				uint_t szc;
2265 
2266 				tp = &tstat_buffer->tdata_pgsz[0];
2267 				for (j = 0; j < tstat_user_pgszs; j++) {
2268 					if ((szc = USERSZC_2_SZC(j)) != j) {
2269 						bcopy(&tp[szc], &tp[j],
2270 						    sizeof (tstat_pgszdata_t));
2271 					}
2272 				}
2273 			}
2274 
2275 			if (copyout(tstat_buffer, (void *)arg, dsize) != 0) {
2276 				mutex_exit(&tstat_lock);
2277 				return (EFAULT);
2278 			}
2279 
2280 			out++;
2281 			arg += dsize;
2282 		}
2283 
2284 		if (out != max_cpuid + 1) {
2285 			processorid_t cpuid = -1;
2286 			arg += offsetof(tstat_data_t, tdata_cpuid);
2287 
2288 			if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) {
2289 				mutex_exit(&tstat_lock);
2290 				return (EFAULT);
2291 			}
2292 		}
2293 
2294 		mutex_exit(&tstat_lock);
2295 
2296 		return (0);
2297 
2298 	case TSTATIOC_TLBDATA:
2299 		return (trapstat_option(TSTAT_OPT_TLBDATA));
2300 
2301 	default:
2302 		break;
2303 	}
2304 
2305 	return (ENOTTY);
2306 }
2307 
2308 /*ARGSUSED*/
2309 static int
2310 trapstat_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2311 {
2312 	int error;
2313 
2314 	switch (infocmd) {
2315 	case DDI_INFO_DEVT2DEVINFO:
2316 		*result = (void *)tstat_devi;
2317 		error = DDI_SUCCESS;
2318 		break;
2319 	case DDI_INFO_DEVT2INSTANCE:
2320 		*result = (void *)0;
2321 		error = DDI_SUCCESS;
2322 		break;
2323 	default:
2324 		error = DDI_FAILURE;
2325 	}
2326 	return (error);
2327 }
2328 
2329 static int
2330 trapstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2331 {
2332 	switch (cmd) {
2333 	case DDI_ATTACH:
2334 		break;
2335 
2336 	case DDI_RESUME:
2337 		return (DDI_SUCCESS);
2338 
2339 	default:
2340 		return (DDI_FAILURE);
2341 	}
2342 
2343 	if (ddi_create_minor_node(devi, "trapstat", S_IFCHR,
2344 	    0, DDI_PSEUDO, 0) == DDI_FAILURE) {
2345 		ddi_remove_minor_node(devi, NULL);
2346 		return (DDI_FAILURE);
2347 	}
2348 
2349 	ddi_report_dev(devi);
2350 	tstat_devi = devi;
2351 
2352 	tstat_pgszs = page_num_pagesizes();
2353 	tstat_user_pgszs = page_num_user_pagesizes(0);
2354 	tstat_data_t_size = sizeof (tstat_data_t) +
2355 	    (tstat_pgszs - 1) * sizeof (tstat_pgszdata_t);
2356 	tstat_data_t_exported_size = sizeof (tstat_data_t) +
2357 	    (tstat_user_pgszs - 1) * sizeof (tstat_pgszdata_t);
2358 #ifndef sun4v
2359 	tstat_data_pages = (tstat_data_t_size >> MMU_PAGESHIFT) + 1;
2360 	tstat_total_pages = TSTAT_INSTR_PAGES + tstat_data_pages;
2361 	tstat_data_size = tstat_data_pages * MMU_PAGESIZE;
2362 	tstat_total_size = TSTAT_INSTR_SIZE + tstat_data_size;
2363 #else
2364 	ASSERT(tstat_data_t_size <= TSTAT_DATA_SIZE);
2365 #endif
2366 
2367 	tstat_percpu = kmem_zalloc((max_cpuid + 1) *
2368 	    sizeof (tstat_percpu_t), KM_SLEEP);
2369 
2370 	/*
2371 	 * Create our own arena backed by segkmem to assure a source of
2372 	 * MMU_PAGESIZE-aligned allocations.  We allocate out of the
2373 	 * heap32_arena to assure that we can address the allocated memory with
2374 	 * a single sethi/simm13 pair in the interposing trap table entries.
2375 	 */
2376 	tstat_arena = vmem_create("trapstat", NULL, 0, MMU_PAGESIZE,
2377 	    segkmem_alloc_permanent, segkmem_free, heap32_arena, 0, VM_SLEEP);
2378 
2379 	tstat_enabled = kmem_alloc(TSTAT_TOTAL_NENT * sizeof (int), KM_SLEEP);
2380 	tstat_buffer = kmem_alloc(tstat_data_t_size, KM_SLEEP);
2381 
2382 	/*
2383 	 * CB_CL_CPR_POST_USER is the class that executes from cpr_resume()
2384 	 * after user threads can be restarted.  By executing in this class,
2385 	 * we are assured of the availability of system services needed to
2386 	 * resume trapstat (specifically, we are assured that all CPUs are
2387 	 * restarted and responding to cross calls).
2388 	 */
2389 	tstat_cprcb =
2390 	    callb_add(trapstat_cpr, NULL, CB_CL_CPR_POST_USER, "trapstat");
2391 
2392 	return (DDI_SUCCESS);
2393 }
2394 
2395 static int
2396 trapstat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2397 {
2398 	int rval;
2399 
2400 	ASSERT(devi == tstat_devi);
2401 
2402 	switch (cmd) {
2403 	case DDI_DETACH:
2404 		break;
2405 
2406 	case DDI_SUSPEND:
2407 		return (DDI_SUCCESS);
2408 
2409 	default:
2410 		return (DDI_FAILURE);
2411 	}
2412 
2413 	ASSERT(!tstat_running);
2414 
2415 	rval = callb_delete(tstat_cprcb);
2416 	ASSERT(rval == 0);
2417 
2418 	kmem_free(tstat_buffer, tstat_data_t_size);
2419 	kmem_free(tstat_enabled, TSTAT_TOTAL_NENT * sizeof (int));
2420 	vmem_destroy(tstat_arena);
2421 	kmem_free(tstat_percpu, (max_cpuid + 1) * sizeof (tstat_percpu_t));
2422 	ddi_remove_minor_node(devi, NULL);
2423 
2424 	return (DDI_SUCCESS);
2425 }
2426 
2427 /*
2428  * Configuration data structures
2429  */
2430 static struct cb_ops trapstat_cb_ops = {
2431 	trapstat_open,		/* open */
2432 	trapstat_close,		/* close */
2433 	nulldev,		/* strategy */
2434 	nulldev,		/* print */
2435 	nodev,			/* dump */
2436 	nodev,			/* read */
2437 	nodev,			/* write */
2438 	trapstat_ioctl,		/* ioctl */
2439 	nodev,			/* devmap */
2440 	nodev,			/* mmap */
2441 	nodev,			/* segmap */
2442 	nochpoll,		/* poll */
2443 	ddi_prop_op,		/* cb_prop_op */
2444 	0,			/* streamtab */
2445 	D_MP | D_NEW		/* Driver compatibility flag */
2446 };
2447 
2448 static struct dev_ops trapstat_ops = {
2449 	DEVO_REV,		/* devo_rev, */
2450 	0,			/* refcnt */
2451 	trapstat_info,		/* getinfo */
2452 	nulldev,		/* identify */
2453 	nulldev,		/* probe */
2454 	trapstat_attach,	/* attach */
2455 	trapstat_detach,	/* detach */
2456 	nulldev,		/* reset */
2457 	&trapstat_cb_ops,	/* cb_ops */
2458 	(struct bus_ops *)0,	/* bus_ops */
2459 };
2460 
2461 static struct modldrv modldrv = {
2462 	&mod_driverops,		/* Type of module.  This one is a driver */
2463 	"Trap Statistics",	/* name of module */
2464 	&trapstat_ops,		/* driver ops */
2465 };
2466 
2467 static struct modlinkage modlinkage = {
2468 	MODREV_1, (void *)&modldrv, NULL
2469 };
2470 
2471 int
2472 _init(void)
2473 {
2474 	return (mod_install(&modlinkage));
2475 }
2476 
2477 int
2478 _fini(void)
2479 {
2480 	return (mod_remove(&modlinkage));
2481 }
2482 
2483 int
2484 _info(struct modinfo *modinfop)
2485 {
2486 	return (mod_info(&modlinkage, modinfop));
2487 }
2488