1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/systm.h> 29 #include <sys/conf.h> 30 #include <sys/stat.h> 31 #include <sys/ddi.h> 32 #include <sys/sunddi.h> 33 #include <sys/modctl.h> 34 #include <sys/cpu_module.h> 35 #include <vm/hat_sfmmu.h> 36 #include <vm/seg_kmem.h> 37 #include <vm/seg_kpm.h> 38 #include <vm/vm_dep.h> 39 #include <sys/machsystm.h> 40 #include <sys/machasi.h> 41 #include <sys/sysmacros.h> 42 #include <sys/callb.h> 43 #include <sys/archsystm.h> 44 #include <sys/trapstat.h> 45 #ifdef sun4v 46 #include <sys/hypervisor_api.h> 47 #endif 48 #ifndef sun4v 49 #include <sys/chip.h> 50 #endif 51 52 /* BEGIN CSTYLED */ 53 /* 54 * trapstat: Trap Statistics through Dynamic Trap Table Interposition 55 * ------------------------------------------------------------------- 56 * 57 * Motivation and Overview 58 * 59 * Despite being a fundamental indicator of system behavior, there has 60 * historically been very little insight provided into the frequency and cost 61 * of machine-specific traps. The lack of insight has been especially acute 62 * on UltraSPARC microprocessors: because these microprocessors handle TLB 63 * misses as software traps, the frequency and duration of traps play a 64 * decisive role in the performance of the memory system. As applications have 65 * increasingly outstripped TLB reach, this has become increasingly true. 66 * 67 * Part of the difficulty of observing trap behavior is that the trap handlers 68 * are so frequently called (e.g. millions of times per second) that any 69 * permanently enabled instrumentation would induce an unacceptable performance 70 * degradation. Thus, it is a constraint on any trap observability 71 * infrastructure that it have no probe effect when not explicitly enabled. 72 * 73 * The basic idea, then, is to create an interposing trap table in which each 74 * entry increments a per-trap, in-memory counter and then jumps to the actual, 75 * underlying trap table entry. To enable trapstat, we atomically write to the 76 * trap base address (%tba) register to point to our interposing trap table. 77 * (Note that per-CPU statistics fall out by creating a different trap table 78 * for each CPU.) 79 * 80 * Implementation Details 81 * 82 * While the idea is straight-forward, a nuance of SPARC V9 slightly 83 * complicates the implementation. Unlike its predecessors, SPARC V9 supports 84 * the notion of nested traps. The trap level is kept in the TL register: 85 * during normal operation it is 0; when a trap is taken, the TL register is 86 * incremented by 1. To aid system software, SPARC V9 breaks the trap table 87 * into two halves: the lower half contains the trap handlers for traps taken 88 * when TL is 0; the upper half contains the trap handlers for traps taken 89 * when TL is greater than 0. Each half is further subdivided into two 90 * subsequent halves: the lower half contains the trap handlers for traps 91 * other than those induced by the trap instruction (Tcc variants); the upper 92 * half contains the trap handlers for traps induced by the trap instruction. 93 * This gives a total of four ranges, with each range containing 256 traps: 94 * 95 * +--------------------------------+- 3ff 96 * | | . 97 * | Trap instruction, TL>0 | . 98 * | | . 99 * |- - - - - - - - - - - - - - - - +- 300 100 * |- - - - - - - - - - - - - - - - +- 2ff 101 * | | . 102 * | Non-trap instruction, TL>0 | . 103 * | | . 104 * |- - - - - - - - - - - - - - - - +- 200 105 * |- - - - - - - - - - - - - - - - +- 1ff 106 * | | . 107 * | Trap instruction, TL=0 | . 108 * | | . 109 * |- - - - - - - - - - - - - - - - +- 100 110 * |- - - - - - - - - - - - - - - - +- 0ff 111 * | | . 112 * | Non-trap instruction, TL=0 | . 113 * | | . 114 * +--------------------------------+- 000 115 * 116 * 117 * Solaris, however, doesn't have reason to support trap instructions when 118 * TL>0 (only privileged code may execute at TL>0; not supporting this only 119 * constrains our own implementation). The trap table actually looks like: 120 * 121 * +--------------------------------+- 2ff 122 * | | . 123 * | Non-trap instruction, TL>0 | . 124 * | | . 125 * |- - - - - - - - - - - - - - - - +- 200 126 * |- - - - - - - - - - - - - - - - +- 1ff 127 * | | . 128 * | Trap instruction, TL=0 | . 129 * | | . 130 * |- - - - - - - - - - - - - - - - +- 100 131 * |- - - - - - - - - - - - - - - - +- 0ff 132 * | | . 133 * | Non-trap instruction, TL=0 | . 134 * | | . 135 * +--------------------------------+- 000 136 * 137 * Putatively to aid system software, SPARC V9 has the notion of multiple 138 * sets of global registers. UltraSPARC defines four sets of global 139 * registers: 140 * 141 * Normal Globals 142 * Alternate Globals (AGs) 143 * MMU Globals (MGs) 144 * Interrupt Globals (IGs) 145 * 146 * The set of globals in use is controlled by bits in PSTATE; when TL is 0 147 * (and PSTATE has not been otherwise explicitly modified), the Normal Globals 148 * are in use. When a trap is issued, PSTATE is modified to point to a set of 149 * globals corresponding to the trap type. Most traps correspond to the 150 * Alternate Globals, with a minority corresponding to the MMU Globals, and 151 * only the interrupt-vector trap (vector 0x60) corresponding to the Interrupt 152 * Globals. (The complete mapping can be found in the UltraSPARC I&II User's 153 * Manual.) 154 * 155 * Note that the sets of globals are per trap _type_, not per trap _level_. 156 * Thus, when executing a TL>0 trap handler, one may not have registers 157 * available (for example, both trap-instruction traps and spill traps execute 158 * on the alternate globals; if a trap-instruction trap induces a window spill, 159 * the window spill handler has no available globals). For trapstat, this is 160 * problematic: a register is required to transfer control from one arbitrary 161 * location (in the interposing trap table) to another (in the actual trap 162 * table). 163 * 164 * We solve this problem by exploiting the trap table's location at the bottom 165 * of valid kernel memory (i.e. at KERNELBASE). We locate the interposing trap 166 * tables just below KERNELBASE -- thereby allowing us to use a branch-always 167 * instruction (ba) instead of a jump instruction (jmp) to transfer control 168 * from the TL>0 entries in the interposing trap table to the TL>0 entries in 169 * the actual trap table. (N.B. while this allows trap table interposition to 170 * work, it necessarily limits trapstat to only recording information about 171 * TL=0 traps -- there is no way to increment a counter without using a 172 * register.) Diagrammatically: 173 * 174 * Actual trap table: 175 * 176 * +--------------------------------+- 2ff 177 * | | . 178 * | Non-trap instruction, TL>0 | . <-----------------------+ 179 * | | . <-----------------------|-+ 180 * |- - - - - - - - - - - - - - - - +- 200 <-----------------------|-|-+ 181 * |- - - - - - - - - - - - - - - - +- 1ff | | | 182 * | | . | | | 183 * | Trap instruction, TL=0 | . <-----------------+ | | | 184 * | | . <-----------------|-+ | | | 185 * |- - - - - - - - - - - - - - - - +- 100 <-----------------|-|-+ | | | 186 * |- - - - - - - - - - - - - - - - +- 0ff | | | | | | 187 * | | . | | | | | | 188 * | Non-trap instruction, TL=0 | . <-----------+ | | | | | | 189 * | | . <-----------|-+ | | | | | | 190 * +--------------------------------+- 000 <-----------|-|-+ | | | | | | 191 * KERNELBASE | | | | | | | | | 192 * | | | | | | | | | 193 * | | | | | | | | | 194 * Interposing trap table: | | | | | | | | | 195 * | | | | | | | | | 196 * +--------------------------------+- 2ff | | | | | | | | | 197 * | ... | . | | | | | | | | | 198 * | ... | . | | | | | | | | | 199 * | ... | . | | | | | | | | | 200 * |- - - - - - - - - - - - - - - - +- 203 | | | | | | | | | 201 * | ba,a | -------------|-|-|-|-|-|-+ | | 202 * |- - - - - - - - - - - - - - - - +- 202 | | | | | | | | 203 * | ba,a | -------------|-|-|-|-|-|---+ | 204 * |- - - - - - - - - - - - - - - - +- 201 | | | | | | | 205 * | ba,a | -------------|-|-|-|-|-|-----+ 206 * |- - - - - - - - - - - - - - - - +- 200 | | | | | | 207 * | ... | . | | | | | | 208 * | ... | . | | | | | | 209 * | ... | . | | | | | | 210 * |- - - - - - - - - - - - - - - - +- 103 | | | | | | 211 * | (Increment counter) | | | | | | | 212 * | ba,a | -------------------+ | | 213 * |- - - - - - - - - - - - - - - - +- 102 | | | | | 214 * | (Increment counter) | | | | | | 215 * | ba,a | ---------------------+ | 216 * |- - - - - - - - - - - - - - - - +- 101 | | | | 217 * | (Increment counter) | | | | | 218 * | ba,a | -----------------------+ 219 * |- - - - - - - - - - - - - - - - +- 100 | | | 220 * | ... | . | | | 221 * | ... | . | | | 222 * | ... | . | | | 223 * |- - - - - - - - - - - - - - - - +- 003 | | | 224 * | (Increment counter) | | | | 225 * | ba,a | -------------+ | | 226 * |- - - - - - - - - - - - - - - - +- 002 | | 227 * | (Increment counter) | | | 228 * | ba,a | ---------------+ | 229 * |- - - - - - - - - - - - - - - - +- 001 | 230 * | (Increment counter) | | 231 * | ba,a | -----------------+ 232 * +--------------------------------+- 000 233 * KERNELBASE - tstat_total_size 234 * 235 * tstat_total_size is the number of pages required for each trap table. It 236 * must be true that KERNELBASE - tstat_total_size is less than the maximum 237 * branch displacement; if each CPU were to consume a disjoint virtual range 238 * below KERNELBASE for its trap table, we could support at most 239 * (maximum_branch_displacement / tstat_total_size) CPUs. The maximum branch 240 * displacement for Bicc variants is just under eight megabytes, and (because 241 * the %tba must be 32K aligned), tstat_total_size must be at least 32K; if 242 * each CPU were to consume a disjoint virtual range, we would have an 243 * unacceptably low upper bound of 256 CPUs. 244 * 245 * While there are tricks that one could use to address this constraint (e.g., 246 * creating trampolines every maximum_branch_displacement bytes), we instead 247 * solve this by not permitting each CPU to consume a disjoint virtual range. 248 * Rather, we have each CPU's interposing trap table use the _same_ virtual 249 * range, but we back the trap tables with disjoint physical memory. Normally, 250 * such one-to-many virtual-to-physical mappings are illegal; this is 251 * permissible here only because the pages for the interposing trap table are 252 * necessarily locked in the TLB. (The CPUs thus never have the opportunity to 253 * discover that they have conflicting translations.) 254 * 255 * On CMT architectures in which CPUs can share MMUs, the above trick will not 256 * work: two CPUs that share an MMU cannot have the same virtual address map 257 * to disjoint physical pages. On these architectures, any CPUs sharing the 258 * same MMU must consume a disjoint 32K virtual address range -- limiting the 259 * number of CPUs sharing an MMU on these architectures to 256 due to the 260 * branch displacement limitation described above. On the sun4v architecture, 261 * there is a further limitation: a guest may not have more than eight locked 262 * TLB entries per MMU. To allow operation under this restriction, the 263 * interposing trap table and the trap statistics are each accessed through 264 * a single 4M TLB entry. This limits the footprint to two locked entries 265 * (one for the I-TLB and one for the D-TLB), but further restricts the number 266 * of CPUs to 128 per MMU. However, support for more than 128 CPUs can easily 267 * be added via a hybrid scheme, where the same 4M virtual address is used 268 * on different MMUs. 269 * 270 * 271 * TLB Statistics 272 * 273 * Because TLB misses are an important component of system performance, we wish 274 * to know much more about these traps than simply the number received. 275 * Specifically, we wish to know: 276 * 277 * (a) The amount of time spent executing the TLB miss handler 278 * (b) TLB misses versus TSB misses 279 * (c) Kernel-level misses versus user-level misses 280 * (d) Misses per pagesize 281 * 282 * TLB Statistics: Time Spent Executing 283 * 284 * To accurately determine the amount of time spent executing the TLB miss 285 * handler, one must get a timestamp on trap entry and trap exit, subtract the 286 * latter from the former, and add the result to an accumulating count. 287 * Consider flow of control during normal TLB miss processing (where "ldx 288 * [%g2], %g2" is an arbitrary TLB-missing instruction): 289 * 290 * + - - - - - - - -+ 291 * : : 292 * : ldx [%g2], %g2 :<-------------------------------------------------------+ 293 * : : Return from trap: | 294 * + - - - - - - - -+ TL <- TL - 1 (0) | 295 * | %pc <- TSTATE[TL].TPC (address of load) | 296 * | TLB miss: | 297 * | TL <- TL + 1 (1) | 298 * | %pc <- TLB-miss-trap-handler | 299 * | | 300 * v | 301 * + - - - - - - - - - - - - - - - + | 302 * : : | 303 * : Lookup VA in TSB : | 304 * : If (hit) : | 305 * : Fill TLB : | 306 * : Else : | 307 * : Lookup VA (hme hash table : | 308 * : or segkpm) : | 309 * : Fill TLB : | 310 * : Endif : | 311 * : Issue "retry" ---------------------------------------------------------+ 312 * : : 313 * + - - - - - - - - - - - - - - - + 314 * TLB-miss-trap-handler 315 * 316 * 317 * As the above diagram indicates, interposing on the trap table allows one 318 * only to determine a timestamp on trap _entry_: when the TLB miss handler 319 * has completed filling the TLB, a "retry" will be issued, and control will 320 * transfer immediately back to the missing %pc. 321 * 322 * To obtain a timestamp on trap exit, we must then somehow interpose between 323 * the "retry" and the subsequent control transfer to the TLB-missing 324 * instruction. To do this, we _push_ a trap level. The basic idea is to 325 * spoof a TLB miss by raising TL, setting the %tpc to be within text 326 * controlled by trapstat (the "TLB return entry") and branching to the 327 * underlying TLB miss handler. When the TLB miss handler issues its "retry", 328 * control will transfer not to the TLB-missing instruction, but rather to the 329 * TLB return entry. This code can then obtain a timestamp, and issue its own 330 * "retry" -- thereby correctly returning to the TLB-missing instruction. 331 * Here is the above TLB miss flow control diagram modified to reflect 332 * trapstat's operation: 333 * 334 * + - - - - - - - -+ 335 * : : 336 * : ldx [%g2], %g2 :<-------------------------------------------------------+ 337 * : : Return from trap: | 338 * + - - - - - - - -+ TL <- TL - 1 (0) | 339 * | %pc <- TSTATE[TL].TPC (address of load) | 340 * | TLB miss: | 341 * | TL <- TL + 1 (1) | 342 * | %pc <- TLB-miss-trap-handler (trapstat) | 343 * | | 344 * v TLB-return-entry (trapstat) | 345 * + - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - + | 346 * : : : : | 347 * : Record timestamp : : Record timestamp : | 348 * : TL <- 2 : : Take timestamp difference : | 349 * : TSTATE[1].TPC <- TLB-return-entry : : Add to running total : | 350 * : ba,a TLB-miss-trap-handler -----------+ : Issue "retry" --------------+ 351 * : : | : : 352 * + - - - - - - - - - - - - - - - - - - + | + - - - - - - - - - - - - - + 353 * TLB-miss-trap-handler | ^ 354 * (trapstat) | | 355 * | | 356 * | | 357 * +-----------------------+ | 358 * | | 359 * | | 360 * v | 361 * + - - - - - - - - - - - - - - - + | 362 * : : | 363 * : Lookup VA in TSB : | 364 * : If (hit) : | 365 * : Fill TLB : | 366 * : Else : | 367 * : Lookup VA (hme hash table : | 368 * : or segkpm) : | 369 * : Fill TLB : | 370 * : Endif : | 371 * : Issue "retry" ------------------------------------------+ 372 * : : Return from trap: 373 * + - - - - - - - - - - - - - - - + TL <- TL - 1 (1) 374 * TLB-miss-trap-handler %pc <- TSTATE[TL].TPC (TLB-return-entry) 375 * 376 * 377 * A final subterfuge is required to complete our artifice: if we miss in 378 * the TLB, the TSB _and_ the subsequent hash or segkpm lookup (that is, if 379 * there is no valid translation for the TLB-missing address), common system 380 * software will need to accurately determine the %tpc as part of its page 381 * fault handling. We therefore modify the kernel to check the %tpc in this 382 * case: if the %tpc falls within the VA range controlled by trapstat and 383 * the TL is 2, TL is simply lowered back to 1 (this check is implemented 384 * by the TSTAT_CHECK_TL1 macro). Lowering TL to 1 has the effect of 385 * discarding the state pushed by trapstat. 386 * 387 * TLB Statistics: TLB Misses versus TSB Misses 388 * 389 * Distinguishing TLB misses from TSB misses requires further interposition 390 * on the TLB miss handler: we cannot know a priori or a posteriori if a 391 * given VA will or has hit in the TSB. 392 * 393 * We achieve this distinction by adding a second TLB return entry almost 394 * identical to the first -- differing only in the address to which it 395 * stores its results. We then modify the TLB miss handlers of the kernel 396 * such that they check the %tpc when they determine that a TLB miss has 397 * subsequently missed in the TSB: if the %tpc lies within trapstat's VA 398 * range and TL is 2 (that is, if trapstat is running), the TLB miss handler 399 * _increments_ the %tpc by the size of the TLB return entry. The ensuing 400 * "retry" will thus transfer control to the second TLB return entry, and 401 * the time spent in the handler will be accumulated in a memory location 402 * specific to TSB misses. 403 * 404 * N.B.: To minimize the amount of knowledge the kernel must have of trapstat, 405 * we do not allow the kernel to hard-code the size of the TLB return entry. 406 * Rather, the actual tsbmiss handler executes a known instruction at the 407 * corresponding tsbmiss patch points (see the tstat_tsbmiss_patch_table) with 408 * the %tpc in %g7: when trapstat is not running, these points contain the 409 * harmless TSTAT_TSBMISS_INSTR instruction ("add %g7, 0, %g7"). Before 410 * running, trapstat modifies the instructions at these patch points such 411 * that the simm13 equals the size of the TLB return entry. 412 * 413 * TLB Statistics: Kernel-level Misses versus User-level Misses 414 * 415 * Differentiating user-level misses from kernel-level misses employs a 416 * similar technique, but is simplified by the ability to distinguish a 417 * user-level miss from a kernel-level miss a priori by reading the context 418 * register: we implement kernel-/user-level differentiation by again doubling 419 * the number of TLB return entries, and setting the %tpc to the appropriate 420 * TLB return entry in trapstat's TLB miss handler. Together with the doubling 421 * of entries required for TLB-miss/TSB-miss differentiation, this yields a 422 * total of four TLB return entries: 423 * 424 * Level TSB hit? Structure member 425 * ------------------------------------------------------------ 426 * Kernel Yes tstat_tlbret_t.ttlbr_ktlb 427 * Kernel No tstat_tlbret_t.ttlbr_ktsb 428 * User Yes tstat_tlbret_t.ttlbr_utlb 429 * User No tstat_tlbret_t.ttlbr_utsb 430 * 431 * TLB Statistics: Misses per Pagesize 432 * 433 * As with the TLB-/TSB-miss differentiation, we have no way of determining 434 * pagesize a priori. This is therefore implemented by mandating a new rule: 435 * whenever the kernel fills the TLB in its TLB miss handler, the TTE 436 * corresponding to the TLB-missing VA must be in %g5 when the handler 437 * executes its "retry". This allows the TLB return entry to determine 438 * pagesize by simply looking at the pagesize field in the TTE stored in 439 * %g5. 440 * 441 * TLB Statistics: Probe Effect 442 * 443 * As one might imagine, gathering TLB statistics by pushing a trap level 444 * induces significant probe effect. To account for this probe effect, 445 * trapstat attempts to observe it by executing a code sequence with a known 446 * number of TLB misses both before and after interposing on the trap table. 447 * This allows trapstat to determine a per-trap probe effect which can then be 448 * factored into the "%tim" fields of the trapstat command. 449 * 450 * Note that on sun4v platforms, TLB misses are normally handled by the 451 * hypervisor or the hardware TSB walker. Thus no fast MMU miss information 452 * is reported for normal operation. However, when trapstat is invoked 453 * with -t or -T option to collect detailed TLB statistics, kernel takes 454 * over TLB miss handling. This results in significantly more overhead 455 * and TLB statistics may not be as accurate as on sun4u platforms. 456 * On some processors, hypervisor or hardware may provide a low overhead 457 * interface to collect TSB hit statistics. This support is exposed via 458 * a well defined CPU module interface (cpu_trapstat_conf to enable this 459 * interface and cpu_trapstat_data to get detailed TSB hit statistics). 460 * In this scenario, TSB miss statistics is collected by intercepting the 461 * IMMU_miss and DMMU_miss traps using above mentioned trap interposition 462 * approach. 463 * 464 * Locking 465 * 466 * The implementation uses two locks: tstat_lock (a local lock) and the global 467 * cpu_lock. tstat_lock is used to assure trapstat's consistency in the 468 * presence of multithreaded /dev/trapstat consumers (while as of this writing 469 * the only consumer of /dev/trapstat is single threaded, it is obviously 470 * necessary to correctly support multithreaded access). cpu_lock is held 471 * whenever CPUs are being manipulated directly, to prevent them from 472 * disappearing in the process. Because trapstat's DR callback 473 * (trapstat_cpu_setup()) must grab tstat_lock and is called with cpu_lock 474 * held, the lock ordering is necessarily cpu_lock before tstat_lock. 475 * 476 */ 477 /* END CSTYLED */ 478 479 static dev_info_t *tstat_devi; /* saved in xxattach() for xxinfo() */ 480 static int tstat_open; /* set if driver is open */ 481 static kmutex_t tstat_lock; /* serialize access */ 482 static vmem_t *tstat_arena; /* arena for TLB-locked pages */ 483 static tstat_percpu_t *tstat_percpu; /* per-CPU data */ 484 static int tstat_running; /* set if trapstat is running */ 485 static tstat_data_t *tstat_buffer; /* staging buffer for outgoing data */ 486 static int tstat_options; /* bit-wise indication of options */ 487 static int *tstat_enabled; /* map of enabled trap entries */ 488 static int tstat_tsbmiss_patched; /* tsbmiss patch flag */ 489 static callb_id_t tstat_cprcb; /* CPR callback */ 490 static char *tstat_probe_area; /* VA range used for probe effect */ 491 static caddr_t tstat_probe_phys; /* physical to back above VA */ 492 static hrtime_t tstat_probe_time; /* time spent on probe effect */ 493 static hrtime_t tstat_probe_before[TSTAT_PROBE_NLAPS]; 494 static hrtime_t tstat_probe_after[TSTAT_PROBE_NLAPS]; 495 static uint_t tstat_pgszs; /* # of kernel page sizes */ 496 static uint_t tstat_user_pgszs; /* # of user page sizes */ 497 498 /* 499 * sizeof tstat_data_t + pgsz data for the kernel. For simplicity's sake, when 500 * we collect data, we do it based upon szc, but when we report data back to 501 * userland, we have to do it based upon the userszc which may not match. 502 * So, these two variables are for internal use and exported use respectively. 503 */ 504 static size_t tstat_data_t_size; 505 static size_t tstat_data_t_exported_size; 506 507 static size_t tstat_data_pages; /* number of pages of tstat data */ 508 static size_t tstat_data_size; /* tstat data size in bytes */ 509 static size_t tstat_total_pages; /* #data pages + #instr pages */ 510 static size_t tstat_total_size; /* tstat data size + instr size */ 511 #ifdef sun4v 512 static caddr_t tstat_va; /* VA of memory reserved for TBA */ 513 static pfn_t tstat_pfn; /* PFN of memory reserved for TBA */ 514 static boolean_t tstat_fast_tlbstat = B_FALSE; 515 #endif 516 517 /* 518 * In the above block comment, see "TLB Statistics: TLB Misses versus 519 * TSB Misses" for an explanation of the tsbmiss patch points. 520 */ 521 extern uint32_t tsbmiss_trapstat_patch_point; 522 extern uint32_t tsbmiss_trapstat_patch_point_kpm; 523 extern uint32_t tsbmiss_trapstat_patch_point_kpm_small; 524 525 /* 526 * Trapstat tsbmiss patch table 527 */ 528 tstat_tsbmiss_patch_entry_t tstat_tsbmiss_patch_table[] = { 529 {(uint32_t *)&tsbmiss_trapstat_patch_point, 0}, 530 {(uint32_t *)&tsbmiss_trapstat_patch_point_kpm, 0}, 531 {(uint32_t *)&tsbmiss_trapstat_patch_point_kpm_small, 0}, 532 {(uint32_t *)NULL, 0} 533 }; 534 535 /* 536 * We define some general SPARC-specific constants to allow more readable 537 * relocations. 538 */ 539 #define NOP 0x01000000 540 #define HI22(v) ((uint32_t)(v) >> 10) 541 #define LO10(v) ((uint32_t)(v) & 0x3ff) 542 #define LO12(v) ((uint32_t)(v) & 0xfff) 543 #define DISP22(from, to) \ 544 ((((uintptr_t)(to) - (uintptr_t)(from)) >> 2) & 0x3fffff) 545 #define ASI(asi) ((asi) << 5) 546 547 /* 548 * The interposing trap table must be locked in the I-TLB, and any data 549 * referred to in the interposing trap handler must be locked in the D-TLB. 550 * This function locks these pages in the appropriate TLBs by creating TTEs 551 * from whole cloth, and manually loading them into the TLB. This function is 552 * called from cross call context. 553 * 554 * On sun4v platforms, we use 4M page size mappings to minimize the number 555 * of locked down entries (i.e. permanent mappings). Each CPU uses a 556 * reserved portion of that 4M page for its TBA and data. 557 */ 558 static void 559 trapstat_load_tlb(void) 560 { 561 #ifndef sun4v 562 int i; 563 #else 564 uint64_t ret; 565 #endif 566 tte_t tte; 567 tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id]; 568 caddr_t va = tcpu->tcpu_vabase; 569 570 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 571 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)); 572 573 #ifndef sun4v 574 for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) { 575 tte.tte_inthi = TTE_VALID_INT | TTE_SZ_INT(TTE8K) | 576 TTE_PFN_INTHI(tcpu->tcpu_pfn[i]); 577 if (i < TSTAT_INSTR_PAGES) { 578 tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) | 579 TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT; 580 sfmmu_itlb_ld(va, KCONTEXT, &tte); 581 } else { 582 tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) | 583 TTE_LCK_INT | TTE_CP_INT | TTE_CV_INT | 584 TTE_PRIV_INT | TTE_HWWR_INT; 585 sfmmu_dtlb_ld(va, KCONTEXT, &tte); 586 } 587 } 588 #else /* sun4v */ 589 tte.tte_inthi = TTE_VALID_INT | TTE_PFN_INTHI(tstat_pfn); 590 tte.tte_intlo = TTE_PFN_INTLO(tstat_pfn) | TTE_CP_INT | 591 TTE_CV_INT | TTE_PRIV_INT | TTE_HWWR_INT | 592 TTE_SZ_INTLO(TTE4M); 593 ret = hv_mmu_map_perm_addr(va, KCONTEXT, *(uint64_t *)&tte, 594 MAP_ITLB | MAP_DTLB); 595 596 if (ret != H_EOK) 597 cmn_err(CE_PANIC, "trapstat: cannot map new TBA " 598 "for cpu %d (error: 0x%lx)", CPU->cpu_id, ret); 599 #endif /* sun4v */ 600 } 601 602 /* 603 * As mentioned in the "TLB Statistics: TLB Misses versus TSB Misses" section 604 * of the block comment, TLB misses are differentiated from TSB misses in 605 * part by hot-patching the instructions at the tsbmiss patch points (see 606 * tstat_tsbmiss_patch_table). This routine is used both to initially patch 607 * the instructions, and to patch them back to their original values upon 608 * restoring the original trap table. 609 */ 610 static void 611 trapstat_hotpatch() 612 { 613 uint32_t instr; 614 uint32_t simm13; 615 tstat_tsbmiss_patch_entry_t *ep; 616 617 ASSERT(MUTEX_HELD(&tstat_lock)); 618 619 if (!(tstat_options & TSTAT_OPT_TLBDATA)) 620 return; 621 622 if (!tstat_tsbmiss_patched) { 623 /* 624 * We haven't patched the TSB paths; do so now. 625 */ 626 /*CONSTCOND*/ 627 ASSERT(offsetof(tstat_tlbret_t, ttlbr_ktsb) - 628 offsetof(tstat_tlbret_t, ttlbr_ktlb) == 629 offsetof(tstat_tlbret_t, ttlbr_utsb) - 630 offsetof(tstat_tlbret_t, ttlbr_utlb)); 631 632 simm13 = offsetof(tstat_tlbret_t, ttlbr_ktsb) - 633 offsetof(tstat_tlbret_t, ttlbr_ktlb); 634 635 for (ep = tstat_tsbmiss_patch_table; ep->tpe_addr; ep++) { 636 ASSERT(ep->tpe_instr == 0); 637 instr = ep->tpe_instr = *ep->tpe_addr; 638 639 /* 640 * Assert that the instruction we're about to patch is 641 * "add %g7, 0, %g7" (0x8e01e000). 642 */ 643 ASSERT(instr == TSTAT_TSBMISS_INSTR); 644 645 instr |= simm13; 646 hot_patch_kernel_text((caddr_t)ep->tpe_addr, 647 instr, sizeof (instr)); 648 } 649 650 tstat_tsbmiss_patched = 1; 651 652 } else { 653 /* 654 * Remove patches from the TSB paths. 655 */ 656 for (ep = tstat_tsbmiss_patch_table; ep->tpe_addr; ep++) { 657 ASSERT(ep->tpe_instr == TSTAT_TSBMISS_INSTR); 658 hot_patch_kernel_text((caddr_t)ep->tpe_addr, 659 ep->tpe_instr, sizeof (instr)); 660 ep->tpe_instr = 0; 661 } 662 663 tstat_tsbmiss_patched = 0; 664 } 665 } 666 667 /* 668 * This is the routine executed to clock the performance of the trap table, 669 * executed both before and after interposing on the trap table to attempt to 670 * determine probe effect. The probe effect is used to adjust the "%tim" 671 * fields of trapstat's -t and -T output; we only use TLB misses to clock the 672 * trap table. We execute the inner loop (which is designed to exceed the 673 * TLB's reach) nlaps times, taking the best time as our time (thereby 674 * factoring out the effects of interrupts, cache misses or other perturbing 675 * events. 676 */ 677 static hrtime_t 678 trapstat_probe_laps(int nlaps, hrtime_t *buf) 679 { 680 int i, j = 0; 681 hrtime_t ts, best = INT64_MAX; 682 683 while (nlaps--) { 684 ts = rdtick(); 685 686 for (i = 0; i < TSTAT_PROBE_SIZE; i += MMU_PAGESIZE) 687 *((volatile char *)&tstat_probe_area[i]); 688 689 if ((ts = rdtick() - ts) < best) 690 best = ts; 691 buf[j++] = ts; 692 } 693 694 return (best); 695 } 696 697 /* 698 * This routine determines the probe effect by calling trapstat_probe_laps() 699 * both without and with the interposing trap table. Note that this is 700 * called from a cross call on the desired CPU, and that it is called on 701 * every CPU (this is necessary because the probe effect may differ from 702 * one CPU to another). 703 */ 704 static void 705 trapstat_probe() 706 { 707 tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id]; 708 hrtime_t before, after; 709 710 if (!(tcpu->tcpu_flags & TSTAT_CPU_SELECTED)) 711 return; 712 713 if (tstat_probe_area == NULL || (tstat_options & TSTAT_OPT_NOGO)) 714 return; 715 716 /* 717 * We very much expect the %tba to be KERNELBASE; this is a 718 * precautionary measure to assure that trapstat doesn't melt the 719 * machine should the %tba point unexpectedly elsewhere. 720 */ 721 if (get_tba() != (caddr_t)KERNELBASE) 722 return; 723 724 /* 725 * Preserve this CPU's data before destroying it by enabling the 726 * interposing trap table. We can safely use tstat_buffer because 727 * the caller of the trapstat_probe() cross call is holding tstat_lock. 728 */ 729 bcopy(tcpu->tcpu_data, tstat_buffer, tstat_data_t_size); 730 731 tstat_probe_time = gethrtime(); 732 733 before = trapstat_probe_laps(TSTAT_PROBE_NLAPS, tstat_probe_before); 734 (void) set_tba(tcpu->tcpu_ibase); 735 736 after = trapstat_probe_laps(TSTAT_PROBE_NLAPS, tstat_probe_after); 737 (void) set_tba((caddr_t)KERNELBASE); 738 739 tstat_probe_time = gethrtime() - tstat_probe_time; 740 741 bcopy(tstat_buffer, tcpu->tcpu_data, tstat_data_t_size); 742 tcpu->tcpu_data->tdata_peffect = (after - before) / TSTAT_PROBE_NPAGES; 743 } 744 745 static void 746 trapstat_probe_alloc() 747 { 748 pfn_t pfn; 749 caddr_t va; 750 int i; 751 752 ASSERT(MUTEX_HELD(&tstat_lock)); 753 ASSERT(tstat_probe_area == NULL); 754 ASSERT(tstat_probe_phys == NULL); 755 756 if (!(tstat_options & TSTAT_OPT_TLBDATA)) 757 return; 758 759 /* 760 * Grab some virtual from the heap arena. 761 */ 762 tstat_probe_area = vmem_alloc(heap_arena, TSTAT_PROBE_SIZE, VM_SLEEP); 763 va = tstat_probe_area; 764 765 /* 766 * Grab a single physical page. 767 */ 768 tstat_probe_phys = vmem_alloc(tstat_arena, MMU_PAGESIZE, VM_SLEEP); 769 pfn = hat_getpfnum(kas.a_hat, tstat_probe_phys); 770 771 /* 772 * Now set the translation for every page in our virtual range 773 * to be our allocated physical page. 774 */ 775 for (i = 0; i < TSTAT_PROBE_NPAGES; i++) { 776 hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, PROT_READ, 777 HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); 778 va += MMU_PAGESIZE; 779 } 780 } 781 782 static void 783 trapstat_probe_free() 784 { 785 caddr_t va; 786 int i; 787 788 ASSERT(MUTEX_HELD(&tstat_lock)); 789 790 if ((va = tstat_probe_area) == NULL) 791 return; 792 793 for (i = 0; i < TSTAT_PROBE_NPAGES; i++) { 794 hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD_UNLOCK); 795 va += MMU_PAGESIZE; 796 } 797 798 vmem_free(tstat_arena, tstat_probe_phys, MMU_PAGESIZE); 799 vmem_free(heap_arena, tstat_probe_area, TSTAT_PROBE_SIZE); 800 801 tstat_probe_phys = NULL; 802 tstat_probe_area = NULL; 803 } 804 805 /* 806 * This routine actually enables a CPU by setting its %tba to be the 807 * CPU's interposing trap table. It is called out of cross call context. 808 */ 809 static void 810 trapstat_enable() 811 { 812 tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id]; 813 814 if (!(tcpu->tcpu_flags & TSTAT_CPU_SELECTED)) 815 return; 816 817 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 818 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)); 819 820 if (get_tba() != (caddr_t)KERNELBASE) 821 return; 822 823 if (!(tstat_options & TSTAT_OPT_NOGO)) 824 (void) set_tba(tcpu->tcpu_ibase); 825 tcpu->tcpu_flags |= TSTAT_CPU_ENABLED; 826 #ifdef sun4v 827 if ((tstat_options & TSTAT_OPT_TLBDATA) && 828 !(tstat_options & TSTAT_OPT_NOGO)) { 829 if (tstat_fast_tlbstat) { 830 /* 831 * Invoke processor specific interface to enable 832 * collection of TSB hit statistics. 833 */ 834 cpu_trapstat_conf(CPU_TSTATCONF_ENABLE); 835 } else { 836 /* 837 * Collect TLB miss statistics by taking over 838 * TLB miss handling from the hypervisor. This 839 * is done by telling the hypervisor that there 840 * is no TSB configured. Also set TSTAT_TLB_STATS 841 * flag so that no user TSB is configured during 842 * context switch time. 843 */ 844 cpu_t *cp = CPU; 845 846 cp->cpu_m.cpu_tstat_flags |= TSTAT_TLB_STATS; 847 (void) hv_set_ctx0(NULL, NULL); 848 (void) hv_set_ctxnon0(NULL, NULL); 849 } 850 } 851 #endif 852 } 853 854 /* 855 * This routine disables a CPU (vis a vis trapstat) by setting its %tba to be 856 * the actual, underlying trap table. It is called out of cross call context. 857 */ 858 static void 859 trapstat_disable() 860 { 861 tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id]; 862 863 if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)) 864 return; 865 866 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED); 867 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 868 869 if (!(tstat_options & TSTAT_OPT_NOGO)) 870 (void) set_tba((caddr_t)KERNELBASE); 871 872 tcpu->tcpu_flags &= ~TSTAT_CPU_ENABLED; 873 874 #ifdef sun4v 875 if ((tstat_options & TSTAT_OPT_TLBDATA) && 876 !(tstat_options & TSTAT_OPT_NOGO)) { 877 if (tstat_fast_tlbstat) { 878 /* 879 * Invoke processor specific interface to disable 880 * collection of TSB hit statistics on each processor. 881 */ 882 cpu_trapstat_conf(CPU_TSTATCONF_DISABLE); 883 } else { 884 /* 885 * As part of collecting TLB miss statistics, we took 886 * over TLB miss handling from the hypervisor by 887 * telling the hypervisor that NO TSB is configured. 888 * We need to restore that by communicating proper 889 * kernel/user TSB information so that TLB misses 890 * can be handled by the hypervisor or the hardware 891 * more efficiently. 892 * 893 * We restore kernel TSB information right away. 894 * However, to minimize any locking dependency, we 895 * don't restore user TSB information right away. 896 * Instead, we simply clear the TSTAT_TLB_STATS flag 897 * so that the user TSB information is automatically 898 * restored on next context switch. 899 * 900 * Note that the call to restore kernel TSB information 901 * will normally not fail, unless wrong information is 902 * passed here. In that scenario, system will still 903 * continue to function properly with the exception of 904 * kernel handling all the TLB misses. 905 */ 906 struct hv_tsb_block *hvbp = &ksfmmup->sfmmu_hvblock; 907 cpu_t *cp = CPU; 908 909 cp->cpu_m.cpu_tstat_flags &= ~TSTAT_TLB_STATS; 910 (void) hv_set_ctx0(hvbp->hv_tsb_info_cnt, 911 hvbp->hv_tsb_info_pa); 912 } 913 } 914 #endif 915 } 916 917 /* 918 * We use %tick as the time base when recording the time spent executing 919 * the trap handler. %tick, however, is not necessarily kept in sync 920 * across CPUs (indeed, different CPUs may have different %tick frequencies). 921 * We therefore cross call onto a CPU to get a snapshot of its data to 922 * copy out; this is the routine executed out of that cross call. 923 */ 924 static void 925 trapstat_snapshot() 926 { 927 tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id]; 928 tstat_data_t *data = tcpu->tcpu_data; 929 930 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED); 931 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 932 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ENABLED); 933 934 data->tdata_snapts = gethrtime(); 935 data->tdata_snaptick = rdtick(); 936 bcopy(data, tstat_buffer, tstat_data_t_size); 937 #ifdef sun4v 938 /* 939 * Invoke processor specific interface to collect TSB hit 940 * statistics on each processor. 941 */ 942 if ((tstat_options & TSTAT_OPT_TLBDATA) && tstat_fast_tlbstat) 943 cpu_trapstat_data((void *) tstat_buffer->tdata_pgsz, 944 tstat_pgszs); 945 #endif 946 } 947 948 /* 949 * The TSTAT_RETENT_* constants define offsets in the TLB return entry. 950 * They are used only in trapstat_tlbretent() (below) and #undef'd 951 * immediately afterwards. Any change to "retent" in trapstat_tlbretent() 952 * will likely require changes to these constants. 953 */ 954 955 #ifndef sun4v 956 #define TSTAT_RETENT_STATHI 1 957 #define TSTAT_RETENT_STATLO 2 958 #define TSTAT_RETENT_SHIFT 11 959 #define TSTAT_RETENT_COUNT_LD 13 960 #define TSTAT_RETENT_COUNT_ST 15 961 #define TSTAT_RETENT_TMPTSHI 16 962 #define TSTAT_RETENT_TMPTSLO 17 963 #define TSTAT_RETENT_TIME_LD 19 964 #define TSTAT_RETENT_TIME_ST 21 965 #else /* sun4v */ 966 #define TSTAT_RETENT_STATHI 1 967 #define TSTAT_RETENT_STATLO 2 968 #define TSTAT_RETENT_SHIFT 5 969 #define TSTAT_RETENT_COUNT_LD 7 970 #define TSTAT_RETENT_COUNT_ST 9 971 #define TSTAT_RETENT_TMPTSHI 10 972 #define TSTAT_RETENT_TMPTSLO 11 973 #define TSTAT_RETENT_TIME_LD 13 974 #define TSTAT_RETENT_TIME_ST 15 975 #endif /* sun4v */ 976 977 static void 978 trapstat_tlbretent(tstat_percpu_t *tcpu, tstat_tlbretent_t *ret, 979 tstat_missdata_t *data) 980 { 981 uint32_t *ent = ret->ttlbrent_instr, shift; 982 uintptr_t base, tmptick = TSTAT_DATA_OFFS(tcpu, tdata_tmptick); 983 984 /* 985 * This is the entry executed upon return from the TLB/TSB miss 986 * handler (i.e. the code interpositioned between the "retry" and 987 * the actual return to the TLB-missing instruction). Detail on its 988 * theory of operation can be found in the "TLB Statistics" section 989 * of the block comment. Note that we expect the TTE just loaded 990 * into the TLB to be in %g5; all other globals are available as 991 * scratch. Finally, note that the page size information in sun4v is 992 * located in the lower bits of the TTE -- requiring us to have a 993 * different return entry on sun4v. 994 */ 995 static const uint32_t retent[TSTAT_TLBRET_NINSTR] = { 996 #ifndef sun4v 997 0x87410000, /* rd %tick, %g3 */ 998 0x03000000, /* sethi %hi(stat), %g1 */ 999 0x82106000, /* or %g1, %lo(stat), %g1 */ 1000 0x89297001, /* sllx %g5, 1, %g4 */ 1001 0x8931303e, /* srlx %g4, 62, %g4 */ 1002 0x8531702e, /* srlx %g5, 46, %g2 */ 1003 0x8408a004, /* and %g2, 4, %g2 */ 1004 0x88110002, /* or %g4, %g2, %g4 */ 1005 0x80a12005, /* cmp %g4, 5 */ 1006 0x34400002, /* bg,a,pn %icc, +8 */ 1007 0x88102004, /* mov 4, %g4 */ 1008 0x89292000, /* sll %g4, shift, %g4 */ 1009 0x82004004, /* add %g1, %g4, %g1 */ 1010 0xc4586000, /* ldx [%g1 + tmiss_count], %g2 */ 1011 0x8400a001, /* add %g2, 1, %g2 */ 1012 0xc4706000, /* stx %g2, [%g1 + tmiss_count] */ 1013 0x0d000000, /* sethi %hi(tdata_tmptick), %g6 */ 1014 0xc459a000, /* ldx [%g6 + %lo(tdata_tmptick)], %g2 */ 1015 0x8620c002, /* sub %g3, %g2, %g3 */ 1016 0xc4586000, /* ldx [%g1 + tmiss_time], %g2 */ 1017 0x84008003, /* add %g2, %g3, %g2 */ 1018 0xc4706000, /* stx %g2, [%g1 + tmiss_time] */ 1019 0x83f00000 /* retry */ 1020 #else /* sun4v */ 1021 0x87410000, /* rd %tick, %g3 */ 1022 0x03000000, /* sethi %hi(stat), %g1 */ 1023 0x82106000, /* or %g1, %lo(stat), %g1 */ 1024 0x8929703d, /* sllx %g5, 61, %g4 */ 1025 0x8931303d, /* srlx %g4, 61, %g4 */ 1026 0x89292000, /* sll %g4, shift, %g4 */ 1027 0x82004004, /* add %g1, %g4, %g1 */ 1028 0xc4586000, /* ldx [%g1 + tmiss_count], %g2 */ 1029 0x8400a001, /* add %g2, 1, %g2 */ 1030 0xc4706000, /* stx %g2, [%g1 + tmiss_count] */ 1031 0x0d000000, /* sethi %hi(tdata_tmptick), %g6 */ 1032 0xc459a000, /* ldx [%g6 + %lo(tdata_tmptick)], %g2 */ 1033 0x8620c002, /* sub %g3, %g2, %g3 */ 1034 0xc4586000, /* ldx [%g1 + tmiss_time], %g2 */ 1035 0x84008003, /* add %g2, %g3, %g2 */ 1036 0xc4706000, /* stx %g2, [%g1 + tmiss_time] */ 1037 0x83f00000 /* retry */ 1038 #endif /* sun4v */ 1039 }; 1040 1041 ASSERT(MUTEX_HELD(&tstat_lock)); 1042 /*CONSTCOND*/ 1043 ASSERT(offsetof(tstat_missdata_t, tmiss_count) <= LO10(-1)); 1044 /*CONSTCOND*/ 1045 ASSERT(offsetof(tstat_missdata_t, tmiss_time) <= LO10(-1)); 1046 /*CONSTCOND*/ 1047 ASSERT(!((sizeof (tstat_pgszdata_t) - 1) & sizeof (tstat_pgszdata_t))); 1048 1049 for (shift = 1; (1 << shift) != sizeof (tstat_pgszdata_t); shift++) 1050 continue; 1051 1052 base = (uintptr_t)tcpu->tcpu_dbase + 1053 ((uintptr_t)data - (uintptr_t)tcpu->tcpu_data); 1054 1055 bcopy(retent, ent, sizeof (retent)); 1056 1057 ent[TSTAT_RETENT_STATHI] |= HI22(base); 1058 ent[TSTAT_RETENT_STATLO] |= LO10(base); 1059 ent[TSTAT_RETENT_SHIFT] |= shift; 1060 /* LINTED E_EXPR_NULL_EFFECT */ 1061 ent[TSTAT_RETENT_COUNT_LD] |= offsetof(tstat_missdata_t, tmiss_count); 1062 /* LINTED E_EXPR_NULL_EFFECT */ 1063 ent[TSTAT_RETENT_COUNT_ST] |= offsetof(tstat_missdata_t, tmiss_count); 1064 ent[TSTAT_RETENT_TMPTSHI] |= HI22(tmptick); 1065 ent[TSTAT_RETENT_TMPTSLO] |= LO10(tmptick); 1066 ent[TSTAT_RETENT_TIME_LD] |= offsetof(tstat_missdata_t, tmiss_time); 1067 ent[TSTAT_RETENT_TIME_ST] |= offsetof(tstat_missdata_t, tmiss_time); 1068 } 1069 1070 #undef TSTAT_RETENT_STATHI 1071 #undef TSTAT_RETENT_STATLO 1072 #undef TSTAT_RETENT_SHIFT 1073 #undef TSTAT_RETENT_COUNT_LD 1074 #undef TSTAT_RETENT_COUNT_ST 1075 #undef TSTAT_RETENT_TMPTSHI 1076 #undef TSTAT_RETENT_TMPTSLO 1077 #undef TSTAT_RETENT_TIME_LD 1078 #undef TSTAT_RETENT_TIME_ST 1079 1080 /* 1081 * The TSTAT_TLBENT_* constants define offsets in the TLB entry. They are 1082 * used only in trapstat_tlbent() (below) and #undef'd immediately afterwards. 1083 * Any change to "tlbent" in trapstat_tlbent() will likely require changes 1084 * to these constants. 1085 */ 1086 1087 #ifndef sun4v 1088 #define TSTAT_TLBENT_STATHI 0 1089 #define TSTAT_TLBENT_STATLO_LD 1 1090 #define TSTAT_TLBENT_STATLO_ST 3 1091 #define TSTAT_TLBENT_MMUASI 15 1092 #define TSTAT_TLBENT_TPCHI 18 1093 #define TSTAT_TLBENT_TPCLO_USER 19 1094 #define TSTAT_TLBENT_TPCLO_KERN 21 1095 #define TSTAT_TLBENT_TSHI 25 1096 #define TSTAT_TLBENT_TSLO 27 1097 #define TSTAT_TLBENT_BA 28 1098 #else /* sun4v */ 1099 #define TSTAT_TLBENT_STATHI 0 1100 #define TSTAT_TLBENT_STATLO_LD 1 1101 #define TSTAT_TLBENT_STATLO_ST 3 1102 #define TSTAT_TLBENT_TAGTARGET 19 1103 #define TSTAT_TLBENT_TPCHI 21 1104 #define TSTAT_TLBENT_TPCLO_USER 22 1105 #define TSTAT_TLBENT_TPCLO_KERN 24 1106 #define TSTAT_TLBENT_TSHI 28 1107 #define TSTAT_TLBENT_TSLO 30 1108 #define TSTAT_TLBENT_BA 31 1109 #endif /* sun4v */ 1110 1111 static void 1112 trapstat_tlbent(tstat_percpu_t *tcpu, int entno) 1113 { 1114 uint32_t *ent; 1115 uintptr_t orig, va, baoffs; 1116 #ifndef sun4v 1117 int itlb = entno == TSTAT_ENT_ITLBMISS; 1118 #else 1119 int itlb = (entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_ITLBMISS); 1120 #endif 1121 int entoffs = entno << TSTAT_ENT_SHIFT; 1122 uintptr_t tmptick, stat, tpc, utpc; 1123 tstat_pgszdata_t *data = &tcpu->tcpu_data->tdata_pgsz[0]; 1124 tstat_tlbdata_t *udata, *kdata; 1125 tstat_tlbret_t *ret; 1126 #ifndef sun4v 1127 uint32_t asi = itlb ? ASI(ASI_IMMU) : ASI(ASI_DMMU); 1128 #else 1129 uint32_t tagtarget_off = itlb ? MMFSA_I_CTX : MMFSA_D_CTX; 1130 #endif 1131 1132 /* 1133 * When trapstat is run with TLB statistics, this is the entry for 1134 * both I- and D-TLB misses; this code performs trap level pushing, 1135 * as described in the "TLB Statistics" section of the block comment. 1136 * This code is executing at TL 1; %tstate[0] contains the saved 1137 * state at the time of the TLB miss. Pushing trap level 1 (and thus 1138 * raising TL to 2) requires us to fill in %tstate[1] with our %pstate, 1139 * %cwp and %asi. We leave %tt unchanged, and we set %tpc and %tnpc to 1140 * the appropriate TLB return entry (based on the context of the miss). 1141 * Finally, we sample %tick, and stash it in the tdata_tmptick member 1142 * the per-CPU tstat_data structure. tdata_tmptick will be used in 1143 * the TLB return entry to determine the amount of time spent in the 1144 * TLB miss handler. 1145 * 1146 * Note that on sun4v platforms, we must obtain the context information 1147 * from the MMU fault status area. (The base address of this MMU fault 1148 * status area is kept in the scratchpad register 0.) 1149 */ 1150 static const uint32_t tlbent[] = { 1151 #ifndef sun4v 1152 0x03000000, /* sethi %hi(stat), %g1 */ 1153 0xc4586000, /* ldx [%g1 + %lo(stat)], %g2 */ 1154 0x8400a001, /* add %g2, 1, %g2 */ 1155 0xc4706000, /* stx %g2, [%g1 + %lo(stat)] */ 1156 0x85524000, /* rdpr %cwp, %g2 */ 1157 0x87518000, /* rdpr %pstate, %g3 */ 1158 0x8728f008, /* sllx %g3, 8, %g3 */ 1159 0x84108003, /* or %g2, %g3, %g2 */ 1160 0x8740c000, /* rd %asi, %g3 */ 1161 0x8728f018, /* sllx %g3, 24, %g3 */ 1162 0x84108003, /* or %g2, %g3, %g2 */ 1163 0x8350c000, /* rdpr %tt, %g1 */ 1164 0x8f902002, /* wrpr %g0, 2, %tl */ 1165 0x85908000, /* wrpr %g2, %g0, %tstate */ 1166 0x87904000, /* wrpr %g1, %g0, %tt */ 1167 0xc2d80000, /* ldxa [%g0]ASI_MMU, %g1 */ 1168 0x83307030, /* srlx %g1, CTXSHIFT, %g1 */ 1169 0x02c04004, /* brz,pn %g1, .+0x10 */ 1170 0x03000000, /* sethi %hi(new_tpc), %g1 */ 1171 0x82106000, /* or %g1, %lo(new_tpc), %g1 */ 1172 0x30800002, /* ba,a .+0x8 */ 1173 0x82106000, /* or %g1, %lo(new_tpc), %g1 */ 1174 0x81904000, /* wrpr %g1, %g0, %tpc */ 1175 0x82006004, /* add %g1, 4, %g1 */ 1176 0x83904000, /* wrpr %g1, %g0, %tnpc */ 1177 0x03000000, /* sethi %hi(tmptick), %g1 */ 1178 0x85410000, /* rd %tick, %g2 */ 1179 0xc4706000, /* stx %g2, [%g1 + %lo(tmptick)] */ 1180 0x30800000, /* ba,a addr */ 1181 NOP, NOP, NOP 1182 #else /* sun4v */ 1183 0x03000000, /* sethi %hi(stat), %g1 */ 1184 0xc4586000, /* ldx [%g1 + %lo(stat)], %g2 */ 1185 0x8400a001, /* add %g2, 1, %g2 */ 1186 0xc4706000, /* stx %g2, [%g1 + %lo(stat)] */ 1187 0x85524000, /* rdpr %cwp, %g2 */ 1188 0x87518000, /* rdpr %pstate, %g3 */ 1189 0x8728f008, /* sllx %g3, 8, %g3 */ 1190 0x84108003, /* or %g2, %g3, %g2 */ 1191 0x8740c000, /* rd %asi, %g3 */ 1192 0x8728f018, /* sllx %g3, 24, %g3 */ 1193 0x83540000, /* rdpr %gl, %g1 */ 1194 0x83287028, /* sllx %g1, 40, %g1 */ 1195 0x86104003, /* or %g1, %g3, %g3 */ 1196 0x84108003, /* or %g2, %g3, %g2 */ 1197 0x8350c000, /* rdpr %tt, %g1 */ 1198 0x8f902002, /* wrpr %g0, 2, %tl */ 1199 0x85908000, /* wrpr %g2, %g0, %tstate */ 1200 0x87904000, /* wrpr %g1, %g0, %tt */ 1201 0xc2d80400, /* ldxa [%g0]ASI_SCRATCHPAD, %g1 */ 1202 0xc2586000, /* ldx [%g1 + MMFSA_?_CTX], %g1 */ 1203 0x02c04004, /* brz,pn %g1, .+0x10 */ 1204 0x03000000, /* sethi %hi(new_tpc), %g1 */ 1205 0x82106000, /* or %g1, %lo(new_tpc), %g1 */ 1206 0x30800002, /* ba,a .+0x8 */ 1207 0x82106000, /* or %g1, %lo(new_tpc), %g1 */ 1208 0x81904000, /* wrpr %g1, %g0, %tpc */ 1209 0x82006004, /* add %g1, 4, %g1 */ 1210 0x83904000, /* wrpr %g1, %g0, %tnpc */ 1211 0x03000000, /* sethi %hi(tmptick), %g1 */ 1212 0x85410000, /* rd %tick, %g2 */ 1213 0xc4706000, /* stx %g2, [%g1 + %lo(tmptick)] */ 1214 0x30800000 /* ba,a addr */ 1215 #endif /* sun4v */ 1216 }; 1217 1218 ASSERT(MUTEX_HELD(&tstat_lock)); 1219 #ifndef sun4v 1220 ASSERT(entno == TSTAT_ENT_ITLBMISS || entno == TSTAT_ENT_DTLBMISS); 1221 #else 1222 ASSERT(entno == TSTAT_ENT_ITLBMISS || entno == TSTAT_ENT_DTLBMISS || 1223 entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_DMMUMISS); 1224 #endif 1225 1226 stat = TSTAT_DATA_OFFS(tcpu, tdata_traps) + entoffs; 1227 tmptick = TSTAT_DATA_OFFS(tcpu, tdata_tmptick); 1228 1229 if (itlb) { 1230 ret = &tcpu->tcpu_instr->tinst_itlbret; 1231 udata = &data->tpgsz_user.tmode_itlb; 1232 kdata = &data->tpgsz_kernel.tmode_itlb; 1233 tpc = TSTAT_INSTR_OFFS(tcpu, tinst_itlbret.ttlbr_ktlb); 1234 } else { 1235 ret = &tcpu->tcpu_instr->tinst_dtlbret; 1236 udata = &data->tpgsz_user.tmode_dtlb; 1237 kdata = &data->tpgsz_kernel.tmode_dtlb; 1238 tpc = TSTAT_INSTR_OFFS(tcpu, tinst_dtlbret.ttlbr_ktlb); 1239 } 1240 1241 utpc = tpc + offsetof(tstat_tlbret_t, ttlbr_utlb) - 1242 offsetof(tstat_tlbret_t, ttlbr_ktlb); 1243 1244 ASSERT(HI22(tpc) == HI22(utpc)); 1245 1246 ent = (uint32_t *)((uintptr_t)tcpu->tcpu_instr + entoffs); 1247 orig = KERNELBASE + entoffs; 1248 va = (uintptr_t)tcpu->tcpu_ibase + entoffs; 1249 baoffs = TSTAT_TLBENT_BA * sizeof (uint32_t); 1250 1251 #ifdef sun4v 1252 if (entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_DMMUMISS) { 1253 /* 1254 * Because of lack of space, interposing tlbent trap 1255 * handler for IMMU_miss and DMMU_miss traps cannot be 1256 * placed in-line. Instead, we copy it to the space set 1257 * aside for these traps in per CPU trapstat area and 1258 * invoke it by placing a branch in the trap table itself. 1259 */ 1260 static const uint32_t mmumiss[TSTAT_ENT_NINSTR] = { 1261 0x30800000, /* ba,a addr */ 1262 NOP, NOP, NOP, NOP, NOP, NOP, NOP 1263 }; 1264 uint32_t *tent = ent; /* trap vector entry */ 1265 uintptr_t tentva = va; /* trap vector entry va */ 1266 1267 if (itlb) { 1268 ent = (uint32_t *)((uintptr_t) 1269 &tcpu->tcpu_instr->tinst_immumiss); 1270 va = TSTAT_INSTR_OFFS(tcpu, tinst_immumiss); 1271 } else { 1272 ent = (uint32_t *)((uintptr_t) 1273 &tcpu->tcpu_instr->tinst_dmmumiss); 1274 va = TSTAT_INSTR_OFFS(tcpu, tinst_dmmumiss); 1275 } 1276 bcopy(mmumiss, tent, sizeof (mmumiss)); 1277 tent[0] |= DISP22(tentva, va); 1278 } 1279 #endif /* sun4v */ 1280 1281 bcopy(tlbent, ent, sizeof (tlbent)); 1282 1283 ent[TSTAT_TLBENT_STATHI] |= HI22(stat); 1284 ent[TSTAT_TLBENT_STATLO_LD] |= LO10(stat); 1285 ent[TSTAT_TLBENT_STATLO_ST] |= LO10(stat); 1286 #ifndef sun4v 1287 ent[TSTAT_TLBENT_MMUASI] |= asi; 1288 #else 1289 ent[TSTAT_TLBENT_TAGTARGET] |= tagtarget_off; 1290 #endif 1291 ent[TSTAT_TLBENT_TPCHI] |= HI22(tpc); 1292 ent[TSTAT_TLBENT_TPCLO_USER] |= LO10(utpc); 1293 ent[TSTAT_TLBENT_TPCLO_KERN] |= LO10(tpc); 1294 ent[TSTAT_TLBENT_TSHI] |= HI22(tmptick); 1295 ent[TSTAT_TLBENT_TSLO] |= LO10(tmptick); 1296 ent[TSTAT_TLBENT_BA] |= DISP22(va + baoffs, orig); 1297 1298 /* 1299 * And now set up the TLB return entries. 1300 */ 1301 trapstat_tlbretent(tcpu, &ret->ttlbr_ktlb, &kdata->ttlb_tlb); 1302 trapstat_tlbretent(tcpu, &ret->ttlbr_ktsb, &kdata->ttlb_tsb); 1303 trapstat_tlbretent(tcpu, &ret->ttlbr_utlb, &udata->ttlb_tlb); 1304 trapstat_tlbretent(tcpu, &ret->ttlbr_utsb, &udata->ttlb_tsb); 1305 } 1306 1307 #undef TSTAT_TLBENT_STATHI 1308 #undef TSTAT_TLBENT_STATLO_LD 1309 #undef TSTAT_TLBENT_STATLO_ST 1310 #ifndef sun4v 1311 #undef TSTAT_TLBENT_MMUASI 1312 #else 1313 #undef TSTAT_TLBENT_TAGTARGET 1314 #endif 1315 #undef TSTAT_TLBENT_TPCHI 1316 #undef TSTAT_TLBENT_TPCLO_USER 1317 #undef TSTAT_TLBENT_TPCLO_KERN 1318 #undef TSTAT_TLBENT_TSHI 1319 #undef TSTAT_TLBENT_TSLO 1320 #undef TSTAT_TLBENT_BA 1321 1322 /* 1323 * The TSTAT_ENABLED_* constants define offsets in the enabled entry; the 1324 * TSTAT_DISABLED_BA constant defines an offset in the disabled entry. Both 1325 * sets of constants are used only in trapstat_make_traptab() (below) and 1326 * #undef'd immediately afterwards. Any change to "enabled" or "disabled" 1327 * in trapstat_make_traptab() will likely require changes to these constants. 1328 */ 1329 #define TSTAT_ENABLED_STATHI 0 1330 #define TSTAT_ENABLED_STATLO_LD 1 1331 #define TSTAT_ENABLED_STATLO_ST 3 1332 #define TSTAT_ENABLED_BA 4 1333 #define TSTAT_DISABLED_BA 0 1334 1335 static void 1336 trapstat_make_traptab(tstat_percpu_t *tcpu) 1337 { 1338 uint32_t *ent; 1339 uint64_t *stat; 1340 uintptr_t orig, va, en_baoffs, dis_baoffs; 1341 int nent; 1342 1343 /* 1344 * This is the entry in the interposing trap table for enabled trap 1345 * table entries. It loads a counter, increments it and stores it 1346 * back before branching to the actual trap table entry. 1347 */ 1348 static const uint32_t enabled[TSTAT_ENT_NINSTR] = { 1349 0x03000000, /* sethi %hi(stat), %g1 */ 1350 0xc4586000, /* ldx [%g1 + %lo(stat)], %g2 */ 1351 0x8400a001, /* add %g2, 1, %g2 */ 1352 0xc4706000, /* stx %g2, [%g1 + %lo(stat)] */ 1353 0x30800000, /* ba,a addr */ 1354 NOP, NOP, NOP 1355 }; 1356 1357 /* 1358 * This is the entry in the interposing trap table for disabled trap 1359 * table entries. It simply branches to the actual, underlying trap 1360 * table entry. As explained in the "Implementation Details" section 1361 * of the block comment, all TL>0 traps _must_ use the disabled entry; 1362 * additional entries may be explicitly disabled through the use 1363 * of TSTATIOC_ENTRY/TSTATIOC_NOENTRY. 1364 */ 1365 static const uint32_t disabled[TSTAT_ENT_NINSTR] = { 1366 0x30800000, /* ba,a addr */ 1367 NOP, NOP, NOP, NOP, NOP, NOP, NOP, 1368 }; 1369 1370 ASSERT(MUTEX_HELD(&tstat_lock)); 1371 1372 ent = tcpu->tcpu_instr->tinst_traptab; 1373 stat = (uint64_t *)TSTAT_DATA_OFFS(tcpu, tdata_traps); 1374 orig = KERNELBASE; 1375 va = (uintptr_t)tcpu->tcpu_ibase; 1376 en_baoffs = TSTAT_ENABLED_BA * sizeof (uint32_t); 1377 dis_baoffs = TSTAT_DISABLED_BA * sizeof (uint32_t); 1378 1379 for (nent = 0; nent < TSTAT_TOTAL_NENT; nent++) { 1380 if (tstat_enabled[nent]) { 1381 bcopy(enabled, ent, sizeof (enabled)); 1382 ent[TSTAT_ENABLED_STATHI] |= HI22((uintptr_t)stat); 1383 ent[TSTAT_ENABLED_STATLO_LD] |= LO10((uintptr_t)stat); 1384 ent[TSTAT_ENABLED_STATLO_ST] |= LO10((uintptr_t)stat); 1385 ent[TSTAT_ENABLED_BA] |= DISP22(va + en_baoffs, orig); 1386 } else { 1387 bcopy(disabled, ent, sizeof (disabled)); 1388 ent[TSTAT_DISABLED_BA] |= DISP22(va + dis_baoffs, orig); 1389 } 1390 1391 stat++; 1392 orig += sizeof (enabled); 1393 ent += sizeof (enabled) / sizeof (*ent); 1394 va += sizeof (enabled); 1395 } 1396 } 1397 1398 #undef TSTAT_ENABLED_STATHI 1399 #undef TSTAT_ENABLED_STATLO_LD 1400 #undef TSTAT_ENABLED_STATLO_ST 1401 #undef TSTAT_ENABLED_BA 1402 #undef TSTAT_DISABLED_BA 1403 1404 #ifndef sun4v 1405 /* 1406 * See Section A.6 in SPARC v9 Manual. 1407 * max branch = 4*((2^21)-1) = 8388604 1408 */ 1409 #define MAX_BICC_BRANCH_DISPLACEMENT (4 * ((1 << 21) - 1)) 1410 #endif 1411 1412 static void 1413 trapstat_setup(processorid_t cpu) 1414 { 1415 tstat_percpu_t *tcpu = &tstat_percpu[cpu]; 1416 #ifndef sun4v 1417 int i; 1418 caddr_t va; 1419 pfn_t *pfn; 1420 cpu_t *cp; 1421 uint_t strand_idx; 1422 size_t tstat_offset; 1423 #endif 1424 1425 ASSERT(tcpu->tcpu_pfn == NULL); 1426 ASSERT(tcpu->tcpu_instr == NULL); 1427 ASSERT(tcpu->tcpu_data == NULL); 1428 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED); 1429 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED)); 1430 ASSERT(MUTEX_HELD(&cpu_lock)); 1431 ASSERT(MUTEX_HELD(&tstat_lock)); 1432 1433 /* 1434 * The lower fifteen bits of the %tba are always read as zero; we must 1435 * align our instruction base address appropriately. 1436 */ 1437 #ifndef sun4v 1438 tstat_offset = tstat_total_size; 1439 1440 cp = cpu_get(cpu); 1441 ASSERT(cp != NULL); 1442 if ((strand_idx = cpu ^ chip_plat_get_coreid(cp)) != 0) { 1443 /* 1444 * On sun4u platforms with multiple CPUs sharing the MMU 1445 * (Olympus-C has 2 strands per core), each CPU uses a 1446 * disjoint trap table. The indexing is based on the 1447 * strand id, which is obtained by XOR'ing the cpuid with 1448 * the coreid. 1449 */ 1450 tstat_offset += tstat_total_size * strand_idx; 1451 1452 /* 1453 * Offset must be less than the maximum PC-relative branch 1454 * displacement for Bicc variants. See the Implementation 1455 * Details comment. 1456 */ 1457 ASSERT(tstat_offset <= MAX_BICC_BRANCH_DISPLACEMENT); 1458 } 1459 1460 tcpu->tcpu_ibase = (caddr_t)((KERNELBASE - tstat_offset) 1461 & TSTAT_TBA_MASK); 1462 tcpu->tcpu_dbase = tcpu->tcpu_ibase + TSTAT_INSTR_SIZE; 1463 tcpu->tcpu_vabase = tcpu->tcpu_ibase; 1464 1465 tcpu->tcpu_pfn = vmem_alloc(tstat_arena, tstat_total_pages, VM_SLEEP); 1466 bzero(tcpu->tcpu_pfn, tstat_total_pages); 1467 pfn = tcpu->tcpu_pfn; 1468 1469 tcpu->tcpu_instr = vmem_alloc(tstat_arena, TSTAT_INSTR_SIZE, VM_SLEEP); 1470 1471 va = (caddr_t)tcpu->tcpu_instr; 1472 for (i = 0; i < TSTAT_INSTR_PAGES; i++, va += MMU_PAGESIZE) 1473 *pfn++ = hat_getpfnum(kas.a_hat, va); 1474 1475 /* 1476 * We must be sure that the pages that we will use to examine the data 1477 * have the same virtual color as the pages to which the data is being 1478 * recorded, hence the alignment and phase constraints on the 1479 * allocation. 1480 */ 1481 tcpu->tcpu_data = vmem_xalloc(tstat_arena, tstat_data_size, 1482 shm_alignment, (uintptr_t)tcpu->tcpu_dbase & (shm_alignment - 1), 1483 0, 0, NULL, VM_SLEEP); 1484 bzero(tcpu->tcpu_data, tstat_data_size); 1485 tcpu->tcpu_data->tdata_cpuid = cpu; 1486 1487 va = (caddr_t)tcpu->tcpu_data; 1488 for (i = 0; i < tstat_data_pages; i++, va += MMU_PAGESIZE) 1489 *pfn++ = hat_getpfnum(kas.a_hat, va); 1490 #else /* sun4v */ 1491 ASSERT(!(tstat_total_size > (1 + ~TSTAT_TBA_MASK))); 1492 tcpu->tcpu_vabase = (caddr_t)(KERNELBASE - MMU_PAGESIZE4M); 1493 tcpu->tcpu_ibase = tcpu->tcpu_vabase + (cpu * (1 + ~TSTAT_TBA_MASK)); 1494 tcpu->tcpu_dbase = tcpu->tcpu_ibase + TSTAT_INSTR_SIZE; 1495 1496 tcpu->tcpu_pfn = &tstat_pfn; 1497 tcpu->tcpu_instr = (tstat_instr_t *)(tstat_va + (cpu * 1498 (1 + ~TSTAT_TBA_MASK))); 1499 tcpu->tcpu_data = (tstat_data_t *)(tstat_va + (cpu * 1500 (1 + ~TSTAT_TBA_MASK)) + TSTAT_INSTR_SIZE); 1501 bzero(tcpu->tcpu_data, tstat_data_size); 1502 tcpu->tcpu_data->tdata_cpuid = cpu; 1503 #endif /* sun4v */ 1504 1505 /* 1506 * Now that we have all of the instruction and data pages allocated, 1507 * make the trap table from scratch. 1508 */ 1509 trapstat_make_traptab(tcpu); 1510 1511 if (tstat_options & TSTAT_OPT_TLBDATA) { 1512 /* 1513 * TLB Statistics have been specified; set up the I- and D-TLB 1514 * entries and corresponding TLB return entries. 1515 */ 1516 #ifndef sun4v 1517 trapstat_tlbent(tcpu, TSTAT_ENT_ITLBMISS); 1518 trapstat_tlbent(tcpu, TSTAT_ENT_DTLBMISS); 1519 #else 1520 if (tstat_fast_tlbstat) { 1521 trapstat_tlbent(tcpu, TSTAT_ENT_IMMUMISS); 1522 trapstat_tlbent(tcpu, TSTAT_ENT_DMMUMISS); 1523 } else { 1524 trapstat_tlbent(tcpu, TSTAT_ENT_ITLBMISS); 1525 trapstat_tlbent(tcpu, TSTAT_ENT_DTLBMISS); 1526 } 1527 #endif 1528 } 1529 1530 tcpu->tcpu_flags |= TSTAT_CPU_ALLOCATED; 1531 1532 /* 1533 * Finally, get the target CPU to load the locked pages into its TLBs. 1534 */ 1535 xc_one(cpu, (xcfunc_t *)trapstat_load_tlb, 0, 0); 1536 } 1537 1538 static void 1539 trapstat_teardown(processorid_t cpu) 1540 { 1541 tstat_percpu_t *tcpu = &tstat_percpu[cpu]; 1542 #ifndef sun4v 1543 int i; 1544 #endif 1545 caddr_t va = tcpu->tcpu_vabase; 1546 1547 ASSERT(tcpu->tcpu_pfn != NULL); 1548 ASSERT(tcpu->tcpu_instr != NULL); 1549 ASSERT(tcpu->tcpu_data != NULL); 1550 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED); 1551 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 1552 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)); 1553 ASSERT(MUTEX_HELD(&cpu_lock)); 1554 ASSERT(MUTEX_HELD(&tstat_lock)); 1555 1556 #ifndef sun4v 1557 vmem_free(tstat_arena, tcpu->tcpu_pfn, tstat_total_pages); 1558 vmem_free(tstat_arena, tcpu->tcpu_instr, TSTAT_INSTR_SIZE); 1559 vmem_free(tstat_arena, tcpu->tcpu_data, tstat_data_size); 1560 1561 for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) { 1562 xt_one(cpu, vtag_flushpage_tl1, (uint64_t)va, KCONTEXT); 1563 } 1564 #else 1565 xt_one(cpu, vtag_unmap_perm_tl1, (uint64_t)va, KCONTEXT); 1566 #endif 1567 1568 tcpu->tcpu_pfn = NULL; 1569 tcpu->tcpu_instr = NULL; 1570 tcpu->tcpu_data = NULL; 1571 tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED; 1572 } 1573 1574 static int 1575 trapstat_go() 1576 { 1577 cpu_t *cp; 1578 1579 mutex_enter(&cpu_lock); 1580 mutex_enter(&tstat_lock); 1581 1582 if (tstat_running) { 1583 mutex_exit(&tstat_lock); 1584 mutex_exit(&cpu_lock); 1585 return (EBUSY); 1586 } 1587 1588 #ifdef sun4v 1589 /* 1590 * Allocate large page to hold interposing tables. 1591 */ 1592 tstat_va = contig_mem_alloc(MMU_PAGESIZE4M); 1593 tstat_pfn = va_to_pfn(tstat_va); 1594 if (tstat_pfn == PFN_INVALID) 1595 return (EAGAIN); 1596 1597 /* 1598 * For detailed TLB statistics, invoke CPU specific interface 1599 * to see if it supports a low overhead interface to collect 1600 * TSB hit statistics. If so, make set tstat_fast_tlbstat flag 1601 * to reflect that. 1602 */ 1603 if (tstat_options & TSTAT_OPT_TLBDATA) { 1604 int error; 1605 1606 error = cpu_trapstat_conf(CPU_TSTATCONF_INIT); 1607 if (error == 0) 1608 tstat_fast_tlbstat = B_TRUE; 1609 else if (error != ENOTSUP) { 1610 contig_mem_free(tstat_va, MMU_PAGESIZE4M); 1611 return (error); 1612 } 1613 } 1614 #endif 1615 1616 /* 1617 * First, perform any necessary hot patching. 1618 */ 1619 trapstat_hotpatch(); 1620 1621 /* 1622 * Allocate the resources we'll need to measure probe effect. 1623 */ 1624 trapstat_probe_alloc(); 1625 1626 1627 cp = cpu_list; 1628 do { 1629 if (!(tstat_percpu[cp->cpu_id].tcpu_flags & TSTAT_CPU_SELECTED)) 1630 continue; 1631 1632 trapstat_setup(cp->cpu_id); 1633 1634 /* 1635 * Note that due to trapstat_probe()'s use of global data, 1636 * we determine the probe effect on each CPU serially instead 1637 * of in parallel with an xc_all(). 1638 */ 1639 xc_one(cp->cpu_id, (xcfunc_t *)trapstat_probe, 0, 0); 1640 } while ((cp = cp->cpu_next) != cpu_list); 1641 1642 xc_all((xcfunc_t *)trapstat_enable, 0, 0); 1643 1644 trapstat_probe_free(); 1645 tstat_running = 1; 1646 mutex_exit(&tstat_lock); 1647 mutex_exit(&cpu_lock); 1648 1649 return (0); 1650 } 1651 1652 static int 1653 trapstat_stop() 1654 { 1655 int i; 1656 1657 mutex_enter(&cpu_lock); 1658 mutex_enter(&tstat_lock); 1659 if (!tstat_running) { 1660 mutex_exit(&tstat_lock); 1661 mutex_exit(&cpu_lock); 1662 return (ENXIO); 1663 } 1664 1665 xc_all((xcfunc_t *)trapstat_disable, 0, 0); 1666 1667 for (i = 0; i <= max_cpuid; i++) { 1668 if (tstat_percpu[i].tcpu_flags & TSTAT_CPU_ALLOCATED) 1669 trapstat_teardown(i); 1670 } 1671 1672 #ifdef sun4v 1673 if (tstat_options & TSTAT_OPT_TLBDATA) 1674 cpu_trapstat_conf(CPU_TSTATCONF_FINI); 1675 contig_mem_free(tstat_va, MMU_PAGESIZE4M); 1676 #endif 1677 trapstat_hotpatch(); 1678 tstat_running = 0; 1679 mutex_exit(&tstat_lock); 1680 mutex_exit(&cpu_lock); 1681 1682 return (0); 1683 } 1684 1685 /* 1686 * This is trapstat's DR CPU configuration callback. It's called (with 1687 * cpu_lock held) to unconfigure a newly powered-off CPU, or to configure a 1688 * powered-off CPU that is to be brought into the system. We need only take 1689 * action in the unconfigure case: because a powered-off CPU will have its 1690 * trap table restored to KERNELBASE if it is ever powered back on, we must 1691 * update the flags to reflect that trapstat is no longer enabled on the 1692 * powered-off CPU. Note that this means that a TSTAT_CPU_ENABLED CPU that 1693 * is unconfigured/powered off and later powered back on/reconfigured will 1694 * _not_ be re-TSTAT_CPU_ENABLED. 1695 */ 1696 static int 1697 trapstat_cpu_setup(cpu_setup_t what, processorid_t cpu) 1698 { 1699 tstat_percpu_t *tcpu = &tstat_percpu[cpu]; 1700 1701 ASSERT(MUTEX_HELD(&cpu_lock)); 1702 mutex_enter(&tstat_lock); 1703 1704 if (!tstat_running) { 1705 mutex_exit(&tstat_lock); 1706 return (0); 1707 } 1708 1709 switch (what) { 1710 case CPU_CONFIG: 1711 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)); 1712 break; 1713 1714 case CPU_UNCONFIG: 1715 if (tcpu->tcpu_flags & TSTAT_CPU_ENABLED) { 1716 tcpu->tcpu_flags &= ~TSTAT_CPU_ENABLED; 1717 #ifdef sun4v 1718 /* 1719 * A power-off, causes the cpu mondo queues to be 1720 * unconfigured on sun4v. Since we can't teardown 1721 * trapstat's mappings on the cpu that is going away, 1722 * we simply mark it as not allocated. This will 1723 * prevent a teardown on a cpu with the same cpu id 1724 * that might have been added while trapstat is running. 1725 */ 1726 if (tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED) { 1727 tcpu->tcpu_pfn = NULL; 1728 tcpu->tcpu_instr = NULL; 1729 tcpu->tcpu_data = NULL; 1730 tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED; 1731 } 1732 #endif 1733 } 1734 break; 1735 1736 default: 1737 break; 1738 } 1739 1740 mutex_exit(&tstat_lock); 1741 return (0); 1742 } 1743 1744 /* 1745 * This is called before a CPR suspend and after a CPR resume. We don't have 1746 * anything to do before a suspend, but after a restart we must restore the 1747 * trap table to be our interposing trap table. However, we don't actually 1748 * know whether or not the CPUs have been powered off -- this routine may be 1749 * called while restoring from a failed CPR suspend. We thus run through each 1750 * TSTAT_CPU_ENABLED CPU, and explicitly destroy and reestablish its 1751 * interposing trap table. This assures that our state is correct regardless 1752 * of whether or not the CPU has been newly powered on. 1753 */ 1754 /*ARGSUSED*/ 1755 static boolean_t 1756 trapstat_cpr(void *arg, int code) 1757 { 1758 cpu_t *cp; 1759 1760 if (code == CB_CODE_CPR_CHKPT) 1761 return (B_TRUE); 1762 1763 ASSERT(code == CB_CODE_CPR_RESUME); 1764 1765 mutex_enter(&cpu_lock); 1766 mutex_enter(&tstat_lock); 1767 1768 if (!tstat_running) { 1769 mutex_exit(&tstat_lock); 1770 mutex_exit(&cpu_lock); 1771 return (B_TRUE); 1772 } 1773 1774 cp = cpu_list; 1775 do { 1776 tstat_percpu_t *tcpu = &tstat_percpu[cp->cpu_id]; 1777 1778 if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)) 1779 continue; 1780 1781 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED); 1782 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 1783 1784 xc_one(cp->cpu_id, (xcfunc_t *)trapstat_disable, 0, 0); 1785 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)); 1786 1787 /* 1788 * Preserve this CPU's data in tstat_buffer and rip down its 1789 * interposing trap table. 1790 */ 1791 bcopy(tcpu->tcpu_data, tstat_buffer, tstat_data_t_size); 1792 trapstat_teardown(cp->cpu_id); 1793 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED)); 1794 1795 /* 1796 * Reestablish the interposing trap table and restore the old 1797 * data. 1798 */ 1799 trapstat_setup(cp->cpu_id); 1800 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 1801 bcopy(tstat_buffer, tcpu->tcpu_data, tstat_data_t_size); 1802 1803 xc_one(cp->cpu_id, (xcfunc_t *)trapstat_enable, 0, 0); 1804 } while ((cp = cp->cpu_next) != cpu_list); 1805 1806 mutex_exit(&tstat_lock); 1807 mutex_exit(&cpu_lock); 1808 1809 return (B_TRUE); 1810 } 1811 1812 /*ARGSUSED*/ 1813 static int 1814 trapstat_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 1815 { 1816 int i; 1817 1818 mutex_enter(&cpu_lock); 1819 mutex_enter(&tstat_lock); 1820 if (tstat_open != 0) { 1821 mutex_exit(&tstat_lock); 1822 mutex_exit(&cpu_lock); 1823 return (EBUSY); 1824 } 1825 1826 /* 1827 * Register this in open() rather than in attach() to prevent deadlock 1828 * with DR code. During attach, I/O device tree locks are grabbed 1829 * before trapstat_attach() is invoked - registering in attach 1830 * will result in the lock order: device tree lock, cpu_lock. 1831 * DR code however requires that cpu_lock be acquired before 1832 * device tree locks. 1833 */ 1834 ASSERT(!tstat_running); 1835 register_cpu_setup_func((cpu_setup_func_t *)trapstat_cpu_setup, NULL); 1836 1837 /* 1838 * Clear all options. And until specific CPUs are specified, we'll 1839 * mark all CPUs as selected. 1840 */ 1841 tstat_options = 0; 1842 1843 for (i = 0; i <= max_cpuid; i++) 1844 tstat_percpu[i].tcpu_flags |= TSTAT_CPU_SELECTED; 1845 1846 /* 1847 * By default, all traps at TL=0 are enabled. Traps at TL>0 must 1848 * be disabled. 1849 */ 1850 for (i = 0; i < TSTAT_TOTAL_NENT; i++) 1851 tstat_enabled[i] = i < TSTAT_NENT ? 1 : 0; 1852 1853 tstat_open = 1; 1854 mutex_exit(&tstat_lock); 1855 mutex_exit(&cpu_lock); 1856 1857 return (0); 1858 } 1859 1860 /*ARGSUSED*/ 1861 static int 1862 trapstat_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 1863 { 1864 (void) trapstat_stop(); 1865 1866 ASSERT(!tstat_running); 1867 1868 mutex_enter(&cpu_lock); 1869 unregister_cpu_setup_func((cpu_setup_func_t *)trapstat_cpu_setup, NULL); 1870 mutex_exit(&cpu_lock); 1871 1872 tstat_open = 0; 1873 return (DDI_SUCCESS); 1874 } 1875 1876 static int 1877 trapstat_option(int option) 1878 { 1879 mutex_enter(&tstat_lock); 1880 1881 if (tstat_running) { 1882 mutex_exit(&tstat_lock); 1883 return (EBUSY); 1884 } 1885 1886 tstat_options |= option; 1887 mutex_exit(&tstat_lock); 1888 1889 return (0); 1890 } 1891 1892 /*ARGSUSED*/ 1893 static int 1894 trapstat_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *crd, int *rval) 1895 { 1896 int i, j, out; 1897 size_t dsize; 1898 1899 switch (cmd) { 1900 case TSTATIOC_GO: 1901 return (trapstat_go()); 1902 1903 case TSTATIOC_NOGO: 1904 return (trapstat_option(TSTAT_OPT_NOGO)); 1905 1906 case TSTATIOC_STOP: 1907 return (trapstat_stop()); 1908 1909 case TSTATIOC_CPU: 1910 if (arg < 0 || arg > max_cpuid) 1911 return (EINVAL); 1912 /*FALLTHROUGH*/ 1913 1914 case TSTATIOC_NOCPU: 1915 mutex_enter(&tstat_lock); 1916 1917 if (tstat_running) { 1918 mutex_exit(&tstat_lock); 1919 return (EBUSY); 1920 } 1921 1922 /* 1923 * If this is the first CPU to be specified (or if we are 1924 * being asked to explicitly de-select CPUs), disable all CPUs. 1925 */ 1926 if (!(tstat_options & TSTAT_OPT_CPU) || cmd == TSTATIOC_NOCPU) { 1927 tstat_options |= TSTAT_OPT_CPU; 1928 1929 for (i = 0; i <= max_cpuid; i++) { 1930 tstat_percpu_t *tcpu = &tstat_percpu[i]; 1931 1932 ASSERT(cmd == TSTATIOC_NOCPU || 1933 (tcpu->tcpu_flags & TSTAT_CPU_SELECTED)); 1934 tcpu->tcpu_flags &= ~TSTAT_CPU_SELECTED; 1935 } 1936 } 1937 1938 if (cmd == TSTATIOC_CPU) 1939 tstat_percpu[arg].tcpu_flags |= TSTAT_CPU_SELECTED; 1940 1941 mutex_exit(&tstat_lock); 1942 1943 return (0); 1944 1945 case TSTATIOC_ENTRY: 1946 mutex_enter(&tstat_lock); 1947 1948 if (tstat_running) { 1949 mutex_exit(&tstat_lock); 1950 return (EBUSY); 1951 } 1952 1953 if (arg >= TSTAT_NENT || arg < 0) { 1954 mutex_exit(&tstat_lock); 1955 return (EINVAL); 1956 } 1957 1958 if (!(tstat_options & TSTAT_OPT_ENTRY)) { 1959 /* 1960 * If this is the first entry that we are explicitly 1961 * enabling, explicitly disable every TL=0 entry. 1962 */ 1963 for (i = 0; i < TSTAT_NENT; i++) 1964 tstat_enabled[i] = 0; 1965 1966 tstat_options |= TSTAT_OPT_ENTRY; 1967 } 1968 1969 tstat_enabled[arg] = 1; 1970 mutex_exit(&tstat_lock); 1971 return (0); 1972 1973 case TSTATIOC_NOENTRY: 1974 mutex_enter(&tstat_lock); 1975 1976 if (tstat_running) { 1977 mutex_exit(&tstat_lock); 1978 return (EBUSY); 1979 } 1980 1981 for (i = 0; i < TSTAT_NENT; i++) 1982 tstat_enabled[i] = 0; 1983 1984 mutex_exit(&tstat_lock); 1985 return (0); 1986 1987 case TSTATIOC_READ: 1988 mutex_enter(&tstat_lock); 1989 1990 if (tstat_options & TSTAT_OPT_TLBDATA) { 1991 dsize = tstat_data_t_exported_size; 1992 } else { 1993 dsize = sizeof (tstat_data_t); 1994 } 1995 1996 for (i = 0, out = 0; i <= max_cpuid; i++) { 1997 tstat_percpu_t *tcpu = &tstat_percpu[i]; 1998 1999 if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)) 2000 continue; 2001 2002 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED); 2003 ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED); 2004 2005 tstat_buffer->tdata_cpuid = -1; 2006 xc_one(i, (xcfunc_t *)trapstat_snapshot, 0, 0); 2007 2008 if (tstat_buffer->tdata_cpuid == -1) { 2009 /* 2010 * This CPU is not currently responding to 2011 * cross calls; we have caught it while it is 2012 * being unconfigured. We'll drop tstat_lock 2013 * and pick up and drop cpu_lock. By the 2014 * time we acquire cpu_lock, the DR operation 2015 * will appear consistent and we can assert 2016 * that trapstat_cpu_setup() has cleared 2017 * TSTAT_CPU_ENABLED. 2018 */ 2019 mutex_exit(&tstat_lock); 2020 mutex_enter(&cpu_lock); 2021 mutex_exit(&cpu_lock); 2022 mutex_enter(&tstat_lock); 2023 ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED)); 2024 continue; 2025 } 2026 2027 /* 2028 * Need to compensate for the difference between page 2029 * sizes exported to users and page sizes available 2030 * within the kernel. 2031 */ 2032 if ((tstat_options & TSTAT_OPT_TLBDATA) && 2033 (tstat_pgszs != tstat_user_pgszs)) { 2034 tstat_pgszdata_t *tp; 2035 uint_t szc; 2036 2037 tp = &tstat_buffer->tdata_pgsz[0]; 2038 for (j = 0; j < tstat_user_pgszs; j++) { 2039 if ((szc = USERSZC_2_SZC(j)) != j) { 2040 bcopy(&tp[szc], &tp[j], 2041 sizeof (tstat_pgszdata_t)); 2042 } 2043 } 2044 } 2045 2046 if (copyout(tstat_buffer, (void *)arg, dsize) != 0) { 2047 mutex_exit(&tstat_lock); 2048 return (EFAULT); 2049 } 2050 2051 out++; 2052 arg += dsize; 2053 } 2054 2055 if (out != max_cpuid + 1) { 2056 processorid_t cpuid = -1; 2057 arg += offsetof(tstat_data_t, tdata_cpuid); 2058 2059 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) { 2060 mutex_exit(&tstat_lock); 2061 return (EFAULT); 2062 } 2063 } 2064 2065 mutex_exit(&tstat_lock); 2066 2067 return (0); 2068 2069 case TSTATIOC_TLBDATA: 2070 return (trapstat_option(TSTAT_OPT_TLBDATA)); 2071 2072 default: 2073 break; 2074 } 2075 2076 return (ENOTTY); 2077 } 2078 2079 /*ARGSUSED*/ 2080 static int 2081 trapstat_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 2082 { 2083 int error; 2084 2085 switch (infocmd) { 2086 case DDI_INFO_DEVT2DEVINFO: 2087 *result = (void *)tstat_devi; 2088 error = DDI_SUCCESS; 2089 break; 2090 case DDI_INFO_DEVT2INSTANCE: 2091 *result = (void *)0; 2092 error = DDI_SUCCESS; 2093 break; 2094 default: 2095 error = DDI_FAILURE; 2096 } 2097 return (error); 2098 } 2099 2100 static int 2101 trapstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 2102 { 2103 switch (cmd) { 2104 case DDI_ATTACH: 2105 break; 2106 2107 case DDI_RESUME: 2108 return (DDI_SUCCESS); 2109 2110 default: 2111 return (DDI_FAILURE); 2112 } 2113 2114 if (ddi_create_minor_node(devi, "trapstat", S_IFCHR, 2115 0, DDI_PSEUDO, 0) == DDI_FAILURE) { 2116 ddi_remove_minor_node(devi, NULL); 2117 return (DDI_FAILURE); 2118 } 2119 2120 ddi_report_dev(devi); 2121 tstat_devi = devi; 2122 2123 tstat_pgszs = page_num_pagesizes(); 2124 tstat_user_pgszs = page_num_user_pagesizes(); 2125 tstat_data_t_size = sizeof (tstat_data_t) + 2126 (tstat_pgszs - 1) * sizeof (tstat_pgszdata_t); 2127 tstat_data_t_exported_size = sizeof (tstat_data_t) + 2128 (tstat_user_pgszs - 1) * sizeof (tstat_pgszdata_t); 2129 #ifndef sun4v 2130 tstat_data_pages = (tstat_data_t_size >> MMU_PAGESHIFT) + 1; 2131 tstat_total_pages = TSTAT_INSTR_PAGES + tstat_data_pages; 2132 tstat_data_size = tstat_data_pages * MMU_PAGESIZE; 2133 tstat_total_size = TSTAT_INSTR_SIZE + tstat_data_size; 2134 #else 2135 tstat_data_pages = 0; 2136 tstat_data_size = tstat_data_t_size; 2137 tstat_total_pages = ((TSTAT_INSTR_SIZE + tstat_data_size) >> 2138 MMU_PAGESHIFT) + 1; 2139 tstat_total_size = tstat_total_pages * MMU_PAGESIZE; 2140 #endif 2141 2142 tstat_percpu = kmem_zalloc((max_cpuid + 1) * 2143 sizeof (tstat_percpu_t), KM_SLEEP); 2144 2145 /* 2146 * Create our own arena backed by segkmem to assure a source of 2147 * MMU_PAGESIZE-aligned allocations. We allocate out of the 2148 * heap32_arena to assure that we can address the allocated memory with 2149 * a single sethi/simm13 pair in the interposing trap table entries. 2150 */ 2151 tstat_arena = vmem_create("trapstat", NULL, 0, MMU_PAGESIZE, 2152 segkmem_alloc_permanent, segkmem_free, heap32_arena, 0, VM_SLEEP); 2153 2154 tstat_enabled = kmem_alloc(TSTAT_TOTAL_NENT * sizeof (int), KM_SLEEP); 2155 tstat_buffer = kmem_alloc(tstat_data_t_size, KM_SLEEP); 2156 2157 /* 2158 * CB_CL_CPR_POST_USER is the class that executes from cpr_resume() 2159 * after user threads can be restarted. By executing in this class, 2160 * we are assured of the availability of system services needed to 2161 * resume trapstat (specifically, we are assured that all CPUs are 2162 * restarted and responding to cross calls). 2163 */ 2164 tstat_cprcb = 2165 callb_add(trapstat_cpr, NULL, CB_CL_CPR_POST_USER, "trapstat"); 2166 2167 return (DDI_SUCCESS); 2168 } 2169 2170 static int 2171 trapstat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 2172 { 2173 int rval; 2174 2175 ASSERT(devi == tstat_devi); 2176 2177 switch (cmd) { 2178 case DDI_DETACH: 2179 break; 2180 2181 case DDI_SUSPEND: 2182 return (DDI_SUCCESS); 2183 2184 default: 2185 return (DDI_FAILURE); 2186 } 2187 2188 ASSERT(!tstat_running); 2189 2190 rval = callb_delete(tstat_cprcb); 2191 ASSERT(rval == 0); 2192 2193 kmem_free(tstat_buffer, tstat_data_t_size); 2194 kmem_free(tstat_enabled, TSTAT_TOTAL_NENT * sizeof (int)); 2195 vmem_destroy(tstat_arena); 2196 kmem_free(tstat_percpu, (max_cpuid + 1) * sizeof (tstat_percpu_t)); 2197 ddi_remove_minor_node(devi, NULL); 2198 2199 return (DDI_SUCCESS); 2200 } 2201 2202 /* 2203 * Configuration data structures 2204 */ 2205 static struct cb_ops trapstat_cb_ops = { 2206 trapstat_open, /* open */ 2207 trapstat_close, /* close */ 2208 nulldev, /* strategy */ 2209 nulldev, /* print */ 2210 nodev, /* dump */ 2211 nodev, /* read */ 2212 nodev, /* write */ 2213 trapstat_ioctl, /* ioctl */ 2214 nodev, /* devmap */ 2215 nodev, /* mmap */ 2216 nodev, /* segmap */ 2217 nochpoll, /* poll */ 2218 ddi_prop_op, /* cb_prop_op */ 2219 0, /* streamtab */ 2220 D_MP | D_NEW /* Driver compatibility flag */ 2221 }; 2222 2223 static struct dev_ops trapstat_ops = { 2224 DEVO_REV, /* devo_rev, */ 2225 0, /* refcnt */ 2226 trapstat_info, /* getinfo */ 2227 nulldev, /* identify */ 2228 nulldev, /* probe */ 2229 trapstat_attach, /* attach */ 2230 trapstat_detach, /* detach */ 2231 nulldev, /* reset */ 2232 &trapstat_cb_ops, /* cb_ops */ 2233 (struct bus_ops *)0, /* bus_ops */ 2234 }; 2235 2236 static struct modldrv modldrv = { 2237 &mod_driverops, /* Type of module. This one is a driver */ 2238 "Trap Statistics", /* name of module */ 2239 &trapstat_ops, /* driver ops */ 2240 }; 2241 2242 static struct modlinkage modlinkage = { 2243 MODREV_1, (void *)&modldrv, NULL 2244 }; 2245 2246 int 2247 _init(void) 2248 { 2249 return (mod_install(&modlinkage)); 2250 } 2251 2252 int 2253 _fini(void) 2254 { 2255 return (mod_remove(&modlinkage)); 2256 } 2257 2258 int 2259 _info(struct modinfo *modinfop) 2260 { 2261 return (mod_info(&modlinkage, modinfop)); 2262 } 2263