1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * This file contains preset event names from the Performance Application 27 * Programming Interface v3.5 which included the following notice: 28 * 29 * Copyright (c) 2005,6 30 * Innovative Computing Labs 31 * Computer Science Department, 32 * University of Tennessee, 33 * Knoxville, TN. 34 * All Rights Reserved. 35 * 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions are met: 39 * 40 * * Redistributions of source code must retain the above copyright notice, 41 * this list of conditions and the following disclaimer. 42 * * Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * * Neither the name of the University of Tennessee nor the names of its 46 * contributors may be used to endorse or promote products derived from 47 * this software without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 50 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 53 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 * 61 * 62 * This open source software license conforms to the BSD License template. 63 */ 64 65 /* 66 * Performance Counter Back-End for Pentiums I, II, and III. 67 */ 68 69 #include <sys/cpuvar.h> 70 #include <sys/param.h> 71 #include <sys/cpc_impl.h> 72 #include <sys/cpc_pcbe.h> 73 #include <sys/modctl.h> 74 #include <sys/inttypes.h> 75 #include <sys/systm.h> 76 #include <sys/cmn_err.h> 77 #include <sys/x86_archext.h> 78 #include <sys/sdt.h> 79 #include <sys/archsystm.h> 80 #include <sys/privregs.h> 81 #include <sys/ddi.h> 82 #include <sys/sunddi.h> 83 84 static int64_t diff3931(uint64_t sample, uint64_t old); 85 static uint64_t trunc3931(uint64_t value); 86 87 static int ptm_pcbe_init(void); 88 static uint_t ptm_pcbe_ncounters(void); 89 static const char *ptm_pcbe_impl_name(void); 90 static const char *ptm_pcbe_cpuref(void); 91 static char *ptm_pcbe_list_events(uint_t picnum); 92 static char *ptm_pcbe_list_attrs(void); 93 static uint64_t ptm_pcbe_event_coverage(char *event); 94 static int ptm_pcbe_pic_index(char *picname); 95 static uint64_t ptm_pcbe_overflow_bitmap(void); 96 static int ptm_pcbe_configure(uint_t picnum, char *event, uint64_t preset, 97 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data, 98 void *token); 99 static void ptm_pcbe_program(void *token); 100 static void ptm_pcbe_allstop(void); 101 static void ptm_pcbe_sample(void *token); 102 static void ptm_pcbe_free(void *config); 103 104 pcbe_ops_t ptm_pcbe_ops = { 105 PCBE_VER_1, 106 0, 107 ptm_pcbe_ncounters, 108 ptm_pcbe_impl_name, 109 ptm_pcbe_cpuref, 110 ptm_pcbe_list_events, 111 ptm_pcbe_list_attrs, 112 ptm_pcbe_event_coverage, 113 ptm_pcbe_overflow_bitmap, 114 ptm_pcbe_configure, 115 ptm_pcbe_program, 116 ptm_pcbe_allstop, 117 ptm_pcbe_sample, 118 ptm_pcbe_free 119 }; 120 121 typedef enum _ptm_ver { 122 PTM_VER_P5, 123 PTM_VER_P6 124 } ptm_ver_t; 125 126 static ptm_ver_t ptm_ver; 127 static const char *ptm_impl_name; 128 static const char *ptm_cpuref; 129 static char *pic_events[2] = { NULL, NULL }; 130 131 /* 132 * Indicates whether the "rdpmc" instruction is available on this processor. 133 */ 134 static int ptm_rdpmc_avail = 0; 135 136 #define ALL_STOPPED 0ULL 137 138 typedef struct _ptm_pcbe_config { 139 uint8_t ptm_picno; /* 0 for pic0 or 1 for pic1 */ 140 uint32_t ptm_ctl; /* P6: PerfEventSelect; P5: cesr, shifted */ 141 uint64_t ptm_rawpic; 142 } ptm_pcbe_config_t; 143 144 struct nametable { 145 uint8_t bits; 146 const char *name; 147 }; 148 149 typedef struct _ptm_generic_events { 150 char *name; 151 char *event; 152 uint8_t umask; 153 } ptm_generic_event_t; 154 155 #define NT_END 0xFF 156 #define CPC_GEN_END { NULL, NULL } 157 158 /* 159 * Basic Pentium events 160 */ 161 #define P5_EVENTS \ 162 {0x0, "data_read"}, \ 163 {0x1, "data_write"}, \ 164 {0x2, "data_tlb_miss"}, \ 165 {0x3, "data_read_miss"}, \ 166 {0x4, "data_write_miss"}, \ 167 {0x5, "write_hit_to_M_or_E"}, \ 168 {0x6, "dcache_lines_wrback"}, \ 169 {0x7, "external_snoops"}, \ 170 {0x8, "external_dcache_snoop_hits"}, \ 171 {0x9, "memory_access_in_both_pipes"}, \ 172 {0xa, "bank_conflicts"}, \ 173 {0xb, "misaligned_ref"}, \ 174 {0xc, "code_read"}, \ 175 {0xd, "code_tlb_miss"}, \ 176 {0xe, "code_cache_miss"}, \ 177 {0xf, "any_segreg_loaded"}, \ 178 {0x12, "branches"}, \ 179 {0x13, "btb_hits"}, \ 180 {0x14, "taken_or_btb_hit"}, \ 181 {0x15, "pipeline_flushes"}, \ 182 {0x16, "instr_exec"}, \ 183 {0x17, "instr_exec_V_pipe"}, \ 184 {0x18, "clks_bus_cycle"}, \ 185 {0x19, "clks_full_wbufs"}, \ 186 {0x1a, "pipe_stall_read"}, \ 187 {0x1b, "stall_on_write_ME"}, \ 188 {0x1c, "locked_bus_cycle"}, \ 189 {0x1d, "io_rw_cycles"}, \ 190 {0x1e, "reads_noncache_mem"}, \ 191 {0x1f, "pipeline_agi_stalls"}, \ 192 {0x22, "flops"}, \ 193 {0x23, "bp_match_dr0"}, \ 194 {0x24, "bp_match_dr1"}, \ 195 {0x25, "bp_match_dr2"}, \ 196 {0x26, "bp_match_dr3"}, \ 197 {0x27, "hw_intrs"}, \ 198 {0x28, "data_rw"}, \ 199 {0x29, "data_rw_miss"} 200 201 static const struct nametable P5mmx_names0[] = { 202 P5_EVENTS, 203 {0x2a, "bus_ownership_latency"}, 204 {0x2b, "mmx_instr_upipe"}, 205 {0x2c, "cache_M_line_sharing"}, 206 {0x2d, "emms_instr"}, 207 {0x2e, "bus_util_processor"}, 208 {0x2f, "sat_mmx_instr"}, 209 {0x30, "clks_not_HLT"}, 210 {0x31, "mmx_data_read"}, 211 {0x32, "clks_fp_stall"}, 212 {0x33, "d1_starv_fifo_0"}, 213 {0x34, "mmx_data_write"}, 214 {0x35, "pipe_flush_wbp"}, 215 {0x36, "mmx_misalign_data_refs"}, 216 {0x37, "rets_pred_incorrect"}, 217 {0x38, "mmx_multiply_unit_interlock"}, 218 {0x39, "rets"}, 219 {0x3a, "btb_false_entries"}, 220 {0x3b, "clocks_stall_full_wb"}, 221 {NT_END, ""} 222 }; 223 224 static const struct nametable P5mmx_names1[] = { 225 P5_EVENTS, 226 {0x2a, "bus_ownership_transfers"}, 227 {0x2b, "mmx_instr_vpipe"}, 228 {0x2c, "cache_lint_sharing"}, 229 {0x2d, "mmx_fp_transitions"}, 230 {0x2e, "writes_noncache_mem"}, 231 {0x2f, "sats_performed"}, 232 {0x30, "clks_dcache_tlb_miss"}, 233 {0x31, "mmx_data_read_miss"}, 234 {0x32, "taken_br"}, 235 {0x33, "d1_starv_fifo_1"}, 236 {0x34, "mmx_data_write_miss"}, 237 {0x35, "pipe_flush_wbp_wb"}, 238 {0x36, "mmx_pipe_stall_data_read"}, 239 {0x37, "rets_pred"}, 240 {0x38, "movd_movq_stall"}, 241 {0x39, "rsb_overflow"}, 242 {0x3a, "btb_mispred_nt"}, 243 {0x3b, "mmx_stall_write_ME"}, 244 {NT_END, ""} 245 }; 246 247 static const struct nametable *P5mmx_names[2] = { 248 P5mmx_names0, 249 P5mmx_names1 250 }; 251 252 /* 253 * Pentium Pro and Pentium II events 254 */ 255 static const struct nametable _P6_names[] = { 256 /* 257 * Data cache unit 258 */ 259 {0x43, "data_mem_refs"}, 260 {0x45, "dcu_lines_in"}, 261 {0x46, "dcu_m_lines_in"}, 262 {0x47, "dcu_m_lines_out"}, 263 {0x48, "dcu_miss_outstanding"}, 264 265 /* 266 * Instruction fetch unit 267 */ 268 {0x80, "ifu_ifetch"}, 269 {0x81, "ifu_ifetch_miss"}, 270 {0x85, "itlb_miss"}, 271 {0x86, "ifu_mem_stall"}, 272 {0x87, "ild_stall"}, 273 274 /* 275 * L2 cache 276 */ 277 {0x28, "l2_ifetch"}, 278 {0x29, "l2_ld"}, 279 {0x2a, "l2_st"}, 280 {0x24, "l2_lines_in"}, 281 {0x26, "l2_lines_out"}, 282 {0x25, "l2_m_lines_inm"}, 283 {0x27, "l2_m_lines_outm"}, 284 {0x2e, "l2_rqsts"}, 285 {0x21, "l2_ads"}, 286 {0x22, "l2_dbus_busy"}, 287 {0x23, "l2_dbus_busy_rd"}, 288 289 /* 290 * External bus logic 291 */ 292 {0x62, "bus_drdy_clocks"}, 293 {0x63, "bus_lock_clocks"}, 294 {0x60, "bus_req_outstanding"}, 295 {0x65, "bus_tran_brd"}, 296 {0x66, "bus_tran_rfo"}, 297 {0x67, "bus_trans_wb"}, 298 {0x68, "bus_tran_ifetch"}, 299 {0x69, "bus_tran_inval"}, 300 {0x6a, "bus_tran_pwr"}, 301 {0x6b, "bus_trans_p"}, 302 {0x6c, "bus_trans_io"}, 303 {0x6d, "bus_tran_def"}, 304 {0x6e, "bus_tran_burst"}, 305 {0x70, "bus_tran_any"}, 306 {0x6f, "bus_tran_mem"}, 307 {0x64, "bus_data_rcv"}, 308 {0x61, "bus_bnr_drv"}, 309 {0x7a, "bus_hit_drv"}, 310 {0x7b, "bus_hitm_drv"}, 311 {0x7e, "bus_snoop_stall"}, 312 313 /* 314 * Floating point unit 315 */ 316 {0xc1, "flops"}, /* 0 only */ 317 {0x10, "fp_comp_ops_exe"}, /* 0 only */ 318 {0x11, "fp_assist"}, /* 1 only */ 319 {0x12, "mul"}, /* 1 only */ 320 {0x13, "div"}, /* 1 only */ 321 {0x14, "cycles_div_busy"}, /* 0 only */ 322 323 /* 324 * Memory ordering 325 */ 326 {0x3, "ld_blocks"}, 327 {0x4, "sb_drains"}, 328 {0x5, "misalign_mem_ref"}, 329 330 /* 331 * Instruction decoding and retirement 332 */ 333 {0xc0, "inst_retired"}, 334 {0xc2, "uops_retired"}, 335 {0xd0, "inst_decoder"}, 336 337 /* 338 * Interrupts 339 */ 340 {0xc8, "hw_int_rx"}, 341 {0xc6, "cycles_int_masked"}, 342 {0xc7, "cycles_int_pending_and_masked"}, 343 344 /* 345 * Branches 346 */ 347 {0xc4, "br_inst_retired"}, 348 {0xc5, "br_miss_pred_retired"}, 349 {0xc9, "br_taken_retired"}, 350 {0xca, "br_miss_pred_taken_ret"}, 351 {0xe0, "br_inst_decoded"}, 352 {0xe2, "btb_misses"}, 353 {0xe4, "br_bogus"}, 354 {0xe6, "baclears"}, 355 356 /* 357 * Stalls 358 */ 359 {0xa2, "resource_stalls"}, 360 {0xd2, "partial_rat_stalls"}, 361 362 /* 363 * Segment register loads 364 */ 365 {0x6, "segment_reg_loads"}, 366 367 /* 368 * Clocks 369 */ 370 {0x79, "cpu_clk_unhalted"}, 371 372 /* 373 * MMX 374 */ 375 {0xb0, "mmx_instr_exec"}, 376 {0xb1, "mmx_sat_instr_exec"}, 377 {0xb2, "mmx_uops_exec"}, 378 {0xb3, "mmx_instr_type_exec"}, 379 {0xcc, "fp_mmx_trans"}, 380 {0xcd, "mmx_assists"}, 381 {0xce, "mmx_instr_ret"}, 382 {0xd4, "seg_rename_stalls"}, 383 {0xd5, "seg_reg_renames"}, 384 {0xd6, "ret_seg_renames"}, 385 386 {NT_END, ""} 387 }; 388 389 static const struct nametable *P6_names[2] = { 390 _P6_names, 391 _P6_names 392 }; 393 394 #define P5_GENERIC_EVENTS \ 395 { "PAPI_tot_ins", "instr_exec", 0x0 }, \ 396 { "PAPI_tlb_dm", "data_tlb_miss", 0x0 }, \ 397 { "PAPI_tlb_im", "code_tlb_miss", 0x0 }, \ 398 { "PAPI_fp_ops", "flops" } 399 400 static const ptm_generic_event_t P5mmx_generic_names0[] = { 401 P5_GENERIC_EVENTS, 402 { "PAPI_tot_cyc", "clks_not_HLT", 0x0 }, 403 CPC_GEN_END 404 }; 405 406 static const ptm_generic_event_t P5mmx_generic_names1[] = { 407 P5_GENERIC_EVENTS, 408 { "PAPI_br_ins", "taken_br", 0x0 }, 409 CPC_GEN_END 410 }; 411 412 static const ptm_generic_event_t *P5mmx_generic_names[2] = { 413 P5mmx_generic_names0, 414 P5mmx_generic_names1 415 }; 416 417 static const ptm_generic_event_t _P6_generic_names[] = { 418 { "PAPI_ca_shr", "l2_ifetch", 0xf }, 419 { "PAPI_ca_cln", "bus_tran_rfo", 0x0 }, 420 { "PAPI_ca_itv", "bus_tran_inval", 0x0 }, 421 { "PAPI_tlb_im", "itlb_miss", 0x0 }, 422 { "PAPI_btac_m", "btb_misses", 0x0 }, 423 { "PAPI_hw_int", "hw_int_rx", 0x0 }, 424 { "PAPI_br_cn", "br_inst_retired", 0x0 }, 425 { "PAPI_br_tkn", "br_taken_retired", 0x0 }, 426 { "PAPI_br_msp", "br_miss_pred_taken_ret", 0x0 }, 427 { "PAPI_br_ins", "br_inst_retired", 0x0 }, 428 { "PAPI_res_stl", "resource_stalls", 0x0 }, 429 { "PAPI_tot_iis", "inst_decoder", 0x0 }, 430 { "PAPI_tot_ins", "inst_retired", 0x0 }, 431 { "PAPI_tot_cyc", "cpu_clk_unhalted", 0x0 }, 432 { "PAPI_l1_dcm", "dcu_lines_in", 0x0 }, 433 { "PAPI_l1_icm", "l2_ifetch", 0xf }, 434 { "PAPI_l1_tcm", "l2_rqsts", 0xf }, 435 { "PAPI_l1_dca", "data_mem_refs", 0x0 }, 436 { "PAPI_l1_stm", "l2_st", 0xf }, 437 { "PAPI_l2_icm", "bus_tran_ifetch", 0x0 }, 438 { "PAPI_l2_dcr", "l2_ld", 0xf }, 439 { "PAPI_l2_dcw", "l2_st", 0xf }, 440 { "PAPI_l2_tcm", "l2_lines_in", 0x0 }, 441 { "PAPI_l2_tca", "l2_rqsts", 0xf }, 442 { "PAPI_l2_tcw", "l2_st", 0xf }, 443 { "PAPI_l2_stm", "l2_m_lines_inm", 0x0 }, 444 { "PAPI_fp_ins", "flops", 0x0 }, 445 { "PAPI_fp_ops", "flops", 0x0 }, 446 { "PAPI_fml_ins", "mul", 0x0 }, 447 { "PAPI_fdv_ins", "div", 0x0 }, 448 CPC_GEN_END 449 }; 450 451 static const ptm_generic_event_t *P6_generic_names[2] = { 452 _P6_generic_names, 453 _P6_generic_names 454 }; 455 456 static const struct nametable **events; 457 static const ptm_generic_event_t **generic_events; 458 459 #define BITS(v, u, l) \ 460 (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1)) 461 462 /* 463 * "Well known" bit fields in the Pentium CES register 464 * The interfaces in libcpc should make these #defines uninteresting. 465 */ 466 #define CPC_P5_CESR_ES0_SHIFT 0 467 #define CPC_P5_CESR_ES0_MASK 0x3f 468 #define CPC_P5_CESR_ES1_SHIFT 16 469 #define CPC_P5_CESR_ES1_MASK 0x3f 470 471 #define CPC_P5_CESR_OS0 6 472 #define CPC_P5_CESR_USR0 7 473 #define CPC_P5_CESR_CLK0 8 474 #define CPC_P5_CESR_PC0 9 475 #define CPC_P5_CESR_OS1 (CPC_P5_CESR_OS0 + 16) 476 #define CPC_P5_CESR_USR1 (CPC_P5_CESR_USR0 + 16) 477 #define CPC_P5_CESR_CLK1 (CPC_P5_CESR_CLK0 + 16) 478 #define CPC_P5_CESR_PC1 (CPC_P5_CESR_PC0 + 16) 479 480 /* 481 * "Well known" bit fields in the Pentium Pro PerfEvtSel registers 482 * The interfaces in libcpc should make these #defines uninteresting. 483 */ 484 #define CPC_P6_PES_INV 23 485 #define CPC_P6_PES_EN 22 486 #define CPC_P6_PES_INT 20 487 #define CPC_P6_PES_PC 19 488 #define CPC_P6_PES_E 18 489 #define CPC_P6_PES_OS 17 490 #define CPC_P6_PES_USR 16 491 492 #define CPC_P6_PES_UMASK_SHIFT 8 493 #define CPC_P6_PES_UMASK_MASK (0xffu) 494 495 #define CPC_P6_PES_CMASK_SHIFT 24 496 #define CPC_P6_PES_CMASK_MASK (0xffu) 497 498 #define CPC_P6_PES_PIC0_MASK (0xffu) 499 #define CPC_P6_PES_PIC1_MASK (0xffu) 500 501 #define P6_PES_EN (UINT32_C(1) << CPC_P6_PES_EN) 502 #define P6_PES_INT (UINT32_C(1) << CPC_P6_PES_INT) 503 #define P6_PES_OS (UINT32_C(1) << CPC_P6_PES_OS) 504 505 /* 506 * Pentium 5 attributes 507 */ 508 #define P5_NOEDGE 0x1 /* "noedge" - no edge detection */ 509 #define P5_PC 0x2 /* "pc" - pin control */ 510 511 /* 512 * Pentium 6 attributes 513 */ 514 #define P6_NOEDGE 0x1 515 #define P6_PC 0x2 516 #define P6_INV 0x4 /* "inv" - count inverted transitions */ 517 #define P6_INT 0x8 /* "int" - interrupt on overflow */ 518 519 /* 520 * CPU reference strings 521 */ 522 523 #define P5_CPUREF "See Appendix A.4 of the \"IA-32 Intel Architecture " \ 524 "Software Developer's Manual Volume 3: System " \ 525 "Programming Guide,\" Order # 245472-012, 2003" 526 527 #define P6_CPUREF "See Appendix A.3 of the \"IA-32 Intel Architecture " \ 528 "Software Developer's Manual Volume 3: System " \ 529 "Programming Guide,\" Order # 245472-012, 2003" 530 531 static int 532 ptm_pcbe_init(void) 533 { 534 const struct nametable *n; 535 const ptm_generic_event_t *gevp; 536 int i; 537 size_t size; 538 539 if (is_x86_feature(x86_featureset, X86FSET_MMX)) 540 ptm_rdpmc_avail = 1; 541 542 /* 543 * Discover type of CPU and set events pointer appropriately. 544 * 545 * Map family and model into the performance 546 * counter architectures we currently understand. 547 * 548 * See application note AP485 (from developer.intel.com) 549 * for further explanation. 550 */ 551 if (cpuid_getvendor(CPU) != X86_VENDOR_Intel) 552 return (-1); 553 switch (cpuid_getfamily(CPU)) { 554 case 5: /* Pentium and Pentium with MMX */ 555 events = P5mmx_names; 556 generic_events = P5mmx_generic_names; 557 ptm_ver = PTM_VER_P5; 558 ptm_cpuref = P5_CPUREF; 559 if (cpuid_getmodel(CPU) < 4) 560 ptm_impl_name = "Pentium"; 561 else 562 ptm_impl_name = "Pentium with MMX"; 563 break; 564 case 6: /* Pentium Pro and Pentium II and III */ 565 events = P6_names; 566 generic_events = P6_generic_names; 567 ptm_ver = PTM_VER_P6; 568 ptm_cpuref = P6_CPUREF; 569 ptm_pcbe_ops.pcbe_caps = CPC_CAP_OVERFLOW_INTERRUPT; 570 if (is_x86_feature(x86_featureset, X86FSET_MMX)) 571 ptm_impl_name = "Pentium Pro with MMX, Pentium II"; 572 else 573 ptm_impl_name = "Pentium Pro, Pentium II"; 574 break; 575 default: 576 return (-1); 577 } 578 579 /* 580 * Initialize the list of events for each PIC. 581 * Do two passes: one to compute the size necessary and another 582 * to copy the strings. Need room for event, comma, and NULL terminator. 583 */ 584 for (i = 0; i < 2; i++) { 585 size = 0; 586 for (n = events[i]; n->bits != NT_END; n++) 587 size += strlen(n->name) + 1; 588 for (gevp = generic_events[i]; gevp->name != NULL; gevp++) 589 size += strlen(gevp->name) + 1; 590 pic_events[i] = kmem_alloc(size + 1, KM_SLEEP); 591 *pic_events[i] = '\0'; 592 for (n = events[i]; n->bits != NT_END; n++) { 593 (void) strcat(pic_events[i], n->name); 594 (void) strcat(pic_events[i], ","); 595 } 596 for (gevp = generic_events[i]; gevp->name != NULL; gevp++) { 597 (void) strcat(pic_events[i], gevp->name); 598 (void) strcat(pic_events[i], ","); 599 } 600 601 /* 602 * Remove trailing comma. 603 */ 604 pic_events[i][size - 1] = '\0'; 605 } 606 607 return (0); 608 } 609 610 static uint_t 611 ptm_pcbe_ncounters(void) 612 { 613 return (2); 614 } 615 616 static const char * 617 ptm_pcbe_impl_name(void) 618 { 619 return (ptm_impl_name); 620 } 621 622 static const char * 623 ptm_pcbe_cpuref(void) 624 { 625 return (ptm_cpuref); 626 } 627 628 static char * 629 ptm_pcbe_list_events(uint_t picnum) 630 { 631 ASSERT(picnum >= 0 && picnum < cpc_ncounters); 632 633 if (pic_events[0] == NULL) { 634 ASSERT(pic_events[1] == NULL); 635 } 636 637 return (pic_events[picnum]); 638 } 639 640 static char * 641 ptm_pcbe_list_attrs(void) 642 { 643 if (ptm_ver == PTM_VER_P5) 644 return ("noedge,pc"); 645 else 646 return ("noedge,pc,inv,int,umask,cmask"); 647 } 648 649 static const ptm_generic_event_t * 650 find_generic_event(int regno, char *name) 651 { 652 const ptm_generic_event_t *gevp; 653 654 for (gevp = generic_events[regno]; gevp->name != NULL; gevp++) 655 if (strcmp(name, gevp->name) == 0) 656 return (gevp); 657 658 return (NULL); 659 } 660 661 static const struct nametable * 662 find_event(int regno, char *name) 663 { 664 const struct nametable *n; 665 666 n = events[regno]; 667 668 for (; n->bits != NT_END; n++) 669 if (strcmp(name, n->name) == 0) 670 return (n); 671 672 return (NULL); 673 } 674 675 static uint64_t 676 ptm_pcbe_event_coverage(char *event) 677 { 678 uint64_t bitmap = 0; 679 680 if ((find_event(0, event) != NULL) || 681 (find_generic_event(0, event) != NULL)) 682 bitmap = 0x1; 683 if ((find_event(1, event) != NULL) || 684 (find_generic_event(1, event) != NULL)) 685 bitmap |= 0x2; 686 687 return (bitmap); 688 } 689 690 static uint64_t 691 ptm_pcbe_overflow_bitmap(void) 692 { 693 uint64_t ret = 0; 694 uint64_t pes[2]; 695 696 /* 697 * P5 is not capable of generating interrupts. 698 */ 699 ASSERT(ptm_ver == PTM_VER_P6); 700 701 /* 702 * CPC could have caused an interrupt provided that 703 * 704 * 1) Counters are enabled 705 * 2) Either counter has requested an interrupt 706 */ 707 708 pes[0] = rdmsr(REG_PERFEVNT0); 709 if (((uint32_t)pes[0] & P6_PES_EN) != P6_PES_EN) 710 return (0); 711 712 /* 713 * If a particular counter requested an interrupt, assume it caused 714 * this interrupt. There is no way to determine which counter overflowed 715 * on this hardware other than by using unreliable heuristics. 716 */ 717 718 pes[1] = rdmsr(REG_PERFEVNT1); 719 if ((uint32_t)pes[0] & P6_PES_INT) 720 ret |= 0x1; 721 if ((uint32_t)pes[1] & P6_PES_INT) 722 ret |= 0x2; 723 724 return (ret); 725 } 726 727 /*ARGSUSED*/ 728 static int 729 ptm_pcbe_configure(uint_t picnum, char *eventname, uint64_t preset, 730 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data, 731 void *token) 732 { 733 ptm_pcbe_config_t *conf; 734 const struct nametable *n; 735 const ptm_generic_event_t *gevp; 736 struct nametable nt_raw = { 0, "raw" }; 737 int i; 738 int ptm_flags = 0; 739 740 /* 741 * If we've been handed an existing configuration, we need only preset 742 * the counter value. 743 */ 744 if (*data != NULL) { 745 conf = *data; 746 conf->ptm_rawpic = trunc3931(preset); 747 return (0); 748 } 749 750 if (picnum != 0 && picnum != 1) 751 return (CPC_INVALID_PICNUM); 752 753 conf = kmem_alloc(sizeof (ptm_pcbe_config_t), KM_SLEEP); 754 755 conf->ptm_picno = picnum; 756 conf->ptm_rawpic = trunc3931(preset); 757 conf->ptm_ctl = 0; 758 759 if ((n = find_event(picnum, eventname)) == NULL) { 760 if ((gevp = find_generic_event(picnum, eventname)) != NULL) { 761 n = find_event(picnum, gevp->event); 762 ASSERT(n != NULL); 763 764 if (nattrs > 0) { 765 kmem_free(conf, sizeof (ptm_pcbe_config_t)); 766 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 767 } 768 769 if (ptm_ver == PTM_VER_P6) 770 conf->ptm_ctl |= gevp->umask << 771 CPC_P6_PES_UMASK_SHIFT; 772 } else { 773 long tmp; 774 775 /* 776 * If ddi_strtol() likes this event, use it as a raw 777 * event code. 778 */ 779 if (ddi_strtol(eventname, NULL, 0, &tmp) != 0) { 780 kmem_free(conf, sizeof (ptm_pcbe_config_t)); 781 return (CPC_INVALID_EVENT); 782 } 783 784 nt_raw.bits = tmp; 785 786 if (ptm_ver == PTM_VER_P5) 787 nt_raw.bits &= CPC_P5_CESR_ES0_MASK; 788 else 789 nt_raw.bits &= CPC_P6_PES_PIC0_MASK; 790 791 n = &nt_raw; 792 } 793 } 794 795 if (ptm_ver == PTM_VER_P5) { 796 int picshift; 797 picshift = (picnum == 0) ? 0 : 16; 798 799 for (i = 0; i < nattrs; i++) { 800 /* 801 * Value of these attributes is ignored; their presence 802 * alone tells us to set the corresponding flag. 803 */ 804 if (strncmp(attrs[i].ka_name, "noedge", 7) == 0) { 805 if (attrs[i].ka_val != 0) 806 ptm_flags |= P5_NOEDGE; 807 } else if (strncmp(attrs[i].ka_name, "pc", 3) == 0) { 808 if (attrs[i].ka_val != 0) 809 ptm_flags |= P5_PC; 810 } else { 811 kmem_free(conf, sizeof (ptm_pcbe_config_t)); 812 return (CPC_INVALID_ATTRIBUTE); 813 } 814 } 815 816 if (flags & CPC_COUNT_USER) 817 conf->ptm_ctl |= (1 << (CPC_P5_CESR_USR0 + picshift)); 818 if (flags & CPC_COUNT_SYSTEM) 819 conf->ptm_ctl |= (1 << (CPC_P5_CESR_OS0 + picshift)); 820 if (ptm_flags & P5_NOEDGE) 821 conf->ptm_ctl |= (1 << (CPC_P5_CESR_CLK0 + picshift)); 822 if (ptm_flags & P5_PC) 823 conf->ptm_ctl |= (1 << (CPC_P5_CESR_PC0 + picshift)); 824 825 ASSERT((n->bits | CPC_P5_CESR_ES0_MASK) == 826 CPC_P5_CESR_ES0_MASK); 827 828 conf->ptm_ctl |= (n->bits << picshift); 829 } else { 830 for (i = 0; i < nattrs; i++) { 831 if (strncmp(attrs[i].ka_name, "noedge", 6) == 0) { 832 if (attrs[i].ka_val != 0) 833 ptm_flags |= P6_NOEDGE; 834 } else if (strncmp(attrs[i].ka_name, "pc", 2) == 0) { 835 if (attrs[i].ka_val != 0) 836 ptm_flags |= P6_PC; 837 } else if (strncmp(attrs[i].ka_name, "inv", 3) == 0) { 838 if (attrs[i].ka_val != 0) 839 ptm_flags |= P6_INV; 840 } else if (strncmp(attrs[i].ka_name, "umask", 5) == 0) { 841 if ((attrs[i].ka_val | CPC_P6_PES_UMASK_MASK) != 842 CPC_P6_PES_UMASK_MASK) { 843 kmem_free(conf, 844 sizeof (ptm_pcbe_config_t)); 845 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 846 } 847 conf->ptm_ctl |= (uint8_t)attrs[i].ka_val << 848 CPC_P6_PES_UMASK_SHIFT; 849 } else if (strncmp(attrs[i].ka_name, "cmask", 5) == 0) { 850 if ((attrs[i].ka_val | CPC_P6_PES_CMASK_MASK) != 851 CPC_P6_PES_CMASK_MASK) { 852 kmem_free(conf, 853 sizeof (ptm_pcbe_config_t)); 854 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 855 } 856 conf->ptm_ctl |= (uint8_t)attrs[i].ka_val << 857 CPC_P6_PES_CMASK_SHIFT; 858 } else if (strncmp(attrs[i].ka_name, "int", 3) == 0) { 859 if (attrs[i].ka_val != 0) 860 ptm_flags |= P6_INT; 861 } else { 862 kmem_free(conf, sizeof (ptm_pcbe_config_t)); 863 return (CPC_INVALID_ATTRIBUTE); 864 } 865 } 866 867 if (flags & CPC_OVF_NOTIFY_EMT) 868 /* 869 * If the user has requested notification of overflows, 870 * we automatically program the hardware to generate 871 * overflow interrupts. 872 */ 873 ptm_flags |= P6_INT; 874 if (flags & CPC_COUNT_USER) 875 conf->ptm_ctl |= (1 << CPC_P6_PES_USR); 876 if (flags & CPC_COUNT_SYSTEM) 877 conf->ptm_ctl |= (1 << CPC_P6_PES_OS); 878 if ((ptm_flags & P6_NOEDGE) == 0) 879 conf->ptm_ctl |= (1 << CPC_P6_PES_E); 880 if (ptm_flags & P6_PC) 881 conf->ptm_ctl |= (1 << CPC_P6_PES_PC); 882 if (ptm_flags & P6_INV) 883 conf->ptm_ctl |= (1 << CPC_P6_PES_INV); 884 if (ptm_flags & P6_INT) 885 conf->ptm_ctl |= (1 << CPC_P6_PES_INT); 886 887 ASSERT((n->bits | CPC_P6_PES_PIC0_MASK) == 888 CPC_P6_PES_PIC0_MASK); 889 890 conf->ptm_ctl |= n->bits; 891 } 892 893 *data = conf; 894 return (0); 895 } 896 897 static void 898 ptm_pcbe_program(void *token) 899 { 900 ptm_pcbe_config_t *pic0; 901 ptm_pcbe_config_t *pic1; 902 ptm_pcbe_config_t *tmp; 903 ptm_pcbe_config_t empty = { 1, 0, 0 }; /* assume pic1 to start */ 904 905 if ((pic0 = kcpc_next_config(token, NULL, NULL)) == NULL) 906 panic("ptm_pcbe: token %p has no configs", token); 907 908 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) 909 pic1 = ∅ 910 911 if (pic0->ptm_picno != 0) { 912 empty.ptm_picno = 0; 913 tmp = pic1; 914 pic1 = pic0; 915 pic0 = tmp; 916 } 917 918 ASSERT(pic0->ptm_picno == 0 && pic1->ptm_picno == 1); 919 920 if (ptm_rdpmc_avail) { 921 ulong_t curcr4 = getcr4(); 922 if (kcpc_allow_nonpriv(token)) 923 setcr4(curcr4 | CR4_PCE); 924 else 925 setcr4(curcr4 & ~CR4_PCE); 926 } 927 928 if (ptm_ver == PTM_VER_P5) { 929 wrmsr(P5_CESR, ALL_STOPPED); 930 wrmsr(P5_CTR0, pic0->ptm_rawpic); 931 wrmsr(P5_CTR1, pic1->ptm_rawpic); 932 wrmsr(P5_CESR, pic0->ptm_ctl | pic1->ptm_ctl); 933 pic0->ptm_rawpic = rdmsr(P5_CTR0); 934 pic1->ptm_rawpic = rdmsr(P5_CTR1); 935 } else { 936 uint64_t pes; 937 wrmsr(REG_PERFEVNT0, ALL_STOPPED); 938 wrmsr(REG_PERFCTR0, pic0->ptm_rawpic); 939 wrmsr(REG_PERFCTR1, pic1->ptm_rawpic); 940 pes = pic1->ptm_ctl; 941 DTRACE_PROBE1(ptm__pes1, uint64_t, pes); 942 wrmsr(REG_PERFEVNT1, pes); 943 pes = pic0->ptm_ctl | (1 << CPC_P6_PES_EN); 944 DTRACE_PROBE1(ptm__pes0, uint64_t, pes); 945 wrmsr(REG_PERFEVNT0, pes); 946 } 947 } 948 949 static void 950 ptm_pcbe_allstop(void) 951 { 952 if (ptm_ver == PTM_VER_P5) 953 wrmsr(P5_CESR, ALL_STOPPED); 954 else { 955 wrmsr(REG_PERFEVNT0, ALL_STOPPED); 956 setcr4(getcr4() & ~CR4_PCE); 957 } 958 } 959 960 static void 961 ptm_pcbe_sample(void *token) 962 { 963 ptm_pcbe_config_t *pic0; 964 ptm_pcbe_config_t *pic1; 965 ptm_pcbe_config_t *swap; 966 ptm_pcbe_config_t empty = { 1, 0, 0 }; /* assume pic1 to start */ 967 uint64_t tmp; 968 uint64_t *pic0_data; 969 uint64_t *pic1_data; 970 uint64_t *dtmp; 971 uint64_t curpic[2]; 972 973 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL) 974 panic("ptm_pcbe: token %p has no configs", token); 975 976 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) { 977 pic1 = ∅ 978 pic1_data = &tmp; 979 } 980 981 if (pic0->ptm_picno != 0) { 982 empty.ptm_picno = 0; 983 swap = pic0; 984 pic0 = pic1; 985 pic1 = swap; 986 dtmp = pic0_data; 987 pic0_data = pic1_data; 988 pic1_data = dtmp; 989 } 990 991 ASSERT(pic0->ptm_picno == 0 && pic1->ptm_picno == 1); 992 993 if (ptm_ver == PTM_VER_P5) { 994 curpic[0] = rdmsr(P5_CTR0); 995 curpic[1] = rdmsr(P5_CTR1); 996 } else { 997 curpic[0] = rdmsr(REG_PERFCTR0); 998 curpic[1] = rdmsr(REG_PERFCTR1); 999 } 1000 1001 DTRACE_PROBE1(ptm__curpic0, uint64_t, curpic[0]); 1002 DTRACE_PROBE1(ptm__curpic1, uint64_t, curpic[1]); 1003 1004 *pic0_data += diff3931(curpic[0], pic0->ptm_rawpic); 1005 pic0->ptm_rawpic = trunc3931(*pic0_data); 1006 1007 *pic1_data += diff3931(curpic[1], pic1->ptm_rawpic); 1008 pic1->ptm_rawpic = trunc3931(*pic1_data); 1009 } 1010 1011 static void 1012 ptm_pcbe_free(void *config) 1013 { 1014 kmem_free(config, sizeof (ptm_pcbe_config_t)); 1015 } 1016 1017 /* 1018 * Virtualizes the 40-bit field of the %pic 1019 * register into a 64-bit software register. 1020 * 1021 * We can retrieve 40 (signed) bits from the counters, 1022 * but we can set only 32 (signed) bits into the counters. 1023 * This makes virtualizing more than 31-bits of registers 1024 * quite tricky. 1025 * 1026 * If bits 39 to 31 are set in the virtualized pic register, 1027 * then we can preset the counter to this value using the fact 1028 * that wrmsr sign extends bit 31. Though it might look easier 1029 * to only use the bottom 31-bits of the register, we have to allow 1030 * the full 40-bits to be used to perform overflow profiling. 1031 */ 1032 1033 #define MASK40 UINT64_C(0xffffffffff) 1034 #define MASK31 UINT64_C(0x7fffffff) 1035 #define BITS_39_31 UINT64_C(0xff80000000) 1036 1037 static int64_t 1038 diff3931(uint64_t sample, uint64_t old) 1039 { 1040 int64_t diff; 1041 1042 if ((old & BITS_39_31) == BITS_39_31) { 1043 diff = (MASK40 & sample) - old; 1044 if (diff < 0) 1045 diff += (UINT64_C(1) << 40); 1046 } else { 1047 diff = (MASK31 & sample) - old; 1048 if (diff < 0) 1049 diff += (UINT64_C(1) << 31); 1050 } 1051 return (diff); 1052 } 1053 1054 static uint64_t 1055 trunc3931(uint64_t value) 1056 { 1057 if ((value & BITS_39_31) == BITS_39_31) 1058 return (MASK40 & value); 1059 return (MASK31 & value); 1060 } 1061 1062 static struct modlpcbe modlpcbe = { 1063 &mod_pcbeops, 1064 "Pentium Performance Counters", 1065 &ptm_pcbe_ops 1066 }; 1067 1068 static struct modlinkage modl = { 1069 MODREV_1, 1070 &modlpcbe, 1071 }; 1072 1073 int 1074 _init(void) 1075 { 1076 if (ptm_pcbe_init() != 0) 1077 return (ENOTSUP); 1078 return (mod_install(&modl)); 1079 } 1080 1081 int 1082 _fini(void) 1083 { 1084 return (mod_remove(&modl)); 1085 } 1086 1087 int 1088 _info(struct modinfo *mi) 1089 { 1090 return (mod_info(&modl, mi)); 1091 } 1092