1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * This file contains preset event names from the Performance Application
27 * Programming Interface v3.5 which included the following notice:
28 *
29 * Copyright (c) 2005,6
30 * Innovative Computing Labs
31 * Computer Science Department,
32 * University of Tennessee,
33 * Knoxville, TN.
34 * All Rights Reserved.
35 *
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions are met:
39 *
40 * * Redistributions of source code must retain the above copyright notice,
41 * this list of conditions and the following disclaimer.
42 * * Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * * Neither the name of the University of Tennessee nor the names of its
46 * contributors may be used to endorse or promote products derived from
47 * this software without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
50 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
53 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
59 * POSSIBILITY OF SUCH DAMAGE.
60 *
61 *
62 * This open source software license conforms to the BSD License template.
63 */
64
65 /*
66 * Performance Counter Back-End for Pentiums I, II, and III.
67 */
68
69 #include <sys/cpuvar.h>
70 #include <sys/param.h>
71 #include <sys/cpc_impl.h>
72 #include <sys/cpc_pcbe.h>
73 #include <sys/modctl.h>
74 #include <sys/inttypes.h>
75 #include <sys/systm.h>
76 #include <sys/cmn_err.h>
77 #include <sys/x86_archext.h>
78 #include <sys/sdt.h>
79 #include <sys/archsystm.h>
80 #include <sys/privregs.h>
81 #include <sys/ddi.h>
82 #include <sys/sunddi.h>
83
84 static int64_t diff3931(uint64_t sample, uint64_t old);
85 static uint64_t trunc3931(uint64_t value);
86
87 static int ptm_pcbe_init(void);
88 static uint_t ptm_pcbe_ncounters(void);
89 static const char *ptm_pcbe_impl_name(void);
90 static const char *ptm_pcbe_cpuref(void);
91 static char *ptm_pcbe_list_events(uint_t picnum);
92 static char *ptm_pcbe_list_attrs(void);
93 static uint64_t ptm_pcbe_event_coverage(char *event);
94 static int ptm_pcbe_pic_index(char *picname);
95 static uint64_t ptm_pcbe_overflow_bitmap(void);
96 static int ptm_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
97 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
98 void *token);
99 static void ptm_pcbe_program(void *token);
100 static void ptm_pcbe_allstop(void);
101 static void ptm_pcbe_sample(void *token);
102 static void ptm_pcbe_free(void *config);
103
104 pcbe_ops_t ptm_pcbe_ops = {
105 PCBE_VER_1,
106 0,
107 ptm_pcbe_ncounters,
108 ptm_pcbe_impl_name,
109 ptm_pcbe_cpuref,
110 ptm_pcbe_list_events,
111 ptm_pcbe_list_attrs,
112 ptm_pcbe_event_coverage,
113 ptm_pcbe_overflow_bitmap,
114 ptm_pcbe_configure,
115 ptm_pcbe_program,
116 ptm_pcbe_allstop,
117 ptm_pcbe_sample,
118 ptm_pcbe_free
119 };
120
121 typedef enum _ptm_ver {
122 PTM_VER_P5,
123 PTM_VER_P6
124 } ptm_ver_t;
125
126 static ptm_ver_t ptm_ver;
127 static const char *ptm_impl_name;
128 static const char *ptm_cpuref;
129 static char *pic_events[2] = { NULL, NULL };
130
131 /*
132 * Indicates whether the "rdpmc" instruction is available on this processor.
133 */
134 static int ptm_rdpmc_avail = 0;
135
136 #define ALL_STOPPED 0ULL
137
138 typedef struct _ptm_pcbe_config {
139 uint8_t ptm_picno; /* 0 for pic0 or 1 for pic1 */
140 uint32_t ptm_ctl; /* P6: PerfEventSelect; P5: cesr, shifted */
141 uint64_t ptm_rawpic;
142 } ptm_pcbe_config_t;
143
144 struct nametable {
145 uint8_t bits;
146 const char *name;
147 };
148
149 typedef struct _ptm_generic_events {
150 char *name;
151 char *event;
152 uint8_t umask;
153 } ptm_generic_event_t;
154
155 #define NT_END 0xFF
156 #define CPC_GEN_END { NULL, NULL }
157
158 /*
159 * Basic Pentium events
160 */
161 #define P5_EVENTS \
162 {0x0, "data_read"}, \
163 {0x1, "data_write"}, \
164 {0x2, "data_tlb_miss"}, \
165 {0x3, "data_read_miss"}, \
166 {0x4, "data_write_miss"}, \
167 {0x5, "write_hit_to_M_or_E"}, \
168 {0x6, "dcache_lines_wrback"}, \
169 {0x7, "external_snoops"}, \
170 {0x8, "external_dcache_snoop_hits"}, \
171 {0x9, "memory_access_in_both_pipes"}, \
172 {0xa, "bank_conflicts"}, \
173 {0xb, "misaligned_ref"}, \
174 {0xc, "code_read"}, \
175 {0xd, "code_tlb_miss"}, \
176 {0xe, "code_cache_miss"}, \
177 {0xf, "any_segreg_loaded"}, \
178 {0x12, "branches"}, \
179 {0x13, "btb_hits"}, \
180 {0x14, "taken_or_btb_hit"}, \
181 {0x15, "pipeline_flushes"}, \
182 {0x16, "instr_exec"}, \
183 {0x17, "instr_exec_V_pipe"}, \
184 {0x18, "clks_bus_cycle"}, \
185 {0x19, "clks_full_wbufs"}, \
186 {0x1a, "pipe_stall_read"}, \
187 {0x1b, "stall_on_write_ME"}, \
188 {0x1c, "locked_bus_cycle"}, \
189 {0x1d, "io_rw_cycles"}, \
190 {0x1e, "reads_noncache_mem"}, \
191 {0x1f, "pipeline_agi_stalls"}, \
192 {0x22, "flops"}, \
193 {0x23, "bp_match_dr0"}, \
194 {0x24, "bp_match_dr1"}, \
195 {0x25, "bp_match_dr2"}, \
196 {0x26, "bp_match_dr3"}, \
197 {0x27, "hw_intrs"}, \
198 {0x28, "data_rw"}, \
199 {0x29, "data_rw_miss"}
200
201 static const struct nametable P5mmx_names0[] = {
202 P5_EVENTS,
203 {0x2a, "bus_ownership_latency"},
204 {0x2b, "mmx_instr_upipe"},
205 {0x2c, "cache_M_line_sharing"},
206 {0x2d, "emms_instr"},
207 {0x2e, "bus_util_processor"},
208 {0x2f, "sat_mmx_instr"},
209 {0x30, "clks_not_HLT"},
210 {0x31, "mmx_data_read"},
211 {0x32, "clks_fp_stall"},
212 {0x33, "d1_starv_fifo_0"},
213 {0x34, "mmx_data_write"},
214 {0x35, "pipe_flush_wbp"},
215 {0x36, "mmx_misalign_data_refs"},
216 {0x37, "rets_pred_incorrect"},
217 {0x38, "mmx_multiply_unit_interlock"},
218 {0x39, "rets"},
219 {0x3a, "btb_false_entries"},
220 {0x3b, "clocks_stall_full_wb"},
221 {NT_END, ""}
222 };
223
224 static const struct nametable P5mmx_names1[] = {
225 P5_EVENTS,
226 {0x2a, "bus_ownership_transfers"},
227 {0x2b, "mmx_instr_vpipe"},
228 {0x2c, "cache_lint_sharing"},
229 {0x2d, "mmx_fp_transitions"},
230 {0x2e, "writes_noncache_mem"},
231 {0x2f, "sats_performed"},
232 {0x30, "clks_dcache_tlb_miss"},
233 {0x31, "mmx_data_read_miss"},
234 {0x32, "taken_br"},
235 {0x33, "d1_starv_fifo_1"},
236 {0x34, "mmx_data_write_miss"},
237 {0x35, "pipe_flush_wbp_wb"},
238 {0x36, "mmx_pipe_stall_data_read"},
239 {0x37, "rets_pred"},
240 {0x38, "movd_movq_stall"},
241 {0x39, "rsb_overflow"},
242 {0x3a, "btb_mispred_nt"},
243 {0x3b, "mmx_stall_write_ME"},
244 {NT_END, ""}
245 };
246
247 static const struct nametable *P5mmx_names[2] = {
248 P5mmx_names0,
249 P5mmx_names1
250 };
251
252 /*
253 * Pentium Pro and Pentium II events
254 */
255 static const struct nametable _P6_names[] = {
256 /*
257 * Data cache unit
258 */
259 {0x43, "data_mem_refs"},
260 {0x45, "dcu_lines_in"},
261 {0x46, "dcu_m_lines_in"},
262 {0x47, "dcu_m_lines_out"},
263 {0x48, "dcu_miss_outstanding"},
264
265 /*
266 * Instruction fetch unit
267 */
268 {0x80, "ifu_ifetch"},
269 {0x81, "ifu_ifetch_miss"},
270 {0x85, "itlb_miss"},
271 {0x86, "ifu_mem_stall"},
272 {0x87, "ild_stall"},
273
274 /*
275 * L2 cache
276 */
277 {0x28, "l2_ifetch"},
278 {0x29, "l2_ld"},
279 {0x2a, "l2_st"},
280 {0x24, "l2_lines_in"},
281 {0x26, "l2_lines_out"},
282 {0x25, "l2_m_lines_inm"},
283 {0x27, "l2_m_lines_outm"},
284 {0x2e, "l2_rqsts"},
285 {0x21, "l2_ads"},
286 {0x22, "l2_dbus_busy"},
287 {0x23, "l2_dbus_busy_rd"},
288
289 /*
290 * External bus logic
291 */
292 {0x62, "bus_drdy_clocks"},
293 {0x63, "bus_lock_clocks"},
294 {0x60, "bus_req_outstanding"},
295 {0x65, "bus_tran_brd"},
296 {0x66, "bus_tran_rfo"},
297 {0x67, "bus_trans_wb"},
298 {0x68, "bus_tran_ifetch"},
299 {0x69, "bus_tran_inval"},
300 {0x6a, "bus_tran_pwr"},
301 {0x6b, "bus_trans_p"},
302 {0x6c, "bus_trans_io"},
303 {0x6d, "bus_tran_def"},
304 {0x6e, "bus_tran_burst"},
305 {0x70, "bus_tran_any"},
306 {0x6f, "bus_tran_mem"},
307 {0x64, "bus_data_rcv"},
308 {0x61, "bus_bnr_drv"},
309 {0x7a, "bus_hit_drv"},
310 {0x7b, "bus_hitm_drv"},
311 {0x7e, "bus_snoop_stall"},
312
313 /*
314 * Floating point unit
315 */
316 {0xc1, "flops"}, /* 0 only */
317 {0x10, "fp_comp_ops_exe"}, /* 0 only */
318 {0x11, "fp_assist"}, /* 1 only */
319 {0x12, "mul"}, /* 1 only */
320 {0x13, "div"}, /* 1 only */
321 {0x14, "cycles_div_busy"}, /* 0 only */
322
323 /*
324 * Memory ordering
325 */
326 {0x3, "ld_blocks"},
327 {0x4, "sb_drains"},
328 {0x5, "misalign_mem_ref"},
329
330 /*
331 * Instruction decoding and retirement
332 */
333 {0xc0, "inst_retired"},
334 {0xc2, "uops_retired"},
335 {0xd0, "inst_decoder"},
336
337 /*
338 * Interrupts
339 */
340 {0xc8, "hw_int_rx"},
341 {0xc6, "cycles_int_masked"},
342 {0xc7, "cycles_int_pending_and_masked"},
343
344 /*
345 * Branches
346 */
347 {0xc4, "br_inst_retired"},
348 {0xc5, "br_miss_pred_retired"},
349 {0xc9, "br_taken_retired"},
350 {0xca, "br_miss_pred_taken_ret"},
351 {0xe0, "br_inst_decoded"},
352 {0xe2, "btb_misses"},
353 {0xe4, "br_bogus"},
354 {0xe6, "baclears"},
355
356 /*
357 * Stalls
358 */
359 {0xa2, "resource_stalls"},
360 {0xd2, "partial_rat_stalls"},
361
362 /*
363 * Segment register loads
364 */
365 {0x6, "segment_reg_loads"},
366
367 /*
368 * Clocks
369 */
370 {0x79, "cpu_clk_unhalted"},
371
372 /*
373 * MMX
374 */
375 {0xb0, "mmx_instr_exec"},
376 {0xb1, "mmx_sat_instr_exec"},
377 {0xb2, "mmx_uops_exec"},
378 {0xb3, "mmx_instr_type_exec"},
379 {0xcc, "fp_mmx_trans"},
380 {0xcd, "mmx_assists"},
381 {0xce, "mmx_instr_ret"},
382 {0xd4, "seg_rename_stalls"},
383 {0xd5, "seg_reg_renames"},
384 {0xd6, "ret_seg_renames"},
385
386 {NT_END, ""}
387 };
388
389 static const struct nametable *P6_names[2] = {
390 _P6_names,
391 _P6_names
392 };
393
394 #define P5_GENERIC_EVENTS \
395 { "PAPI_tot_ins", "instr_exec", 0x0 }, \
396 { "PAPI_tlb_dm", "data_tlb_miss", 0x0 }, \
397 { "PAPI_tlb_im", "code_tlb_miss", 0x0 }, \
398 { "PAPI_fp_ops", "flops" }
399
400 static const ptm_generic_event_t P5mmx_generic_names0[] = {
401 P5_GENERIC_EVENTS,
402 { "PAPI_tot_cyc", "clks_not_HLT", 0x0 },
403 CPC_GEN_END
404 };
405
406 static const ptm_generic_event_t P5mmx_generic_names1[] = {
407 P5_GENERIC_EVENTS,
408 { "PAPI_br_ins", "taken_br", 0x0 },
409 CPC_GEN_END
410 };
411
412 static const ptm_generic_event_t *P5mmx_generic_names[2] = {
413 P5mmx_generic_names0,
414 P5mmx_generic_names1
415 };
416
417 static const ptm_generic_event_t _P6_generic_names[] = {
418 { "PAPI_ca_shr", "l2_ifetch", 0xf },
419 { "PAPI_ca_cln", "bus_tran_rfo", 0x0 },
420 { "PAPI_ca_itv", "bus_tran_inval", 0x0 },
421 { "PAPI_tlb_im", "itlb_miss", 0x0 },
422 { "PAPI_btac_m", "btb_misses", 0x0 },
423 { "PAPI_hw_int", "hw_int_rx", 0x0 },
424 { "PAPI_br_cn", "br_inst_retired", 0x0 },
425 { "PAPI_br_tkn", "br_taken_retired", 0x0 },
426 { "PAPI_br_msp", "br_miss_pred_taken_ret", 0x0 },
427 { "PAPI_br_ins", "br_inst_retired", 0x0 },
428 { "PAPI_res_stl", "resource_stalls", 0x0 },
429 { "PAPI_tot_iis", "inst_decoder", 0x0 },
430 { "PAPI_tot_ins", "inst_retired", 0x0 },
431 { "PAPI_tot_cyc", "cpu_clk_unhalted", 0x0 },
432 { "PAPI_l1_dcm", "dcu_lines_in", 0x0 },
433 { "PAPI_l1_icm", "l2_ifetch", 0xf },
434 { "PAPI_l1_tcm", "l2_rqsts", 0xf },
435 { "PAPI_l1_dca", "data_mem_refs", 0x0 },
436 { "PAPI_l1_stm", "l2_st", 0xf },
437 { "PAPI_l2_icm", "bus_tran_ifetch", 0x0 },
438 { "PAPI_l2_dcr", "l2_ld", 0xf },
439 { "PAPI_l2_dcw", "l2_st", 0xf },
440 { "PAPI_l2_tcm", "l2_lines_in", 0x0 },
441 { "PAPI_l2_tca", "l2_rqsts", 0xf },
442 { "PAPI_l2_tcw", "l2_st", 0xf },
443 { "PAPI_l2_stm", "l2_m_lines_inm", 0x0 },
444 { "PAPI_fp_ins", "flops", 0x0 },
445 { "PAPI_fp_ops", "flops", 0x0 },
446 { "PAPI_fml_ins", "mul", 0x0 },
447 { "PAPI_fdv_ins", "div", 0x0 },
448 CPC_GEN_END
449 };
450
451 static const ptm_generic_event_t *P6_generic_names[2] = {
452 _P6_generic_names,
453 _P6_generic_names
454 };
455
456 static const struct nametable **events;
457 static const ptm_generic_event_t **generic_events;
458
459 #define BITS(v, u, l) \
460 (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))
461
462 /*
463 * "Well known" bit fields in the Pentium CES register
464 * The interfaces in libcpc should make these #defines uninteresting.
465 */
466 #define CPC_P5_CESR_ES0_SHIFT 0
467 #define CPC_P5_CESR_ES0_MASK 0x3f
468 #define CPC_P5_CESR_ES1_SHIFT 16
469 #define CPC_P5_CESR_ES1_MASK 0x3f
470
471 #define CPC_P5_CESR_OS0 6
472 #define CPC_P5_CESR_USR0 7
473 #define CPC_P5_CESR_CLK0 8
474 #define CPC_P5_CESR_PC0 9
475 #define CPC_P5_CESR_OS1 (CPC_P5_CESR_OS0 + 16)
476 #define CPC_P5_CESR_USR1 (CPC_P5_CESR_USR0 + 16)
477 #define CPC_P5_CESR_CLK1 (CPC_P5_CESR_CLK0 + 16)
478 #define CPC_P5_CESR_PC1 (CPC_P5_CESR_PC0 + 16)
479
480 /*
481 * "Well known" bit fields in the Pentium Pro PerfEvtSel registers
482 * The interfaces in libcpc should make these #defines uninteresting.
483 */
484 #define CPC_P6_PES_INV 23
485 #define CPC_P6_PES_EN 22
486 #define CPC_P6_PES_INT 20
487 #define CPC_P6_PES_PC 19
488 #define CPC_P6_PES_E 18
489 #define CPC_P6_PES_OS 17
490 #define CPC_P6_PES_USR 16
491
492 #define CPC_P6_PES_UMASK_SHIFT 8
493 #define CPC_P6_PES_UMASK_MASK (0xffu)
494
495 #define CPC_P6_PES_CMASK_SHIFT 24
496 #define CPC_P6_PES_CMASK_MASK (0xffu)
497
498 #define CPC_P6_PES_PIC0_MASK (0xffu)
499 #define CPC_P6_PES_PIC1_MASK (0xffu)
500
501 #define P6_PES_EN (UINT32_C(1) << CPC_P6_PES_EN)
502 #define P6_PES_INT (UINT32_C(1) << CPC_P6_PES_INT)
503 #define P6_PES_OS (UINT32_C(1) << CPC_P6_PES_OS)
504
505 /*
506 * Pentium 5 attributes
507 */
508 #define P5_NOEDGE 0x1 /* "noedge" - no edge detection */
509 #define P5_PC 0x2 /* "pc" - pin control */
510
511 /*
512 * Pentium 6 attributes
513 */
514 #define P6_NOEDGE 0x1
515 #define P6_PC 0x2
516 #define P6_INV 0x4 /* "inv" - count inverted transitions */
517 #define P6_INT 0x8 /* "int" - interrupt on overflow */
518
519 /*
520 * CPU reference strings
521 */
522
523 #define P5_CPUREF "See Appendix A.4 of the \"IA-32 Intel Architecture " \
524 "Software Developer's Manual Volume 3: System " \
525 "Programming Guide,\" Order # 245472-012, 2003"
526
527 #define P6_CPUREF "See Appendix A.3 of the \"IA-32 Intel Architecture " \
528 "Software Developer's Manual Volume 3: System " \
529 "Programming Guide,\" Order # 245472-012, 2003"
530
531 static int
ptm_pcbe_init(void)532 ptm_pcbe_init(void)
533 {
534 const struct nametable *n;
535 const ptm_generic_event_t *gevp;
536 int i;
537 size_t size;
538
539 if (is_x86_feature(x86_featureset, X86FSET_MMX))
540 ptm_rdpmc_avail = 1;
541
542 /*
543 * Discover type of CPU and set events pointer appropriately.
544 *
545 * Map family and model into the performance
546 * counter architectures we currently understand.
547 *
548 * See application note AP485 (from developer.intel.com)
549 * for further explanation.
550 */
551 if (cpuid_getvendor(CPU) != X86_VENDOR_Intel)
552 return (-1);
553 switch (cpuid_getfamily(CPU)) {
554 case 5: /* Pentium and Pentium with MMX */
555 events = P5mmx_names;
556 generic_events = P5mmx_generic_names;
557 ptm_ver = PTM_VER_P5;
558 ptm_cpuref = P5_CPUREF;
559 if (cpuid_getmodel(CPU) < 4)
560 ptm_impl_name = "Pentium";
561 else
562 ptm_impl_name = "Pentium with MMX";
563 break;
564 case 6: /* Pentium Pro and Pentium II and III */
565 events = P6_names;
566 generic_events = P6_generic_names;
567 ptm_ver = PTM_VER_P6;
568 ptm_cpuref = P6_CPUREF;
569 ptm_pcbe_ops.pcbe_caps = CPC_CAP_OVERFLOW_INTERRUPT;
570 if (is_x86_feature(x86_featureset, X86FSET_MMX))
571 ptm_impl_name = "Pentium Pro with MMX, Pentium II";
572 else
573 ptm_impl_name = "Pentium Pro, Pentium II";
574 break;
575 default:
576 return (-1);
577 }
578
579 /*
580 * Initialize the list of events for each PIC.
581 * Do two passes: one to compute the size necessary and another
582 * to copy the strings. Need room for event, comma, and NULL terminator.
583 */
584 for (i = 0; i < 2; i++) {
585 size = 0;
586 for (n = events[i]; n->bits != NT_END; n++)
587 size += strlen(n->name) + 1;
588 for (gevp = generic_events[i]; gevp->name != NULL; gevp++)
589 size += strlen(gevp->name) + 1;
590 pic_events[i] = kmem_alloc(size + 1, KM_SLEEP);
591 *pic_events[i] = '\0';
592 for (n = events[i]; n->bits != NT_END; n++) {
593 (void) strcat(pic_events[i], n->name);
594 (void) strcat(pic_events[i], ",");
595 }
596 for (gevp = generic_events[i]; gevp->name != NULL; gevp++) {
597 (void) strcat(pic_events[i], gevp->name);
598 (void) strcat(pic_events[i], ",");
599 }
600
601 /*
602 * Remove trailing comma.
603 */
604 pic_events[i][size - 1] = '\0';
605 }
606
607 return (0);
608 }
609
610 static uint_t
ptm_pcbe_ncounters(void)611 ptm_pcbe_ncounters(void)
612 {
613 return (2);
614 }
615
616 static const char *
ptm_pcbe_impl_name(void)617 ptm_pcbe_impl_name(void)
618 {
619 return (ptm_impl_name);
620 }
621
622 static const char *
ptm_pcbe_cpuref(void)623 ptm_pcbe_cpuref(void)
624 {
625 return (ptm_cpuref);
626 }
627
628 static char *
ptm_pcbe_list_events(uint_t picnum)629 ptm_pcbe_list_events(uint_t picnum)
630 {
631 ASSERT(picnum >= 0 && picnum < cpc_ncounters);
632
633 if (pic_events[0] == NULL) {
634 ASSERT(pic_events[1] == NULL);
635 }
636
637 return (pic_events[picnum]);
638 }
639
640 static char *
ptm_pcbe_list_attrs(void)641 ptm_pcbe_list_attrs(void)
642 {
643 if (ptm_ver == PTM_VER_P5)
644 return ("noedge,pc");
645 else
646 return ("noedge,pc,inv,int,umask,cmask");
647 }
648
649 static const ptm_generic_event_t *
find_generic_event(int regno,char * name)650 find_generic_event(int regno, char *name)
651 {
652 const ptm_generic_event_t *gevp;
653
654 for (gevp = generic_events[regno]; gevp->name != NULL; gevp++)
655 if (strcmp(name, gevp->name) == 0)
656 return (gevp);
657
658 return (NULL);
659 }
660
661 static const struct nametable *
find_event(int regno,char * name)662 find_event(int regno, char *name)
663 {
664 const struct nametable *n;
665
666 n = events[regno];
667
668 for (; n->bits != NT_END; n++)
669 if (strcmp(name, n->name) == 0)
670 return (n);
671
672 return (NULL);
673 }
674
675 static uint64_t
ptm_pcbe_event_coverage(char * event)676 ptm_pcbe_event_coverage(char *event)
677 {
678 uint64_t bitmap = 0;
679
680 if ((find_event(0, event) != NULL) ||
681 (find_generic_event(0, event) != NULL))
682 bitmap = 0x1;
683 if ((find_event(1, event) != NULL) ||
684 (find_generic_event(1, event) != NULL))
685 bitmap |= 0x2;
686
687 return (bitmap);
688 }
689
690 static uint64_t
ptm_pcbe_overflow_bitmap(void)691 ptm_pcbe_overflow_bitmap(void)
692 {
693 uint64_t ret = 0;
694 uint64_t pes[2];
695
696 /*
697 * P5 is not capable of generating interrupts.
698 */
699 ASSERT(ptm_ver == PTM_VER_P6);
700
701 /*
702 * CPC could have caused an interrupt provided that
703 *
704 * 1) Counters are enabled
705 * 2) Either counter has requested an interrupt
706 */
707
708 pes[0] = rdmsr(REG_PERFEVNT0);
709 if (((uint32_t)pes[0] & P6_PES_EN) != P6_PES_EN)
710 return (0);
711
712 /*
713 * If a particular counter requested an interrupt, assume it caused
714 * this interrupt. There is no way to determine which counter overflowed
715 * on this hardware other than by using unreliable heuristics.
716 */
717
718 pes[1] = rdmsr(REG_PERFEVNT1);
719 if ((uint32_t)pes[0] & P6_PES_INT)
720 ret |= 0x1;
721 if ((uint32_t)pes[1] & P6_PES_INT)
722 ret |= 0x2;
723
724 return (ret);
725 }
726
727 /*ARGSUSED*/
728 static int
ptm_pcbe_configure(uint_t picnum,char * eventname,uint64_t preset,uint32_t flags,uint_t nattrs,kcpc_attr_t * attrs,void ** data,void * token)729 ptm_pcbe_configure(uint_t picnum, char *eventname, uint64_t preset,
730 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
731 void *token)
732 {
733 ptm_pcbe_config_t *conf;
734 const struct nametable *n;
735 const ptm_generic_event_t *gevp;
736 struct nametable nt_raw = { 0, "raw" };
737 int i;
738 int ptm_flags = 0;
739
740 /*
741 * If we've been handed an existing configuration, we need only preset
742 * the counter value.
743 */
744 if (*data != NULL) {
745 conf = *data;
746 conf->ptm_rawpic = trunc3931(preset);
747 return (0);
748 }
749
750 if (picnum != 0 && picnum != 1)
751 return (CPC_INVALID_PICNUM);
752
753 conf = kmem_alloc(sizeof (ptm_pcbe_config_t), KM_SLEEP);
754
755 conf->ptm_picno = picnum;
756 conf->ptm_rawpic = trunc3931(preset);
757 conf->ptm_ctl = 0;
758
759 if ((n = find_event(picnum, eventname)) == NULL) {
760 if ((gevp = find_generic_event(picnum, eventname)) != NULL) {
761 n = find_event(picnum, gevp->event);
762 ASSERT(n != NULL);
763
764 if (nattrs > 0) {
765 kmem_free(conf, sizeof (ptm_pcbe_config_t));
766 return (CPC_ATTRIBUTE_OUT_OF_RANGE);
767 }
768
769 if (ptm_ver == PTM_VER_P6)
770 conf->ptm_ctl |= gevp->umask <<
771 CPC_P6_PES_UMASK_SHIFT;
772 } else {
773 long tmp;
774
775 /*
776 * If ddi_strtol() likes this event, use it as a raw
777 * event code.
778 */
779 if (ddi_strtol(eventname, NULL, 0, &tmp) != 0) {
780 kmem_free(conf, sizeof (ptm_pcbe_config_t));
781 return (CPC_INVALID_EVENT);
782 }
783
784 nt_raw.bits = tmp;
785
786 if (ptm_ver == PTM_VER_P5)
787 nt_raw.bits &= CPC_P5_CESR_ES0_MASK;
788 else
789 nt_raw.bits &= CPC_P6_PES_PIC0_MASK;
790
791 n = &nt_raw;
792 }
793 }
794
795 if (ptm_ver == PTM_VER_P5) {
796 int picshift;
797 picshift = (picnum == 0) ? 0 : 16;
798
799 for (i = 0; i < nattrs; i++) {
800 /*
801 * Value of these attributes is ignored; their presence
802 * alone tells us to set the corresponding flag.
803 */
804 if (strncmp(attrs[i].ka_name, "noedge", 7) == 0) {
805 if (attrs[i].ka_val != 0)
806 ptm_flags |= P5_NOEDGE;
807 } else if (strncmp(attrs[i].ka_name, "pc", 3) == 0) {
808 if (attrs[i].ka_val != 0)
809 ptm_flags |= P5_PC;
810 } else {
811 kmem_free(conf, sizeof (ptm_pcbe_config_t));
812 return (CPC_INVALID_ATTRIBUTE);
813 }
814 }
815
816 if (flags & CPC_COUNT_USER)
817 conf->ptm_ctl |= (1 << (CPC_P5_CESR_USR0 + picshift));
818 if (flags & CPC_COUNT_SYSTEM)
819 conf->ptm_ctl |= (1 << (CPC_P5_CESR_OS0 + picshift));
820 if (ptm_flags & P5_NOEDGE)
821 conf->ptm_ctl |= (1 << (CPC_P5_CESR_CLK0 + picshift));
822 if (ptm_flags & P5_PC)
823 conf->ptm_ctl |= (1 << (CPC_P5_CESR_PC0 + picshift));
824
825 ASSERT((n->bits | CPC_P5_CESR_ES0_MASK) ==
826 CPC_P5_CESR_ES0_MASK);
827
828 conf->ptm_ctl |= (n->bits << picshift);
829 } else {
830 for (i = 0; i < nattrs; i++) {
831 if (strncmp(attrs[i].ka_name, "noedge", 6) == 0) {
832 if (attrs[i].ka_val != 0)
833 ptm_flags |= P6_NOEDGE;
834 } else if (strncmp(attrs[i].ka_name, "pc", 2) == 0) {
835 if (attrs[i].ka_val != 0)
836 ptm_flags |= P6_PC;
837 } else if (strncmp(attrs[i].ka_name, "inv", 3) == 0) {
838 if (attrs[i].ka_val != 0)
839 ptm_flags |= P6_INV;
840 } else if (strncmp(attrs[i].ka_name, "umask", 5) == 0) {
841 if ((attrs[i].ka_val | CPC_P6_PES_UMASK_MASK) !=
842 CPC_P6_PES_UMASK_MASK) {
843 kmem_free(conf,
844 sizeof (ptm_pcbe_config_t));
845 return (CPC_ATTRIBUTE_OUT_OF_RANGE);
846 }
847 conf->ptm_ctl |= (uint8_t)attrs[i].ka_val <<
848 CPC_P6_PES_UMASK_SHIFT;
849 } else if (strncmp(attrs[i].ka_name, "cmask", 5) == 0) {
850 if ((attrs[i].ka_val | CPC_P6_PES_CMASK_MASK) !=
851 CPC_P6_PES_CMASK_MASK) {
852 kmem_free(conf,
853 sizeof (ptm_pcbe_config_t));
854 return (CPC_ATTRIBUTE_OUT_OF_RANGE);
855 }
856 conf->ptm_ctl |= (uint8_t)attrs[i].ka_val <<
857 CPC_P6_PES_CMASK_SHIFT;
858 } else if (strncmp(attrs[i].ka_name, "int", 3) == 0) {
859 if (attrs[i].ka_val != 0)
860 ptm_flags |= P6_INT;
861 } else {
862 kmem_free(conf, sizeof (ptm_pcbe_config_t));
863 return (CPC_INVALID_ATTRIBUTE);
864 }
865 }
866
867 if (flags & CPC_OVF_NOTIFY_EMT)
868 /*
869 * If the user has requested notification of overflows,
870 * we automatically program the hardware to generate
871 * overflow interrupts.
872 */
873 ptm_flags |= P6_INT;
874 if (flags & CPC_COUNT_USER)
875 conf->ptm_ctl |= (1 << CPC_P6_PES_USR);
876 if (flags & CPC_COUNT_SYSTEM)
877 conf->ptm_ctl |= (1 << CPC_P6_PES_OS);
878 if ((ptm_flags & P6_NOEDGE) == 0)
879 conf->ptm_ctl |= (1 << CPC_P6_PES_E);
880 if (ptm_flags & P6_PC)
881 conf->ptm_ctl |= (1 << CPC_P6_PES_PC);
882 if (ptm_flags & P6_INV)
883 conf->ptm_ctl |= (1 << CPC_P6_PES_INV);
884 if (ptm_flags & P6_INT)
885 conf->ptm_ctl |= (1 << CPC_P6_PES_INT);
886
887 ASSERT((n->bits | CPC_P6_PES_PIC0_MASK) ==
888 CPC_P6_PES_PIC0_MASK);
889
890 conf->ptm_ctl |= n->bits;
891 }
892
893 *data = conf;
894 return (0);
895 }
896
897 static void
ptm_pcbe_program(void * token)898 ptm_pcbe_program(void *token)
899 {
900 ptm_pcbe_config_t *pic0;
901 ptm_pcbe_config_t *pic1;
902 ptm_pcbe_config_t *tmp;
903 ptm_pcbe_config_t empty = { 1, 0, 0 }; /* assume pic1 to start */
904
905 if ((pic0 = kcpc_next_config(token, NULL, NULL)) == NULL)
906 panic("ptm_pcbe: token %p has no configs", token);
907
908 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL)
909 pic1 = ∅
910
911 if (pic0->ptm_picno != 0) {
912 empty.ptm_picno = 0;
913 tmp = pic1;
914 pic1 = pic0;
915 pic0 = tmp;
916 }
917
918 ASSERT(pic0->ptm_picno == 0 && pic1->ptm_picno == 1);
919
920 if (ptm_rdpmc_avail) {
921 ulong_t curcr4 = getcr4();
922 if (kcpc_allow_nonpriv(token))
923 setcr4(curcr4 | CR4_PCE);
924 else
925 setcr4(curcr4 & ~CR4_PCE);
926 }
927
928 if (ptm_ver == PTM_VER_P5) {
929 wrmsr(P5_CESR, ALL_STOPPED);
930 wrmsr(P5_CTR0, pic0->ptm_rawpic);
931 wrmsr(P5_CTR1, pic1->ptm_rawpic);
932 wrmsr(P5_CESR, pic0->ptm_ctl | pic1->ptm_ctl);
933 pic0->ptm_rawpic = rdmsr(P5_CTR0);
934 pic1->ptm_rawpic = rdmsr(P5_CTR1);
935 } else {
936 uint64_t pes;
937 wrmsr(REG_PERFEVNT0, ALL_STOPPED);
938 wrmsr(REG_PERFCTR0, pic0->ptm_rawpic);
939 wrmsr(REG_PERFCTR1, pic1->ptm_rawpic);
940 pes = pic1->ptm_ctl;
941 DTRACE_PROBE1(ptm__pes1, uint64_t, pes);
942 wrmsr(REG_PERFEVNT1, pes);
943 pes = pic0->ptm_ctl | (1 << CPC_P6_PES_EN);
944 DTRACE_PROBE1(ptm__pes0, uint64_t, pes);
945 wrmsr(REG_PERFEVNT0, pes);
946 }
947 }
948
949 static void
ptm_pcbe_allstop(void)950 ptm_pcbe_allstop(void)
951 {
952 if (ptm_ver == PTM_VER_P5)
953 wrmsr(P5_CESR, ALL_STOPPED);
954 else {
955 wrmsr(REG_PERFEVNT0, ALL_STOPPED);
956 setcr4(getcr4() & ~CR4_PCE);
957 }
958 }
959
960 static void
ptm_pcbe_sample(void * token)961 ptm_pcbe_sample(void *token)
962 {
963 ptm_pcbe_config_t *pic0;
964 ptm_pcbe_config_t *pic1;
965 ptm_pcbe_config_t *swap;
966 ptm_pcbe_config_t empty = { 1, 0, 0 }; /* assume pic1 to start */
967 uint64_t tmp;
968 uint64_t *pic0_data;
969 uint64_t *pic1_data;
970 uint64_t *dtmp;
971 uint64_t curpic[2];
972
973 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
974 panic("ptm_pcbe: token %p has no configs", token);
975
976 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
977 pic1 = ∅
978 pic1_data = &tmp;
979 }
980
981 if (pic0->ptm_picno != 0) {
982 empty.ptm_picno = 0;
983 swap = pic0;
984 pic0 = pic1;
985 pic1 = swap;
986 dtmp = pic0_data;
987 pic0_data = pic1_data;
988 pic1_data = dtmp;
989 }
990
991 ASSERT(pic0->ptm_picno == 0 && pic1->ptm_picno == 1);
992
993 if (ptm_ver == PTM_VER_P5) {
994 curpic[0] = rdmsr(P5_CTR0);
995 curpic[1] = rdmsr(P5_CTR1);
996 } else {
997 curpic[0] = rdmsr(REG_PERFCTR0);
998 curpic[1] = rdmsr(REG_PERFCTR1);
999 }
1000
1001 DTRACE_PROBE1(ptm__curpic0, uint64_t, curpic[0]);
1002 DTRACE_PROBE1(ptm__curpic1, uint64_t, curpic[1]);
1003
1004 *pic0_data += diff3931(curpic[0], pic0->ptm_rawpic);
1005 pic0->ptm_rawpic = trunc3931(*pic0_data);
1006
1007 *pic1_data += diff3931(curpic[1], pic1->ptm_rawpic);
1008 pic1->ptm_rawpic = trunc3931(*pic1_data);
1009 }
1010
1011 static void
ptm_pcbe_free(void * config)1012 ptm_pcbe_free(void *config)
1013 {
1014 kmem_free(config, sizeof (ptm_pcbe_config_t));
1015 }
1016
1017 /*
1018 * Virtualizes the 40-bit field of the %pic
1019 * register into a 64-bit software register.
1020 *
1021 * We can retrieve 40 (signed) bits from the counters,
1022 * but we can set only 32 (signed) bits into the counters.
1023 * This makes virtualizing more than 31-bits of registers
1024 * quite tricky.
1025 *
1026 * If bits 39 to 31 are set in the virtualized pic register,
1027 * then we can preset the counter to this value using the fact
1028 * that wrmsr sign extends bit 31. Though it might look easier
1029 * to only use the bottom 31-bits of the register, we have to allow
1030 * the full 40-bits to be used to perform overflow profiling.
1031 */
1032
1033 #define MASK40 UINT64_C(0xffffffffff)
1034 #define MASK31 UINT64_C(0x7fffffff)
1035 #define BITS_39_31 UINT64_C(0xff80000000)
1036
1037 static int64_t
diff3931(uint64_t sample,uint64_t old)1038 diff3931(uint64_t sample, uint64_t old)
1039 {
1040 int64_t diff;
1041
1042 if ((old & BITS_39_31) == BITS_39_31) {
1043 diff = (MASK40 & sample) - old;
1044 if (diff < 0)
1045 diff += (UINT64_C(1) << 40);
1046 } else {
1047 diff = (MASK31 & sample) - old;
1048 if (diff < 0)
1049 diff += (UINT64_C(1) << 31);
1050 }
1051 return (diff);
1052 }
1053
1054 static uint64_t
trunc3931(uint64_t value)1055 trunc3931(uint64_t value)
1056 {
1057 if ((value & BITS_39_31) == BITS_39_31)
1058 return (MASK40 & value);
1059 return (MASK31 & value);
1060 }
1061
1062 static struct modlpcbe modlpcbe = {
1063 &mod_pcbeops,
1064 "Pentium Performance Counters",
1065 &ptm_pcbe_ops
1066 };
1067
1068 static struct modlinkage modl = {
1069 MODREV_1,
1070 &modlpcbe,
1071 };
1072
1073 int
_init(void)1074 _init(void)
1075 {
1076 if (ptm_pcbe_init() != 0)
1077 return (ENOTSUP);
1078 return (mod_install(&modl));
1079 }
1080
1081 int
_fini(void)1082 _fini(void)
1083 {
1084 return (mod_remove(&modl));
1085 }
1086
1087 int
_info(struct modinfo * mi)1088 _info(struct modinfo *mi)
1089 {
1090 return (mod_info(&modl, mi));
1091 }
1092