xref: /titanic_41/usr/src/uts/sun4u/pcbe/us234_pcbe.c (revision bdfc6d18da790deeec2e0eb09c625902defe2498)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * UltraSPARC Performance Counter Backend
29  */
30 
31 #pragma ident	"%Z%%M%	%I%	%E% SMI"
32 
33 #include <sys/cpuvar.h>
34 #include <sys/systm.h>
35 #include <sys/cmn_err.h>
36 #include <sys/spitregs.h>
37 #include <sys/cheetahregs.h>
38 #include <sys/cpc_impl.h>
39 #include <sys/cpc_pcbe.h>
40 #include <sys/modctl.h>
41 #include <sys/machsystm.h>
42 #include <sys/sdt.h>
43 
44 static int us_pcbe_init(void);
45 static uint_t us_pcbe_ncounters(void);
46 static const char *us_pcbe_impl_name(void);
47 static const char *us_pcbe_cpuref(void);
48 static char *us_pcbe_list_events(uint_t picnum);
49 static char *us_pcbe_list_attrs(void);
50 static uint64_t us_pcbe_event_coverage(char *event);
51 static uint64_t us_pcbe_overflow_bitmap(void);
52 static int us_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
53     uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
54     void *token);
55 static void us_pcbe_program(void *token);
56 static void us_pcbe_allstop(void);
57 static void us_pcbe_sample(void *token);
58 static void us_pcbe_free(void *config);
59 
60 extern void ultra_setpcr(uint64_t);
61 extern uint64_t ultra_getpcr(void);
62 extern void ultra_setpic(uint64_t);
63 extern uint64_t ultra_getpic(void);
64 extern uint64_t ultra_gettick(void);
65 
66 pcbe_ops_t us_pcbe_ops = {
67 	PCBE_VER_1,
68 	CPC_CAP_OVERFLOW_INTERRUPT,
69 	us_pcbe_ncounters,
70 	us_pcbe_impl_name,
71 	us_pcbe_cpuref,
72 	us_pcbe_list_events,
73 	us_pcbe_list_attrs,
74 	us_pcbe_event_coverage,
75 	us_pcbe_overflow_bitmap,
76 	us_pcbe_configure,
77 	us_pcbe_program,
78 	us_pcbe_allstop,
79 	us_pcbe_sample,
80 	us_pcbe_free
81 };
82 
83 typedef struct _us_pcbe_config {
84 	uint8_t		us_picno;	/* 0 for pic0 or 1 for pic1 */
85 	uint32_t	us_bits;	/* %pcr event code unshifted */
86 	uint32_t	us_flags;	/* user/system/priv */
87 	uint32_t	us_pic;		/* unshifted raw %pic value */
88 } us_pcbe_config_t;
89 
90 struct nametable {
91 	const uint8_t	bits;
92 	const char	*name;
93 };
94 
95 #define	PIC0_MASK (((uint64_t)1 << 32) - 1)
96 
97 #define	ULTRA_PCR_SYS		(UINT64_C(1) << CPC_ULTRA_PCR_SYS)
98 #define	ULTRA_PCR_PRIVPIC	(UINT64_C(1) << CPC_ULTRA_PCR_PRIVPIC)
99 
100 #define	CPC_ULTRA_PCR_USR		2
101 #define	CPC_ULTRA_PCR_SYS		1
102 #define	CPC_ULTRA_PCR_PRIVPIC		0
103 
104 #define	CPC_ULTRA_PCR_PIC0_SHIFT	4
105 #define	CPC_ULTRA2_PCR_PIC_MASK		UINT64_C(0xf)
106 #define	CPC_ULTRA3_PCR_PIC_MASK		UINT64_C(0x3f)
107 #define	CPC_ULTRA_PCR_PIC1_SHIFT	11
108 
109 #define	NT_END 0xFF
110 
111 static const uint64_t   allstopped = ULTRA_PCR_PRIVPIC;
112 
113 #define	USall_EVENTS_0						\
114 	{0x0,	"Cycle_cnt"},					\
115 	{0x1,	"Instr_cnt"},					\
116 	{0x2,	"Dispatch0_IC_miss"},				\
117 	{0x8,	"IC_ref"},					\
118 	{0x9,	"DC_rd"},					\
119 	{0xa,	"DC_wr"},					\
120 	{0xc,	"EC_ref"},					\
121 	{0xe,	"EC_snoop_inv"}
122 
123 static const struct nametable US12_names0[] = {
124 	USall_EVENTS_0,
125 	{0x3,	"Dispatch0_storeBuf"},
126 	{0xb,	"Load_use"},
127 	{0xd,	"EC_write_hit_RDO"},
128 	{0xf,	"EC_rd_hit"},
129 	{NT_END, ""}
130 };
131 
132 #define	US3all_EVENTS_0						\
133 	{0x3,	"Dispatch0_br_target"},				\
134 	{0x4,	"Dispatch0_2nd_br"},				\
135 	{0x5,	"Rstall_storeQ"},				\
136 	{0x6,	"Rstall_IU_use"},				\
137 	{0xd,	"EC_write_hit_RTO"},				\
138 	{0xf,	"EC_rd_miss"},					\
139 	{0x10,	"PC_port0_rd"},					\
140 	{0x11,	"SI_snoop"},					\
141 	{0x12,	"SI_ciq_flow"},					\
142 	{0x13,	"SI_owned"},					\
143 	{0x14,	"SW_count_0"},					\
144 	{0x15,	"IU_Stat_Br_miss_taken"},			\
145 	{0x16,	"IU_Stat_Br_count_taken"},			\
146 	{0x17,	"Dispatch_rs_mispred"},				\
147 	{0x18,	"FA_pipe_completion"}
148 
149 #define	US3_MC_EVENTS_0						\
150 	{0x20,	"MC_reads_0"},					\
151 	{0x21,	"MC_reads_1"},					\
152 	{0x22,	"MC_reads_2"},					\
153 	{0x23,	"MC_reads_3"},					\
154 	{0x24,	"MC_stalls_0"},					\
155 	{0x25,	"MC_stalls_2"}
156 
157 #define	US3_I_MC_EVENTS_0					\
158 	{0x20,	"MC_read_dispatched"},				\
159 	{0x21,	"MC_write_dispatched"},				\
160 	{0x22,	"MC_read_returned_to_JBU"},			\
161 	{0x23,	"MC_msl_busy_stall"},				\
162 	{0x24,	"MC_mdb_overflow_stall"},			\
163 	{0x25,	"MC_miu_spec_request"}
164 
165 #define	USall_EVENTS_1						\
166 	{0x0,	"Cycle_cnt"},					\
167 	{0x1,	"Instr_cnt"},					\
168 	{0x2,	"Dispatch0_mispred"},				\
169 	{0xd,	"EC_wb"},					\
170 	{0xe,	"EC_snoop_cb"}
171 
172 static const struct nametable US3_names0[] = {
173 	USall_EVENTS_0,
174 	US3all_EVENTS_0,
175 	US3_MC_EVENTS_0,
176 	{NT_END, ""}
177 };
178 
179 static const struct nametable US3_PLUS_names0[] = {
180 	USall_EVENTS_0,
181 	US3all_EVENTS_0,
182 	US3_MC_EVENTS_0,
183 	{0x19,	"EC_wb_remote"},
184 	{0x1a,	"EC_miss_local"},
185 	{0x1b,	"EC_miss_mtag_remote"},
186 	{NT_END, ""}
187 };
188 
189 static const struct nametable US3_I_names0[] = {
190 	USall_EVENTS_0,
191 	US3all_EVENTS_0,
192 	US3_I_MC_EVENTS_0,
193 	{NT_END, ""}
194 };
195 
196 static const struct nametable US4_PLUS_names0[] = {
197 	{0x0,   "Cycle_cnt"},
198 	{0x1,   "Instr_cnt"},
199 	{0x2,   "Dispatch0_IC_miss"},
200 	{0x3,   "IU_stat_jmp_correct_pred"},
201 	{0x4,   "Dispatch0_2nd_br"},
202 	{0x5,   "Rstall_storeQ"},
203 	{0x6,   "Rstall_IU_use"},
204 	{0x7,   "IU_stat_ret_correct_pred"},
205 	{0x8,   "IC_ref"},
206 	{0x9,   "DC_rd"},
207 	{0xa,   "Rstall_FP_use"},
208 	{0xb,   "SW_pf_instr"},
209 	{0xc,   "L2_ref"},
210 	{0xd,   "L2_write_hit_RTO"},
211 	{0xe,   "L2_snoop_inv_sh"},
212 	{0xf,   "L2_rd_miss"},
213 	{0x10,  "PC_rd"},
214 	{0x11,  "SI_snoop_sh"},
215 	{0x12,  "SI_ciq_flow_sh"},
216 	{0x13,  "Re_DC_miss"},
217 	{0x14,  "SW_count_NOP"},
218 	{0x15,  "IU_stat_br_miss_taken"},
219 	{0x16,  "IU_stat_br_count_untaken"},
220 	{0x17,  "HW_pf_exec"},
221 	{0x18,  "FA_pipe_completion"},
222 	{0x19,  "SSM_L3_wb_remote"},
223 	{0x1a,  "SSM_L3_miss_local"},
224 	{0x1b,  "SSM_L3_miss_mtag_remote"},
225 	{0x1c,  "SW_pf_str_trapped"},
226 	{0x1d,  "SW_pf_PC_installed"},
227 	{0x1e,  "IPB_to_IC_fill"},
228 	{0x1f,  "L2_write_miss"},
229 	{0x20,  "MC_reads_0_sh"},
230 	{0x21,  "MC_reads_1_sh"},
231 	{0x22,  "MC_reads_2_sh"},
232 	{0x23,  "MC_reads_3_sh"},
233 	{0x24,  "MC_stalls_0_sh"},
234 	{0x25,  "MC_stalls_2_sh"},
235 	{0x26,  "L2_hit_other_half"},
236 	{0x28,  "L3_rd_miss"},
237 	{0x29,  "Re_L2_miss"},
238 	{0x2a,  "IC_miss_cancelled"},
239 	{0x2b,  "DC_wr_miss"},
240 	{0x2c,  "L3_hit_I_state_sh"},
241 	{0x2d,  "SI_RTS_src_data"},
242 	{0x2e,  "L2_IC_miss"},
243 	{0x2f,  "SSM_new_transaction_sh"},
244 	{0x30,  "L2_SW_pf_miss"},
245 	{0x31,  "L2_wb"},
246 	{0x32,  "L2_wb_sh"},
247 	{0x33,  "L2_snoop_cb_sh"},
248 	{NT_END, ""}
249 };
250 
251 
252 #define	US3all_EVENTS_1				\
253 	{0x3,	"IC_miss_cancelled"},		\
254 	{0x5,	"Re_FPU_bypass"},		\
255 	{0x6,	"Re_DC_miss"},			\
256 	{0x7,	"Re_EC_miss"},			\
257 	{0x8,	"IC_miss"},			\
258 	{0x9,	"DC_rd_miss"},			\
259 	{0xa,	"DC_wr_miss"},			\
260 	{0xb,	"Rstall_FP_use"},		\
261 	{0xc,	"EC_misses"},			\
262 	{0xf,	"EC_ic_miss"},			\
263 	{0x10,	"Re_PC_miss"},			\
264 	{0x11,	"ITLB_miss"},			\
265 	{0x12,	"DTLB_miss"},			\
266 	{0x13,	"WC_miss"},			\
267 	{0x14,	"WC_snoop_cb"},			\
268 	{0x15,	"WC_scrubbed"},			\
269 	{0x16,	"WC_wb_wo_read"},		\
270 	{0x18,	"PC_soft_hit"},			\
271 	{0x19,	"PC_snoop_inv"},		\
272 	{0x1a,	"PC_hard_hit"},			\
273 	{0x1b,	"PC_port1_rd"},			\
274 	{0x1c,	"SW_count_1"},			\
275 	{0x1d,	"IU_Stat_Br_miss_untaken"},	\
276 	{0x1e,	"IU_Stat_Br_count_untaken"},	\
277 	{0x1f,	"PC_MS_misses"},		\
278 	{0x26,	"Re_RAW_miss"},			\
279 	{0x27,	"FM_pipe_completion"}
280 
281 #define	US3_MC_EVENTS_1				\
282 	{0x20,	"MC_writes_0"},			\
283 	{0x21,	"MC_writes_1"},			\
284 	{0x22,	"MC_writes_2"},			\
285 	{0x23,	"MC_writes_3"},			\
286 	{0x24,	"MC_stalls_1"},			\
287 	{0x25,	"MC_stalls_3"}
288 
289 #define	US3_I_MC_EVENTS_1			\
290 	{0x20,	"MC_open_bank_cmds"},		\
291 	{0x21,	"MC_reads"},			\
292 	{0x22,	"MC_writes"},			\
293 	{0x23,	"MC_page_close_stall"}
294 
295 static const struct nametable US3_names1[] = {
296 	USall_EVENTS_1,
297 	US3all_EVENTS_1,
298 	US3_MC_EVENTS_1,
299 	{0x4,	"Re_endian_miss"},
300 	{NT_END, ""}
301 };
302 
303 static const struct nametable US3_PLUS_names1[] = {
304 	USall_EVENTS_1,
305 	US3all_EVENTS_1,
306 	US3_MC_EVENTS_1,
307 	{0x4,	"Re_DC_missovhd"},
308 	{0x28,	"EC_miss_mtag_remote"},
309 	{0x29,	"EC_miss_remote"},
310 	{NT_END, ""}
311 };
312 
313 static const struct nametable US3_I_names1[] = {
314 	USall_EVENTS_1,
315 	US3all_EVENTS_1,
316 	US3_I_MC_EVENTS_1,
317 	{0x4,	"Re_DC_missovhd"},
318 	{NT_END, ""}
319 };
320 
321 static const struct nametable US4_PLUS_names1[] = {
322 	{0x0,   "Cycle_cnt"},
323 	{0x1,   "Instr_cnt"},
324 	{0x2,   "Dispatch0_other"},
325 	{0x3,   "DC_wr"},
326 	{0x4,   "Re_DC_missovhd"},
327 	{0x5,   "Re_FPU_bypass"},
328 	{0x6,   "L3_write_hit_RTO"},
329 	{0x7,   "L2L3_snoop_inv_sh"},
330 	{0x8,   "IC_L2_req"},
331 	{0x9,   "DC_rd_miss"},
332 	{0xa,   "L2_hit_I_state_sh"},
333 	{0xb,   "L3_write_miss_RTO"},
334 	{0xc,   "L2_miss"},
335 	{0xd,   "SI_owned_sh"},
336 	{0xe,   "SI_RTO_src_data"},
337 	{0xf,   "SW_pf_duplicate"},
338 	{0x10,  "IU_stat_jmp_mispred"},
339 	{0x11,  "ITLB_miss"},
340 	{0x12,  "DTLB_miss"},
341 	{0x13,  "WC_miss"},
342 	{0x14,  "IC_fill"},
343 	{0x15,  "IU_stat_ret_mispred"},
344 	{0x16,  "Re_L3_miss"},
345 	{0x17,  "Re_PFQ_full"},
346 	{0x18,  "PC_soft_hit"},
347 	{0x19,  "PC_inv"},
348 	{0x1a,  "PC_hard_hit"},
349 	{0x1b,  "IC_pf"},
350 	{0x1c,  "SW_count_NOP"},
351 	{0x1d,  "IU_stat_br_miss_untaken"},
352 	{0x1e,  "IU_stat_br_count_taken"},
353 	{0x1f,  "PC_miss"},
354 	{0x20,  "MC_writes_0_sh"},
355 	{0x21,  "MC_writes_1_sh"},
356 	{0x22,  "MC_writes_2_sh"},
357 	{0x23,  "MC_writes_3_sh"},
358 	{0x24,  "MC_stalls_1_sh"},
359 	{0x25,  "MC_stalls_3_sh"},
360 	{0x26,  "Re_RAW_miss"},
361 	{0x27,  "FM_pipe_completion"},
362 	{0x28,  "SSM_L3_miss_mtag_remote"},
363 	{0x29,  "SSM_L3_miss_remote"},
364 	{0x2a,  "SW_pf_exec"},
365 	{0x2b,  "SW_pf_str_exec"},
366 	{0x2c,  "SW_pf_dropped"},
367 	{0x2d,  "SW_pf_L2_installed"},
368 	{0x2f,  "L2_HW_pf_miss"},
369 	{0x31,  "L3_miss"},
370 	{0x32,  "L3_IC_miss"},
371 	{0x33,  "L3_SW_pf_miss"},
372 	{0x34,  "L3_hit_other_half"},
373 	{0x35,  "L3_wb"},
374 	{0x36,  "L3_wb_sh"},
375 	{0x37,  "L2L3_snoop_cb_sh"},
376 	{NT_END, ""}
377 };
378 
379 static const struct nametable US12_names1[] = {
380 	USall_EVENTS_1,
381 	{0x3,	"Dispatch0_FP_use"},
382 	{0x8,	"IC_hit"},
383 	{0x9,	"DC_rd_hit"},
384 	{0xa,	"DC_wr_hit"},
385 	{0xb,	"Load_use_RAW"},
386 	{0xc,	"EC_hit"},
387 	{0xf,	"EC_ic_hit"},
388 	{NT_END, ""}
389 };
390 
391 static const struct nametable *US12_names[2] = {
392 	US12_names0,
393 	US12_names1
394 };
395 
396 static const struct nametable *US3_names[2] = {
397 	US3_names0,
398 	US3_names1
399 };
400 
401 static const struct nametable *US3_PLUS_names[2] = {
402 	US3_PLUS_names0,
403 	US3_PLUS_names1
404 };
405 
406 static const struct nametable *US4_PLUS_names[2] = {
407 	US4_PLUS_names0,
408 	US4_PLUS_names1
409 };
410 
411 static const struct nametable *US3_I_names[2] = {
412 	US3_I_names0,
413 	US3_I_names1
414 };
415 
416 static const struct nametable **events;
417 static const char *us_impl_name;
418 static const char *us_cpuref;
419 static char *pic_events[2];
420 static uint16_t pcr_pic_mask;
421 
422 #define	CPU_REF_URL " Documentation for Sun processors can be found at: " \
423 			"http://www.sun.com/processors/manuals"
424 
425 static const char *us_2_ref = "See the \"UltraSPARC I/II User\'s Manual\" "
426 			"(Part No. 802-7220-02) "
427 			"for descriptions of these events." CPU_REF_URL;
428 
429 static const char *us_3cu_ref = "See the \"UltraSPARC III Cu User's Manual\" "
430 			"for descriptions of these events." CPU_REF_URL;
431 
432 static const char *us4_plus_ref = "See the \"UltraSPARC IV+ User's Manual\" "
433 			"for descriptions of these events." CPU_REF_URL;
434 
435 static const char *us_3i_ref = "See the \"UltraSPARC IIIi User's Manual\"  "
436 			"for descriptions of these events." CPU_REF_URL;
437 
438 static int
439 us_pcbe_init(void)
440 {
441 	const struct nametable	*n;
442 	int			i;
443 	size_t			size;
444 
445 	/*
446 	 * Discover type of CPU
447 	 *
448 	 * Point nametable to that CPU's table
449 	 */
450 	switch (ULTRA_VER_IMPL(ultra_getver())) {
451 	case SPITFIRE_IMPL:
452 	case BLACKBIRD_IMPL:
453 	case SABRE_IMPL:
454 	case HUMMBRD_IMPL:
455 		events = US12_names;
456 		us_impl_name = "UltraSPARC I&II";
457 		us_cpuref = us_2_ref;
458 		pcr_pic_mask = CPC_ULTRA2_PCR_PIC_MASK;
459 		us_pcbe_ops.pcbe_caps &= ~CPC_CAP_OVERFLOW_INTERRUPT;
460 		break;
461 	case CHEETAH_IMPL:
462 		events = US3_names;
463 		us_impl_name = "UltraSPARC III";
464 		us_cpuref = us_3cu_ref;
465 		pcr_pic_mask = CPC_ULTRA3_PCR_PIC_MASK;
466 		break;
467 	case CHEETAH_PLUS_IMPL:
468 	case JAGUAR_IMPL:
469 		events = US3_PLUS_names;
470 		us_impl_name = "UltraSPARC III+ & IV";
471 		us_cpuref = us_3cu_ref;
472 		pcr_pic_mask = CPC_ULTRA3_PCR_PIC_MASK;
473 		break;
474 	case PANTHER_IMPL:
475 		events = US4_PLUS_names;
476 		us_impl_name = "UltraSPARC IV+";
477 		us_cpuref = us4_plus_ref;
478 		pcr_pic_mask = CPC_ULTRA3_PCR_PIC_MASK;
479 		break;
480 	case JALAPENO_IMPL:
481 	case SERRANO_IMPL:
482 		events = US3_I_names;
483 		us_impl_name = "UltraSPARC IIIi & IIIi+";
484 		us_cpuref = us_3i_ref;
485 		pcr_pic_mask = CPC_ULTRA3_PCR_PIC_MASK;
486 		break;
487 	default:
488 		return (-1);
489 	}
490 
491 	/*
492 	 * Initialize the list of events for each PIC.
493 	 * Do two passes: one to compute the size necessary and another
494 	 * to copy the strings. Need room for event, comma, and NULL terminator.
495 	 */
496 	for (i = 0; i < 2; i++) {
497 		size = 0;
498 		for (n = events[i]; n->bits != NT_END; n++)
499 			size += strlen(n->name) + 1;
500 		pic_events[i] = kmem_alloc(size + 1, KM_SLEEP);
501 		*pic_events[i] = '\0';
502 		for (n = events[i]; n->bits != NT_END; n++) {
503 			(void) strcat(pic_events[i], n->name);
504 			(void) strcat(pic_events[i], ",");
505 		}
506 		/*
507 		 * Remove trailing comma.
508 		 */
509 		pic_events[i][size - 1] = '\0';
510 	}
511 
512 	return (0);
513 }
514 
515 static uint_t
516 us_pcbe_ncounters(void)
517 {
518 	return (2);
519 }
520 
521 static const char *
522 us_pcbe_impl_name(void)
523 {
524 	return (us_impl_name);
525 }
526 
527 static const char *
528 us_pcbe_cpuref(void)
529 {
530 	return (us_cpuref);
531 }
532 
533 static char *
534 us_pcbe_list_events(uint_t picnum)
535 {
536 	ASSERT(picnum >= 0 && picnum < cpc_ncounters);
537 
538 	return (pic_events[picnum]);
539 }
540 
541 static char *
542 us_pcbe_list_attrs(void)
543 {
544 	return ("");
545 }
546 
547 static const struct nametable *
548 find_event(int regno, char *name)
549 {
550 	const struct nametable *n;
551 
552 	n = events[regno];
553 
554 	for (; n->bits != NT_END; n++)
555 		if (strcmp(name, n->name) == 0)
556 			return (n);
557 
558 	return (NULL);
559 }
560 
561 static uint64_t
562 us_pcbe_event_coverage(char *event)
563 {
564 	uint64_t bitmap = 0;
565 
566 	if (find_event(0, event) != NULL)
567 		bitmap = 0x1;
568 	if (find_event(1, event) != NULL)
569 		bitmap |= 0x2;
570 
571 	return (bitmap);
572 }
573 
574 /*
575  * These processors cannot tell which counter overflowed. The PCBE interface
576  * requires such processors to act as if _all_ counters had overflowed.
577  */
578 static uint64_t
579 us_pcbe_overflow_bitmap(void)
580 {
581 	return (0x3);
582 }
583 
584 /*ARGSUSED*/
585 static int
586 us_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
587     uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
588 {
589 	us_pcbe_config_t *conf;
590 	const struct nametable *n;
591 	us_pcbe_config_t *other_config;
592 
593 	/*
594 	 * If we've been handed an existing configuration, we need only preset
595 	 * the counter value.
596 	 */
597 	if (*data != NULL) {
598 		conf = *data;
599 		conf->us_pic = (uint32_t)preset;
600 		return (0);
601 	}
602 
603 	if (picnum < 0 || picnum > 1)
604 		return (CPC_INVALID_PICNUM);
605 
606 	if (nattrs != 0)
607 		return (CPC_INVALID_ATTRIBUTE);
608 
609 	/*
610 	 * Find other requests that will be programmed with this one, and ensure
611 	 * the flags don't conflict.
612 	 */
613 	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
614 	    (other_config->us_flags != flags))
615 		return (CPC_CONFLICTING_REQS);
616 
617 	if ((n = find_event(picnum, event)) == NULL)
618 		return (CPC_INVALID_EVENT);
619 
620 	conf = kmem_alloc(sizeof (us_pcbe_config_t), KM_SLEEP);
621 
622 	conf->us_picno = picnum;
623 	conf->us_bits = (uint32_t)n->bits;
624 	conf->us_flags = flags;
625 	conf->us_pic = (uint32_t)preset;
626 
627 	*data = conf;
628 	return (0);
629 }
630 
631 static void
632 us_pcbe_program(void *token)
633 {
634 	us_pcbe_config_t	*pic0;
635 	us_pcbe_config_t	*pic1;
636 	us_pcbe_config_t	*tmp;
637 	us_pcbe_config_t	empty = { 1, 0x1c, 0, 0 }; /* SW_count_1 */
638 	uint64_t		pcr;
639 	uint64_t		curpic;
640 
641 	if ((pic0 = (us_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) ==
642 	    NULL)
643 		panic("us_pcbe: token %p has no configs", token);
644 
645 	if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) {
646 		pic1 = &empty;
647 		empty.us_flags = pic0->us_flags;
648 	}
649 
650 	if (pic0->us_picno != 0) {
651 		/*
652 		 * pic0 is counter 1, so if we need the empty config it should
653 		 * be counter 0.
654 		 */
655 		empty.us_picno = 0;
656 		empty.us_bits = 0x14; /* SW_count_0 - won't overflow */
657 		tmp = pic0;
658 		pic0 = pic1;
659 		pic1 = tmp;
660 	}
661 
662 	if (pic0->us_picno != 0 || pic1->us_picno != 1)
663 		panic("us_pcbe: bad config on token %p\n", token);
664 
665 	/*
666 	 * UltraSPARC does not allow pic0 to be configured differently
667 	 * from pic1. If the flags on these two configurations are
668 	 * different, they are incompatible. This condition should be
669 	 * caught at configure time.
670 	 */
671 	ASSERT(pic0->us_flags == pic1->us_flags);
672 
673 	ultra_setpcr(allstopped);
674 	ultra_setpic(((uint64_t)pic1->us_pic << 32) | (uint64_t)pic0->us_pic);
675 
676 	pcr = (pic0->us_bits & pcr_pic_mask) <<
677 	    CPC_ULTRA_PCR_PIC0_SHIFT;
678 	pcr |= (pic1->us_bits & pcr_pic_mask) <<
679 	    CPC_ULTRA_PCR_PIC1_SHIFT;
680 
681 	if (pic0->us_flags & CPC_COUNT_USER)
682 		pcr |= (1ull << CPC_ULTRA_PCR_USR);
683 	if (pic0->us_flags & CPC_COUNT_SYSTEM)
684 		pcr |= (1ull << CPC_ULTRA_PCR_SYS);
685 
686 	DTRACE_PROBE1(ultra__pcr, uint64_t, pcr);
687 
688 	ultra_setpcr(pcr);
689 
690 	/*
691 	 * On UltraSPARC, only read-to-read counts are accurate. We cannot
692 	 * expect the value we wrote into the PIC, above, to be there after
693 	 * starting the counter. We must sample the counter value now and use
694 	 * that as the baseline for future samples.
695 	 */
696 	curpic = ultra_getpic();
697 	pic0->us_pic = (uint32_t)(curpic & PIC0_MASK);
698 	pic1->us_pic = (uint32_t)(curpic >> 32);
699 }
700 
701 static void
702 us_pcbe_allstop(void)
703 {
704 	ultra_setpcr(allstopped);
705 }
706 
707 
708 static void
709 us_pcbe_sample(void *token)
710 {
711 	uint64_t		curpic;
712 	int64_t			diff;
713 	uint64_t		*pic0_data;
714 	uint64_t		*pic1_data;
715 	uint64_t		*dtmp;
716 	uint64_t		tmp;
717 	us_pcbe_config_t	*pic0;
718 	us_pcbe_config_t	*pic1;
719 	us_pcbe_config_t	empty = { 1, 0, 0, 0 };
720 	us_pcbe_config_t	*ctmp;
721 
722 	curpic = ultra_getpic();
723 
724 	if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
725 		panic("us_pcbe: token %p has no configs", token);
726 
727 	if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
728 		pic1 = &empty;
729 		pic1_data = &tmp;
730 	}
731 
732 	if (pic0->us_picno != 0) {
733 		empty.us_picno = 0;
734 		ctmp = pic0;
735 		pic0 = pic1;
736 		pic1 = ctmp;
737 		dtmp = pic0_data;
738 		pic0_data = pic1_data;
739 		pic1_data = dtmp;
740 	}
741 
742 	if (pic0->us_picno != 0 || pic1->us_picno != 1)
743 		panic("us_pcbe: bad config on token %p\n", token);
744 
745 	diff = (curpic & PIC0_MASK) - (uint64_t)pic0->us_pic;
746 	if (diff < 0)
747 		diff += (1ll << 32);
748 	*pic0_data += diff;
749 
750 	diff = (curpic >> 32) - (uint64_t)pic1->us_pic;
751 	if (diff < 0)
752 		diff += (1ll << 32);
753 	*pic1_data += diff;
754 
755 	pic0->us_pic = (uint32_t)(curpic & PIC0_MASK);
756 	pic1->us_pic = (uint32_t)(curpic >> 32);
757 }
758 
759 static void
760 us_pcbe_free(void *config)
761 {
762 	kmem_free(config, sizeof (us_pcbe_config_t));
763 }
764 
765 
766 static struct modlpcbe modlpcbe = {
767 	&mod_pcbeops,
768 	"UltraSPARC Performance Counters v%I%",
769 	&us_pcbe_ops
770 };
771 
772 static struct modlinkage modl = {
773 	MODREV_1,
774 	&modlpcbe,
775 };
776 
777 int
778 _init(void)
779 {
780 	if (us_pcbe_init() != 0)
781 		return (ENOTSUP);
782 	return (mod_install(&modl));
783 }
784 
785 int
786 _fini(void)
787 {
788 	return (mod_remove(&modl));
789 }
790 
791 int
792 _info(struct modinfo *mi)
793 {
794 	return (mod_info(&modl, mi));
795 }
796