xref: /titanic_44/usr/src/uts/sun4u/pcbe/opl_pcbe.c (revision a237e38e9161f0acd6451439d4a7dd597e66291d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * SPARC64 VI Performance Counter Backend
28  */
29 
30 #pragma ident	"%Z%%M%	%I%	%E% SMI"
31 
32 #include <sys/cpuvar.h>
33 #include <sys/systm.h>
34 #include <sys/cmn_err.h>
35 #include <sys/cpc_impl.h>
36 #include <sys/cpc_pcbe.h>
37 #include <sys/modctl.h>
38 #include <sys/machsystm.h>
39 #include <sys/sdt.h>
40 #include <sys/cpu_impl.h>
41 
42 static int opl_pcbe_init(void);
43 static uint_t opl_pcbe_ncounters(void);
44 static const char *opl_pcbe_impl_name(void);
45 static const char *opl_pcbe_cpuref(void);
46 static char *opl_pcbe_list_events(uint_t picnum);
47 static char *opl_pcbe_list_attrs(void);
48 static uint64_t opl_pcbe_event_coverage(char *event);
49 static uint64_t opl_pcbe_overflow_bitmap(void);
50 static int opl_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
51     uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
52     void *token);
53 static void opl_pcbe_program(void *token);
54 static void opl_pcbe_allstop(void);
55 static void opl_pcbe_sample(void *token);
56 static void opl_pcbe_free(void *config);
57 
58 extern void ultra_setpcr(uint64_t);
59 extern uint64_t ultra_getpcr(void);
60 extern void ultra_setpic(uint64_t);
61 extern uint64_t ultra_getpic(void);
62 extern uint64_t ultra_gettick(void);
63 
64 pcbe_ops_t opl_pcbe_ops = {
65 	PCBE_VER_1,
66 	CPC_CAP_OVERFLOW_INTERRUPT,
67 	opl_pcbe_ncounters,
68 	opl_pcbe_impl_name,
69 	opl_pcbe_cpuref,
70 	opl_pcbe_list_events,
71 	opl_pcbe_list_attrs,
72 	opl_pcbe_event_coverage,
73 	opl_pcbe_overflow_bitmap,
74 	opl_pcbe_configure,
75 	opl_pcbe_program,
76 	opl_pcbe_allstop,
77 	opl_pcbe_sample,
78 	opl_pcbe_free
79 };
80 
81 typedef struct _opl_pcbe_config {
82 	uint8_t		opl_picno;	/* From 0 to 7 */
83 	uint32_t	opl_bits;	/* %pcr event code unshifted */
84 	uint32_t	opl_flags;	/* user/system/priv */
85 	uint32_t	opl_pic;	/* unshifted raw %pic value */
86 } opl_pcbe_config_t;
87 
88 struct nametable {
89 	const uint8_t	bits;
90 	const char	*name;
91 };
92 
93 /*
94  * Performance Control Register (PCR)
95  *
96  * +----------+-----+-----+------+----+
97  * |      0   | OVF |  0  | OVR0 | 0  |
98  * +----------+-----+-----+------+----+
99  * 63     48  47:32  31:27   26    25
100  *
101  * +----+----+--- -+----+-----+---+-----+-----+----+----+----+
102  * | NC |  0 | SC  | 0  | SU  | 0 | SL  |ULRO | UT | ST |PRIV|
103  * +----+----+-----+----+-----+---+-----+-----+----+----+----+
104  * 24:22  21  20:18  17  16:11 10  9:4     3    2    1    0
105  *
106  * ULRO and OVRO bits should be on upon accessing pcr unless
107  * those fields need to be updated.
108  * Turn off these bits when updating SU/SL or OVF field
109  * (during initialization, etc.).
110  *
111  *
112  * Performance Instrumentation Counter (PIC)
113  * Four PICs are implemented in SPARC64 VI,
114  * each PIC is accessed using PCR.SC as a select field.
115  *
116  * +------------------------+--------------------------+
117  * |         PICU	    |		PICL	       |
118  * +------------------------+--------------------------+
119  *  63			 32  31			      0
120  */
121 
122 #define	PIC_MASK (((uint64_t)1 << 32) - 1)
123 
124 #define	SPARC64_VI_PCR_PRIVPIC  UINT64_C(0)
125 
126 #define	CPC_SPARC64_VI_PCR_SYS_SHIFT	1
127 #define	CPC_SPARC64_VI_PCR_USR_SHIFT	2
128 
129 #define	CPC_SPARC64_VI_PCR_PICL_SHIFT	4
130 #define	CPC_SPARC64_VI_PCR_PICU_SHIFT	11
131 #define	CPC_SPARC64_VI_PCR_PIC_MASK	UINT64_C(0x3F)
132 
133 #define	CPC_SPARC64_VI_NPIC		8
134 
135 #define	CPC_SPARC64_VI_PCR_ULRO_SHIFT	3
136 #define	CPC_SPARC64_VI_PCR_SC_SHIFT	18
137 #define	CPC_SPARC64_VI_PCR_SC_MASK	UINT64_C(0x7)
138 #define	CPC_SPARC64_VI_PCR_NC_SHIFT	22
139 #define	CPC_SPARC64_VI_PCR_NC_MASK	UINT64_C(0x7)
140 #define	CPC_SPARC64_VI_PCR_OVRO_SHIFT	26
141 #define	CPC_SPARC64_VI_PCR_OVF_SHIFT	32
142 #define	CPC_SPARC64_VI_PCR_OVF_MASK	UINT64_C(0xffff)
143 
144 #define	SPARC64_VI_PCR_SYS	(UINT64_C(1) << CPC_SPARC64_VI_PCR_SYS_SHIFT)
145 #define	SPARC64_VI_PCR_USR	(UINT64_C(1) << CPC_SPARC64_VI_PCR_USR_SHIFT)
146 #define	SPARC64_VI_PCR_ULRO	(UINT64_C(1) << CPC_SPARC64_VI_PCR_ULRO_SHIFT)
147 #define	SPARC64_VI_PCR_OVRO	(UINT64_C(1) << CPC_SPARC64_VI_PCR_OVRO_SHIFT)
148 #define	SPARC64_VI_PCR_OVF	(CPC_SPARC64_VI_PCR_OVF_MASK << \
149 					CPC_SPARC64_VI_PCR_OVF_SHIFT)
150 
151 #define	SPARC64_VI_NUM_PIC_PAIRS	4
152 
153 #define	SPARC64_VI_PCR_SEL_PIC(pcr, picno) {				\
154 	pcr &= ~((CPC_SPARC64_VI_PCR_SC_MASK				\
155 		<< CPC_SPARC64_VI_PCR_SC_SHIFT));			\
156 									\
157 	pcr |= (((picno) & CPC_SPARC64_VI_PCR_SC_MASK)			\
158 		<< CPC_SPARC64_VI_PCR_SC_SHIFT);			\
159 }
160 
161 #define	SPARC64_VI_PCR_SEL_EVENT(pcr, sl, su) {				\
162 	pcr &= ~((CPC_SPARC64_VI_PCR_PIC_MASK				\
163 		<< CPC_SPARC64_VI_PCR_PICL_SHIFT)			\
164 	    | (CPC_SPARC64_VI_PCR_PIC_MASK				\
165 		<< CPC_SPARC64_VI_PCR_PICU_SHIFT));			\
166 									\
167 	pcr |= (((sl) & CPC_SPARC64_VI_PCR_PIC_MASK)			\
168 		<< CPC_SPARC64_VI_PCR_PICL_SHIFT);			\
169 	pcr |= (((su) & CPC_SPARC64_VI_PCR_PIC_MASK)			\
170 		<< CPC_SPARC64_VI_PCR_PICU_SHIFT);			\
171 }
172 
173 #define	SPARC64_VI_CHK_OVF(pcr, picno)					\
174 	((pcr) & (UINT64_C(1) << (CPC_SPARC64_VI_PCR_OVF_SHIFT + picno)))
175 
176 #define	SPARC64_VI_CLR_OVF(pcr, picno) {				\
177 	pcr &= ~(UINT64_C(1) << (CPC_SPARC64_VI_PCR_OVF_SHIFT + picno)); \
178 }
179 
180 #define	NT_END 0xFF
181 
182 static const uint64_t   allstopped = SPARC64_VI_PCR_PRIVPIC |
183 	SPARC64_VI_PCR_ULRO | SPARC64_VI_PCR_OVRO;
184 
185 #define	SPARC64_VI_EVENTS_comm_0		\
186 	{0x0,	"cycle_counts"},		\
187 	{0x1,	"instruction_counts"}
188 
189 #define	SPARC64_VI_EVENTS_comm_1		\
190 	{0x5,	"op_stv_wait"},			\
191 	{0x8,	"load_store_instructions"},	\
192 	{0x9,	"branch_instructions"},		\
193 	{0xa,	"floating_instructions"},	\
194 	{0xb,	"impdep2_instructions"},	\
195 	{0xc,	"prefetch_instructions"}
196 
197 #define	SPARC64_VI_EVENTS_comm_2		\
198 	{0x1a,	"active_cycle_count"}
199 
200 static const struct nametable SPARC64_VI_names_l0[] = {
201 	SPARC64_VI_EVENTS_comm_0,
202 	SPARC64_VI_EVENTS_comm_1,
203 	{0x12,	"flush_rs"},
204 	{0x13,	"2iid_use"},
205 	{0x15,	"toq_rsbr_phantom"},
206 	{0x16,	"trap_int_vector"},
207 	{0x18,	"ts_by_sxmiss"},
208 	SPARC64_VI_EVENTS_comm_2,
209 	{0x1d,	"op_stv_wait_sxmiss"},
210 	{0x1e,	"eu_comp_wait"},
211 	{0x24,	"swpf_fail_all"},
212 	{0x30,	"sx_miss_wait_pf"},
213 	{0x31,	"jbus_cpi_count"},
214 	{0x36,	"jbus_reqbus1_busy"},
215 	{NT_END, ""}
216 };
217 
218 static const struct nametable SPARC64_VI_names_u0[] = {
219 	SPARC64_VI_EVENTS_comm_0,
220 	{0x2,	"instruction_flow_counts"},
221 	{0x3,	"iwr_empty"},
222 	SPARC64_VI_EVENTS_comm_1,
223 	{0x12,	"rs1"},
224 	{0x13,	"1iid_use"},
225 	{0x16,	"trap_all"},
226 	{0x18,	"thread_switch_all"},
227 	SPARC64_VI_EVENTS_comm_2,
228 	{0x1d,	"act_thread_suspend"},
229 	{0x1e,	"cse_window_empty"},
230 	{0x1f,	"inh_cmit_gpr_2write"},
231 	{0x24,	"swpf_success_all"},
232 	{0x30,	"sx_miss_wait_dm"},
233 	{0x31,	"jbus_bi_count"},
234 	{0x34,	"lost_softpf_pfp_full"},
235 	{0x36,	"jbus_reqbus0_busy"},
236 	{NT_END, ""}
237 };
238 
239 static const struct nametable SPARC64_VI_names_l1[] = {
240 	SPARC64_VI_EVENTS_comm_0,
241 	SPARC64_VI_EVENTS_comm_1,
242 	{0x13,	"4iid_use"},
243 	{0x15,	"flush_rs"},
244 	{0x16,	"trap_spill"},
245 	{0x18,	"ts_by_timer"},
246 	SPARC64_VI_EVENTS_comm_2,
247 	{0x1b,	"0iid_use"},
248 	{0x1d,	"op_stv_wait_nc_pend"},
249 	{0x1e,	"0endop"},
250 	{0x20,	"write_op_uTLB"},
251 	{0x30,	"sx_miss_count_pf"},
252 	{0x31,	"jbus_cpd_count"},
253 	{0x32,	"snres_64"},
254 	{0x36,	"jbus_reqbus3_busy"},
255 	{NT_END, ""}
256 };
257 
258 static const struct nametable SPARC64_VI_names_u1[] = {
259 	SPARC64_VI_EVENTS_comm_0,
260 	SPARC64_VI_EVENTS_comm_1,
261 	{0x13,	"3iid_use"},
262 	{0x16,	"trap_int_level"},
263 	{0x18,	"ts_by_data_arrive"},
264 	SPARC64_VI_EVENTS_comm_2,
265 	{0x1b,	"op_stv_wait_nc_pend"},
266 	{0x1d,	"op_stv_wait_sxmiss_ex"},
267 	{0x1e,	"branch_comp_wait"},
268 	{0x20,	"write_if_uTLB"},
269 	{0x30,	"sx_miss_count_dm"},
270 	{0x31,	"jbus_cpb_count"},
271 	{0x32,	"snres_256"},
272 	{0x34,	"lost_softpf_by_abort"},
273 	{0x36,	"jbus_reqbus2_busy"},
274 	{NT_END, ""}
275 };
276 
277 static const struct nametable SPARC64_VI_names_l2[] = {
278 	SPARC64_VI_EVENTS_comm_0,
279 	SPARC64_VI_EVENTS_comm_1,
280 	{0x13,	"sync_intlk"},
281 	{0x16,	"trap_trap_inst"},
282 	{0x18,	"ts_by_if"},
283 	SPARC64_VI_EVENTS_comm_2,
284 	{0x1e,	"fl_comp_wait"},
285 	{0x20,	"op_r_iu_req_mi_go"},
286 	{0x30,	"sx_read_count_pf"},
287 	{0x31,	"jbus_odrbus_busy"},
288 	{0x33,	"sx_miss_count_dm_if"},
289 	{0x36,	"jbus_odrbus1_busy"},
290 	{NT_END, ""}
291 };
292 
293 static const struct nametable SPARC64_VI_names_u2[] = {
294 	SPARC64_VI_EVENTS_comm_0,
295 	{0x2,	"instruction_flow_counts"},
296 	{0x3,	"iwr_empty"},
297 	SPARC64_VI_EVENTS_comm_1,
298 	{0x16,	"trap_fill"},
299 	{0x18,	"ts_by_intr"},
300 	SPARC64_VI_EVENTS_comm_2,
301 	{0x1b,	"flush_rs"},
302 	{0x1d,	"cse_window_empty_sp_full"},
303 	{0x1e,	"op_stv_wait_ex"},
304 	{0x1f,	"3endop"},
305 	{0x20,	"if_r_iu_req_mi_go"},
306 	{0x24,	"swpf_lbs_hit"},
307 	{0x30,	"sx_read_count_dm"},
308 	{0x31,	"jbus_reqbus_busy"},
309 	{0x33,	"sx_btc_count"},
310 	{0x36,	"jbus_odrbus0_busy"},
311 	{NT_END, ""}
312 };
313 
314 static const struct nametable SPARC64_VI_names_l3[] = {
315 	SPARC64_VI_EVENTS_comm_0,
316 	SPARC64_VI_EVENTS_comm_1,
317 	{0x16,	"trap_DMMU_miss"},
318 	{0x18,	"ts_by_suspend"},
319 	{0x19,	"ts_by_other"},
320 	SPARC64_VI_EVENTS_comm_2,
321 	{0x1b,	"decall_intlk"},
322 	{0x1e,	"2endop"},
323 	{0x1f,	"op_stv_wait_sxmiss"},
324 	{0x20,	"op_wait_all"},
325 	{0x30,	"dvp_count_pf"},
326 	{0x33,	"sx_miss_count_dm_opex"},
327 	{0x36,	"jbus_odrbus3_busy"},
328 	{NT_END, ""}
329 };
330 
331 static const struct nametable SPARC64_VI_names_u3[] = {
332 	SPARC64_VI_EVENTS_comm_0,
333 	SPARC64_VI_EVENTS_comm_1,
334 	{0x13,	"regwin_intlk"},
335 	{0x15,	"rs1"},
336 	{0x16,	"trap_IMMU_miss"},
337 	SPARC64_VI_EVENTS_comm_2,
338 	{0x1e,	"1endop"},
339 	{0x1f,	"op_stv_wait_sxmiss_ex"},
340 	{0x20,	"if_wait_all"},
341 	{0x30,	"dvp_count_dm"},
342 	{0x33,	"sx_miss_count_dm_opsh"},
343 	{0x36,	"jbus_odrbus2_busy"},
344 	{NT_END, ""}
345 };
346 
347 #undef	SPARC64_VI_EVENTS_comm_0
348 #undef	SPARC64_VI_EVENTS_comm_1
349 #undef	SPARC64_VI_EVENTS_comm_2
350 
351 static const struct nametable *SPARC64_VI_names[CPC_SPARC64_VI_NPIC] = {
352 	SPARC64_VI_names_l0,
353 	SPARC64_VI_names_u0,
354 	SPARC64_VI_names_l1,
355 	SPARC64_VI_names_u1,
356 	SPARC64_VI_names_l2,
357 	SPARC64_VI_names_u2,
358 	SPARC64_VI_names_l3,
359 	SPARC64_VI_names_u3
360 };
361 
362 opl_pcbe_config_t nullpic[CPC_SPARC64_VI_NPIC] = {
363 	{0, 0x3f, 0, 0},
364 	{1, 0x3f, 0, 0},
365 	{2, 0x3f, 0, 0},
366 	{3, 0x3f, 0, 0},
367 	{4, 0x3f, 0, 0},
368 	{5, 0x3f, 0, 0},
369 	{6, 0x3f, 0, 0},
370 	{7, 0x3f, 0, 0}
371 };
372 
373 static const struct nametable **events;
374 static const char *opl_impl_name;
375 static const char *opl_cpuref;
376 static char *pic_events[CPC_SPARC64_VI_NPIC];
377 
378 static const char *sp_6_ref = "See the \"SPARC64 VI extensions\" "
379 			"for descriptions of these events.";
380 
381 static int
382 opl_pcbe_init(void)
383 {
384 	const struct nametable	*n;
385 	int			i;
386 	size_t			size;
387 
388 	/*
389 	 * Discover type of CPU
390 	 *
391 	 * Point nametable to that CPU's table
392 	 */
393 	switch (ULTRA_VER_IMPL(ultra_getver())) {
394 	case OLYMPUS_C_IMPL:
395 		events = SPARC64_VI_names;
396 		opl_impl_name = "SPARC64 VI";
397 		opl_cpuref = sp_6_ref;
398 		break;
399 	default:
400 		return (-1);
401 	}
402 
403 	/*
404 	 * Initialize the list of events for each PIC.
405 	 * Do two passes: one to compute the size necessary and another
406 	 * to copy the strings. Need room for event, comma, and NULL terminator.
407 	 */
408 	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
409 		size = 0;
410 		for (n = events[i]; n->bits != NT_END; n++)
411 			size += strlen(n->name) + 1;
412 		pic_events[i] = kmem_alloc(size + 1, KM_SLEEP);
413 		*pic_events[i] = '\0';
414 		for (n = events[i]; n->bits != NT_END; n++) {
415 			(void) strcat(pic_events[i], n->name);
416 			(void) strcat(pic_events[i], ",");
417 		}
418 		/*
419 		 * Remove trailing comma.
420 		 */
421 		pic_events[i][size - 1] = '\0';
422 	}
423 
424 	return (0);
425 }
426 
427 static uint_t
428 opl_pcbe_ncounters(void)
429 {
430 	return (CPC_SPARC64_VI_NPIC);
431 }
432 
433 static const char *
434 opl_pcbe_impl_name(void)
435 {
436 	return (opl_impl_name);
437 }
438 
439 static const char *
440 opl_pcbe_cpuref(void)
441 {
442 	return (opl_cpuref);
443 }
444 
445 static char *
446 opl_pcbe_list_events(uint_t picnum)
447 {
448 	ASSERT(picnum >= 0 && picnum < cpc_ncounters);
449 
450 	return (pic_events[picnum]);
451 }
452 
453 static char *
454 opl_pcbe_list_attrs(void)
455 {
456 	return ("");
457 }
458 
459 static const struct nametable *
460 find_event(int regno, char *name)
461 {
462 	const struct nametable *n;
463 
464 	n = events[regno];
465 
466 	for (; n->bits != NT_END; n++)
467 		if (strcmp(name, n->name) == 0)
468 			return (n);
469 
470 	return (NULL);
471 }
472 
473 static uint64_t
474 opl_pcbe_event_coverage(char *event)
475 {
476 	uint64_t bitmap = 0;
477 
478 	int	i;
479 	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
480 		if (find_event(i, event) != NULL)
481 			bitmap |= (1 << i);
482 	}
483 
484 	return (bitmap);
485 }
486 
487 /*
488  * Check if counter overflow and clear it.
489  */
490 static uint64_t
491 opl_pcbe_overflow_bitmap(void)
492 {
493 	uint64_t	pcr;
494 
495 	pcr = ultra_getpcr();
496 	DTRACE_PROBE1(sparc64__getpcr, uint64_t, pcr);
497 
498 	return ((pcr & SPARC64_VI_PCR_OVF) >> CPC_SPARC64_VI_PCR_OVF_SHIFT);
499 }
500 
501 /*ARGSUSED*/
502 static int
503 opl_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
504     uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
505 {
506 	opl_pcbe_config_t *conf;
507 	const struct nametable *n;
508 	opl_pcbe_config_t *other_config;
509 
510 	/*
511 	 * If we've been handed an existing configuration, we need only preset
512 	 * the counter value.
513 	 */
514 	if (*data != NULL) {
515 		conf = *data;
516 		conf->opl_pic = (uint32_t)preset;
517 		return (0);
518 	}
519 
520 	if (picnum < 0 || picnum >= CPC_SPARC64_VI_NPIC)
521 		return (CPC_INVALID_PICNUM);
522 
523 	if (nattrs != 0)
524 		return (CPC_INVALID_ATTRIBUTE);
525 
526 	/*
527 	 * Find other requests that will be programmed with this one, and ensure
528 	 * the flags don't conflict.
529 	 */
530 	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
531 	    (other_config->opl_flags != flags))
532 		return (CPC_CONFLICTING_REQS);
533 
534 	if ((n = find_event(picnum, event)) == NULL)
535 		return (CPC_INVALID_EVENT);
536 
537 	conf = kmem_alloc(sizeof (opl_pcbe_config_t), KM_SLEEP);
538 
539 	conf->opl_picno = picnum;
540 	conf->opl_bits = (uint32_t)n->bits;
541 	conf->opl_flags = flags;
542 	conf->opl_pic = (uint32_t)preset;
543 
544 	*data = conf;
545 	return (0);
546 }
547 
548 static void
549 opl_pcbe_program(void *token)
550 {
551 	opl_pcbe_config_t	*pic[CPC_SPARC64_VI_NPIC];
552 	opl_pcbe_config_t	*firstconfig;
553 	opl_pcbe_config_t	*tmp;
554 	uint64_t		pcr;
555 	uint64_t		curpic;
556 	uint8_t			bitmap = 0;	/* for used pic config */
557 	int			i;
558 	opl_pcbe_config_t	dummypic[CPC_SPARC64_VI_NPIC];
559 
560 	/* Get next pic config */
561 	firstconfig = tmp = kcpc_next_config(token, NULL, NULL);
562 
563 	while (tmp != NULL) {
564 		ASSERT(tmp->opl_picno < CPC_SPARC64_VI_NPIC);
565 		ASSERT(firstconfig->opl_flags == tmp->opl_flags);
566 		pic[tmp->opl_picno] = tmp;
567 		bitmap |= (uint8_t)(1 << tmp->opl_picno);
568 		tmp = kcpc_next_config(token, tmp, NULL);
569 	}
570 	if (bitmap == 0)
571 		panic("opl_pcbe: token %p has no configs", token);
572 
573 	/* Fill in unused pic config */
574 	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
575 		if (bitmap & (1 << i))
576 			continue;
577 
578 		dummypic[i] = nullpic[i];
579 		dummypic[i].opl_flags = firstconfig->opl_flags;
580 		pic[i] = &dummypic[i];
581 	}
582 
583 	/*
584 	 * For each counter pair, initialize event settings and
585 	 * counter values.
586 	 */
587 	ultra_setpcr(allstopped);
588 	pcr = allstopped;
589 	pcr &= ~SPARC64_VI_PCR_ULRO;
590 	for (i = 0; i < SPARC64_VI_NUM_PIC_PAIRS; i++) {
591 		SPARC64_VI_PCR_SEL_PIC(pcr, i);
592 		SPARC64_VI_PCR_SEL_EVENT(pcr, pic[i*2]->opl_bits,
593 		    pic[i*2 + 1]->opl_bits);
594 
595 		ultra_setpcr(pcr);
596 		curpic = (uint64_t)(pic[i*2]->opl_pic |
597 		    ((uint64_t)pic[i*2 + 1]->opl_pic << 32));
598 		ultra_setpic(curpic);
599 	}
600 
601 	/*
602 	 * For each counter pair, enable the trace flags to start
603 	 * counting. Re-read the counters to sample the counter value now
604 	 * and use that as the baseline for future samples.
605 	 */
606 
607 	/* Get PCR */
608 	pcr = ultra_getpcr();
609 	pcr |= SPARC64_VI_PCR_ULRO;
610 	pcr &= ~(SPARC64_VI_PCR_OVRO | SPARC64_VI_PCR_OVF);
611 
612 	if (pic[0]->opl_flags & CPC_COUNT_USER)
613 		pcr |= SPARC64_VI_PCR_USR;
614 	if (pic[0]->opl_flags & CPC_COUNT_SYSTEM)
615 		pcr |= SPARC64_VI_PCR_SYS;
616 
617 	/* Set counter values */
618 
619 	for (i = 0; i < SPARC64_VI_NUM_PIC_PAIRS; i++) {
620 		SPARC64_VI_PCR_SEL_PIC(pcr, i);
621 		SPARC64_VI_PCR_SEL_EVENT(pcr, pic[i*2]->opl_bits,
622 		    pic[i*2 + 1]->opl_bits);
623 
624 		ultra_setpcr(pcr);
625 		DTRACE_PROBE1(sparc64__setpcr, uint64_t, pcr);
626 
627 		curpic = ultra_getpic();
628 		DTRACE_PROBE1(sparc64__newpic, uint64_t, curpic);
629 		pic[i*2]->opl_pic = (uint32_t)(curpic & PIC_MASK);
630 		pic[i*2 + 1]->opl_pic = (uint32_t)(curpic >> 32);
631 	}
632 	pcr |= SPARC64_VI_PCR_OVRO;
633 	ultra_setpcr(pcr);
634 }
635 
636 static void
637 opl_pcbe_allstop(void)
638 {
639 	ultra_setpcr(allstopped);
640 }
641 
642 
643 static void
644 opl_pcbe_sample(void *token)
645 {
646 	uint64_t		curpic;
647 	uint64_t		pcr;
648 	uint64_t		overflow;
649 	int64_t			diff;
650 	uint64_t		*pic_data[CPC_SPARC64_VI_NPIC];
651 	uint64_t		*dtmp;
652 	opl_pcbe_config_t	*pic[CPC_SPARC64_VI_NPIC];
653 	opl_pcbe_config_t	*ctmp;
654 	opl_pcbe_config_t	*firstconfig;
655 	uint8_t			bitmap = 0;	/* for used pic config */
656 	int			i;
657 	opl_pcbe_config_t dummypic[CPC_SPARC64_VI_NPIC];
658 	uint64_t dummypic_data[CPC_SPARC64_VI_NPIC];
659 
660 	/* Get next pic config */
661 	firstconfig = ctmp = kcpc_next_config(token, NULL, &dtmp);
662 
663 	while (ctmp != NULL) {
664 		ASSERT(ctmp->opl_picno < CPC_SPARC64_VI_NPIC);
665 		ASSERT(firstconfig->opl_flags == ctmp->opl_flags);
666 		pic[ctmp->opl_picno] = ctmp;
667 		pic_data[ctmp->opl_picno] = dtmp;
668 		bitmap |= (uint8_t)(1 << ctmp->opl_picno);
669 		ctmp = kcpc_next_config(token, ctmp, &dtmp);
670 	}
671 	if (bitmap == 0)
672 		panic("opl_pcbe: token %p has no configs", token);
673 
674 	/* Fill in unuse pic config */
675 	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
676 		if (bitmap & (1 << i))
677 			continue;
678 
679 		dummypic[i] = nullpic[i];
680 		dummypic[i].opl_flags = firstconfig->opl_flags;
681 		pic[i] = &dummypic[i];
682 
683 		dummypic_data[i] = 0;
684 		pic_data[i] = &dummypic_data[i];
685 	}
686 
687 	pcr = ultra_getpcr();
688 	pcr &= ~SPARC64_VI_PCR_OVRO;
689 	pcr |= SPARC64_VI_PCR_ULRO;
690 
691 	for (i = 0; i < SPARC64_VI_NUM_PIC_PAIRS; i++) {
692 		SPARC64_VI_PCR_SEL_PIC(pcr, i);
693 		SPARC64_VI_PCR_SEL_EVENT(pcr, pic[i*2]->opl_bits,
694 		    pic[i*2 + 1]->opl_bits);
695 
696 		ultra_setpcr(pcr);
697 
698 		curpic = ultra_getpic();
699 		DTRACE_PROBE1(sparc64__getpic, unit64_t, curpic);
700 
701 		diff = (curpic & PIC_MASK) - (uint64_t)pic[i*2]->opl_pic;
702 		overflow = SPARC64_VI_CHK_OVF(pcr, i*2);
703 		if (overflow || (diff < 0)) {
704 			SPARC64_VI_CLR_OVF(pcr, i*2);
705 			ultra_setpcr(pcr);
706 			diff += (1ll << 32);
707 		}
708 		*pic_data[i*2] += diff;
709 
710 		diff = (curpic >> 32) - (uint64_t)pic[i*2 + 1]->opl_pic;
711 		overflow = SPARC64_VI_CHK_OVF(pcr, i*2 + 1);
712 		if (overflow || (diff < 0)) {
713 			SPARC64_VI_CLR_OVF(pcr, i*2 + 1);
714 			ultra_setpcr(pcr);
715 			diff += (1ll << 32);
716 		}
717 		*pic_data[i*2 + 1] += diff;
718 
719 		pic[i*2]->opl_pic = (uint32_t)(curpic & PIC_MASK);
720 		pic[i*2 + 1]->opl_pic = (uint32_t)(curpic >> 32);
721 	}
722 	pcr = ultra_getpcr();
723 	pcr |= SPARC64_VI_PCR_OVRO;
724 	ultra_setpcr(pcr);
725 }
726 
727 static void
728 opl_pcbe_free(void *config)
729 {
730 	kmem_free(config, sizeof (opl_pcbe_config_t));
731 }
732 
733 
734 static struct modlpcbe modlpcbe = {
735 	&mod_pcbeops,
736 	"SPARC64 VI Performance Counters v%I%",
737 	&opl_pcbe_ops
738 };
739 
740 static struct modlinkage modl = {
741 	MODREV_1,
742 	&modlpcbe,
743 };
744 
745 int
746 _init(void)
747 {
748 	if (opl_pcbe_init() != 0)
749 		return (ENOTSUP);
750 	return (mod_install(&modl));
751 }
752 
753 int
754 _fini(void)
755 {
756 	return (mod_remove(&modl));
757 }
758 
759 int
760 _info(struct modinfo *mi)
761 {
762 	return (mod_info(&modl, mi));
763 }
764