xref: /titanic_50/usr/src/uts/sun4v/pcbe/niagara2_pcbe.c (revision e127a3e717f822eb855235fa3bd08235b2cf533d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Niagara2 Performance Counter Backend
28  */
29 
30 #pragma ident	"%Z%%M%	%I%	%E% SMI"
31 
32 #include <sys/cpuvar.h>
33 #include <sys/systm.h>
34 #include <sys/archsystm.h>
35 #include <sys/cmn_err.h>
36 #include <sys/cpc_impl.h>
37 #include <sys/cpc_pcbe.h>
38 #include <sys/modctl.h>
39 #include <sys/machsystm.h>
40 #include <sys/sdt.h>
41 #include <sys/niagara2regs.h>
42 #include <sys/hsvc.h>
43 #include <sys/hypervisor_api.h>
44 #include <sys/disp.h>
45 
46 static int ni2_pcbe_init(void);
47 static uint_t ni2_pcbe_ncounters(void);
48 static const char *ni2_pcbe_impl_name(void);
49 static const char *ni2_pcbe_cpuref(void);
50 static char *ni2_pcbe_list_events(uint_t picnum);
51 static char *ni2_pcbe_list_attrs(void);
52 static uint64_t ni2_pcbe_event_coverage(char *event);
53 static uint64_t ni2_pcbe_overflow_bitmap(void);
54 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
55     uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
56     void *token);
57 static void ni2_pcbe_program(void *token);
58 static void ni2_pcbe_allstop(void);
59 static void ni2_pcbe_sample(void *token);
60 static void ni2_pcbe_free(void *config);
61 
62 extern void ultra_setpcr(uint64_t);
63 extern uint64_t ultra_getpcr(void);
64 extern void ultra_setpic(uint64_t);
65 extern uint64_t ultra_getpic(void);
66 extern uint64_t ultra_gettick(void);
67 extern char cpu_module_name[];
68 
69 #ifdef N2_1x_CPC_WORKAROUNDS
70 extern uint64_t ni2_1x_perf_workarounds;
71 #endif
72 
73 pcbe_ops_t ni2_pcbe_ops = {
74 	PCBE_VER_1,
75 	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,
76 	ni2_pcbe_ncounters,
77 	ni2_pcbe_impl_name,
78 	ni2_pcbe_cpuref,
79 	ni2_pcbe_list_events,
80 	ni2_pcbe_list_attrs,
81 	ni2_pcbe_event_coverage,
82 	ni2_pcbe_overflow_bitmap,
83 	ni2_pcbe_configure,
84 	ni2_pcbe_program,
85 	ni2_pcbe_allstop,
86 	ni2_pcbe_sample,
87 	ni2_pcbe_free
88 };
89 
90 typedef struct _ni2_pcbe_config {
91 	uint_t		pcbe_picno;	/* 0 for pic0 or 1 for pic1 */
92 	uint32_t	pcbe_evsel;	/* %pcr event code unshifted */
93 	uint32_t	pcbe_flags;	/* hpriv/user/system/priv */
94 	uint32_t	pcbe_pic;	/* unshifted raw %pic value */
95 } ni2_pcbe_config_t;
96 
97 typedef struct _ni2_event {
98 	const char	*name;
99 	const uint32_t	emask;
100 	const uint32_t	emask_valid;	/* Mask of unreserved MASK bits */
101 } ni2_event_t;
102 
103 #define	ULTRA_PCR_PRIVPIC	(UINT64_C(1) << CPC_NIAGARA2_PCR_PRIV_SHIFT)
104 #define	EV_END {NULL, 0, 0}
105 
106 static const uint64_t	allstopped = (ULTRA_PCR_PRIVPIC |
107 	CPC_NIAGARA2_PCR_HOLDOV0 | CPC_NIAGARA2_PCR_HOLDOV1);
108 
109 /*
110  * We update this array in the program and allstop routine. The array
111  * is checked in the sample routine to allow us to only perform the
112  * PCR.ht bit check when counting is in progress.
113  */
114 static boolean_t ni2_cpc_counting[NCPU];
115 
116 #ifdef N2_1x_CPC_WORKAROUNDS
117 static ni2_event_t ni2_1x_events[] = {
118 	{ "Idle_strands",			0x000, 0x00 },
119 	{ "Br_completed",			0x201, 0x7f },
120 	{ "Br_taken",				0x202, 0x7f },
121 	{ "Instr_FGU_arithmetic",		0x204, 0x7f },
122 	{ "Instr_ld",				0x208, 0x7f },
123 	{ "Instr_st",				0x210, 0x7f },
124 	{ "Instr_sw",				0x220, 0x7f },
125 	{ "Instr_other",			0x240, 0x7f },
126 	{ "Instr_cnt",				0x27d, 0x7f },
127 	{ "IC_miss",				0x301, 0x3f },
128 	{ "DC_miss",				0x302, 0x3f },
129 	{ "ITLB_miss",				0x304, 0x3f },
130 	{ "DTLB_miss",				0x308, 0x3f },
131 	{ "L2_imiss",				0x310, 0x3f },
132 	{ "L2_dmiss_ld",			0x320, 0x3f },
133 	{ "ITLB_HWTW_ref_L2",			0x404, 0x3c },
134 	{ "DTLB_HWTW_ref_L2",			0x408, 0x3c },
135 	{ "ITLB_HWTW_miss_L2",			0x410, 0x3c },
136 	{ "DTLB_HWTW_miss_L2",			0x420, 0x3c },
137 	{ "Stream_ld_to_PCX",			0x501, 0x3f },
138 	{ "Stream_st_to_PCX",			0x502, 0x3f },
139 	{ "CPU_ld_to_PCX",			0x504, 0x3f },
140 	{ "CPU_ifetch_to_PCX",			0x508, 0x3f },
141 	{ "CPU_st_to_PCX",			0x510, 0x3f },
142 	{ "MMU_ld_to_PCX",			0x520, 0x3f },
143 	{ "DES_3DES_op",			0x601, 0x3f },
144 	{ "AES_op",				0x602, 0x3f },
145 	{ "RC4_op",				0x604, 0x3f },
146 	{ "MD5_SHA-1_SHA-256_op",		0x608, 0x3f },
147 	{ "MA_op",				0x610, 0x3f },
148 	{ "CRC_TCPIP_cksum",			0x620, 0x3f },
149 	{ "DES_3DES_busy_cycle",		0x701, 0x3f },
150 	{ "AES_busy_cycle",			0x702, 0x3f },
151 	{ "RC4_busy_cycle",			0x704, 0x3f },
152 	{ "MD5_SHA-1_SHA-256_busy_cycle",	0x708, 0x3f },
153 	{ "MA_busy_cycle",			0x710, 0x3f },
154 	{ "CRC_MPA_cksum",			0x720, 0x3f },
155 	EV_END
156 };
157 #endif
158 
159 static ni2_event_t ni2_2x_events[] = {
160 	{ "Idle_strands",			0x000, 0x00 },
161 	{ "Br_completed",			0x201, 0xff },
162 	{ "Br_taken",				0x202, 0xff },
163 	{ "Instr_FGU_arithmetic",		0x204, 0xff },
164 	{ "Instr_ld",				0x208, 0xff },
165 	{ "Instr_st",				0x210, 0xff },
166 	{ "Instr_sw",				0x220, 0xff },
167 	{ "Instr_other",			0x240, 0xff },
168 	{ "Atomics",				0x280, 0xff },
169 	{ "Instr_cnt",				0x2fd, 0xff },
170 	{ "IC_miss",				0x301, 0x33 },
171 	{ "DC_miss",				0x302, 0x33 },
172 	{ "L2_imiss",				0x310, 0x33 },
173 	{ "L2_dmiss_ld",			0x320, 0x33 },
174 	{ "ITLB_HWTW_ref_L2",			0x404, 0x3c },
175 	{ "DTLB_HWTW_ref_L2",			0x408, 0x3c },
176 	{ "ITLB_HWTW_miss_L2",			0x410, 0x3c },
177 	{ "DTLB_HWTW_miss_L2",			0x420, 0x3c },
178 	{ "Stream_ld_to_PCX",			0x501, 0x3f },
179 	{ "Stream_st_to_PCX",			0x502, 0x3f },
180 	{ "CPU_ld_to_PCX",			0x504, 0x3f },
181 	{ "CPU_ifetch_to_PCX",			0x508, 0x3f },
182 	{ "CPU_st_to_PCX",			0x510, 0x3f },
183 	{ "MMU_ld_to_PCX",			0x520, 0x3f },
184 	{ "DES_3DES_op",			0x601, 0x3f },
185 	{ "AES_op",				0x602, 0x3f },
186 	{ "RC4_op",				0x604, 0x3f },
187 	{ "MD5_SHA-1_SHA-256_op",		0x608, 0x3f },
188 	{ "MA_op",				0x610, 0x3f },
189 	{ "CRC_TCPIP_cksum",			0x620, 0x3f },
190 	{ "DES_3DES_busy_cycle",		0x701, 0x3f },
191 	{ "AES_busy_cycle",			0x702, 0x3f },
192 	{ "RC4_busy_cycle",			0x704, 0x3f },
193 	{ "MD5_SHA-1_SHA-256_busy_cycle",	0x708, 0x3f },
194 	{ "MA_busy_cycle",			0x710, 0x3f },
195 	{ "CRC_MPA_cksum",			0x720, 0x3f },
196 	{ "ITLB_miss",				0xb04, 0x0c },
197 	{ "DTLB_miss",				0xb08, 0x0c },
198 	{ "TLB_miss",				0xb0c, 0x0c },
199 	EV_END
200 };
201 
202 static ni2_event_t	*ni2_events = ni2_2x_events;
203 static const char	*ni2_impl_name = "UltraSPARC T2";
204 static char		*evlist;
205 static size_t		evlist_sz;
206 static uint16_t 	pcr_pic0_mask;
207 static uint16_t 	pcr_pic1_mask;
208 
209 #define	CPU_REF_URL " Documentation for Sun processors can be found at: " \
210 			"http://www.sun.com/processors/manuals"
211 
212 static const char *niagara2_cpuref = "See the \"UltraSPARC T2 User's Manual\" "
213 			"for descriptions of these events." CPU_REF_URL;
214 
215 static boolean_t niagara2_hsvc_available = B_TRUE;
216 
217 static int
218 ni2_pcbe_init(void)
219 {
220 	ni2_event_t	*evp;
221 	int		status;
222 	uint64_t	niagara2_hsvc_major;
223 	uint64_t	niagara2_hsvc_minor;
224 
225 	pcr_pic0_mask = CPC_NIAGARA2_PCR_PIC0_MASK;
226 	pcr_pic1_mask = CPC_NIAGARA2_PCR_PIC1_MASK;
227 
228 	/*
229 	 * Validate API version for Niagara2 specific hypervisor services
230 	 */
231 	status = hsvc_version(HSVC_GROUP_NIAGARA2_CPU, &niagara2_hsvc_major,
232 	    &niagara2_hsvc_minor);
233 	if ((status != 0) || (niagara2_hsvc_major != NIAGARA2_HSVC_MAJOR)) {
234 		cmn_err(CE_WARN, "hypervisor services not negotiated "
235 		    "or unsupported major number: group: 0x%x major: 0x%lx "
236 		    "minor: 0x%lx errno: %d", HSVC_GROUP_NIAGARA2_CPU,
237 		    niagara2_hsvc_major, niagara2_hsvc_minor, status);
238 		niagara2_hsvc_available = B_FALSE;
239 	}
240 #ifdef N2_1x_CPC_WORKAROUNDS
241 	if (ni2_1x_perf_workarounds)
242 		ni2_events = ni2_1x_events;
243 #endif
244 	/*
245 	 * Construct event list.
246 	 *
247 	 * First pass:  Calculate size needed. We'll need an additional byte
248 	 *		for the NULL pointer during the last strcat.
249 	 *
250 	 * Second pass: Copy strings.
251 	 */
252 	for (evp = ni2_events; evp->name != NULL; evp++)
253 		evlist_sz += strlen(evp->name) + 1;
254 
255 	evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP);
256 	evlist[0] = '\0';
257 
258 	for (evp = ni2_events; evp->name != NULL; evp++) {
259 		(void) strcat(evlist, evp->name);
260 		(void) strcat(evlist, ",");
261 	}
262 	/*
263 	 * Remove trailing comma.
264 	 */
265 	evlist[evlist_sz - 1] = '\0';
266 
267 	return (0);
268 }
269 
270 static uint_t
271 ni2_pcbe_ncounters(void)
272 {
273 	return (2);
274 }
275 
276 static const char *
277 ni2_pcbe_impl_name(void)
278 {
279 	return (ni2_impl_name);
280 }
281 
282 static const char *
283 ni2_pcbe_cpuref(void)
284 {
285 	return (niagara2_cpuref);
286 }
287 
288 static char *
289 ni2_pcbe_list_events(uint_t picnum)
290 {
291 	ASSERT(picnum < cpc_ncounters);
292 
293 	return (evlist);
294 }
295 
296 static char *
297 ni2_pcbe_list_attrs(void)
298 {
299 	if (niagara2_hsvc_available == B_TRUE)
300 		return ("hpriv,emask");
301 	else
302 		return ("emask");
303 }
304 
305 static ni2_event_t *
306 find_event(char *name)
307 {
308 	ni2_event_t	*evp;
309 
310 	for (evp = ni2_events; evp->name != NULL; evp++)
311 		if (strcmp(name, evp->name) == 0)
312 			return (evp);
313 
314 	return (NULL);
315 }
316 
317 /*ARGSUSED*/
318 static uint64_t
319 ni2_pcbe_event_coverage(char *event)
320 {
321 	/*
322 	 * Fortunately, both pic0 and pic1 can count all events.
323 	 */
324 	return (0x3);
325 }
326 
327 #ifdef N2_1x_CPC_WORKAROUNDS
328 uint64_t	ni2_ov_tstamp[NCPU];	/* last overflow time stamp */
329 uint64_t	ni2_ov_spurious_range = 1000000; /* 1 msec at 1GHz */
330 #endif
331 
332 static uint64_t
333 ni2_pcbe_overflow_bitmap(void)
334 {
335 	uint64_t	pcr, overflow;
336 	uint64_t	pic;
337 	uint32_t	pic0, pic1;
338 	boolean_t	update_pic = B_FALSE;
339 
340 	ASSERT(getpil() >= DISP_LEVEL);
341 	pcr = ultra_getpcr();
342 	DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
343 	overflow =  (pcr & CPC_NIAGARA2_PCR_OV0_MASK) >>
344 	    CPC_NIAGARA2_PCR_OV0_SHIFT;
345 	overflow |=  (pcr & CPC_NIAGARA2_PCR_OV1_MASK) >>
346 	    CPC_NIAGARA2_PCR_OV1_SHIFT;
347 
348 	pic = ultra_getpic();
349 	pic0 = (uint32_t)(pic & PIC0_MASK);
350 	pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK);
351 
352 #ifdef N2_1x_CPC_WORKAROUNDS
353 	if (ni2_1x_perf_workarounds) {
354 		uint64_t	tstamp;
355 		processorid_t	cpun;
356 
357 		/*
358 		 * Niagara2 1.x silicon can generate a duplicate overflow
359 		 * trap per event. If we take an overflow trap with no
360 		 * counters overflowing, return a non-zero bitmask with no
361 		 * OV bit set for supported counter so that the framework
362 		 * can ignore this trap.
363 		 */
364 		cpun = CPU->cpu_id;
365 		tstamp = ultra_gettick();
366 		if (overflow)
367 			ni2_ov_tstamp[cpun] = tstamp;
368 		else if (tstamp < (ni2_ov_tstamp[cpun] + ni2_ov_spurious_range))
369 			overflow |= 1ULL << 63;
370 
371 		/*
372 		 * In Niagara2 1.x silicon, PMU doesn't set OV bit for
373 		 * precise events. So, if we take a trap with the counter
374 		 * within the overflow range and the OV bit is not set, we
375 		 * assume OV bit should have been set.
376 		 */
377 		if (PIC_IN_OV_RANGE(pic0))
378 			overflow |= 0x1;
379 		if (PIC_IN_OV_RANGE(pic1))
380 			overflow |= 0x2;
381 	}
382 #endif
383 
384 	pcr |= (CPC_NIAGARA2_PCR_HOLDOV0 | CPC_NIAGARA2_PCR_HOLDOV1);
385 
386 	if (overflow & 0x1) {
387 		pcr &= ~(CPC_NIAGARA2_PCR_OV0_MASK |
388 		    CPC_NIAGARA2_PCR_HOLDOV0);
389 		if (PIC_IN_OV_RANGE(pic0)) {
390 			pic0 = 0;
391 			update_pic = B_TRUE;
392 		}
393 	}
394 
395 	if (overflow & 0x2) {
396 		pcr &= ~(CPC_NIAGARA2_PCR_OV1_MASK |
397 		    CPC_NIAGARA2_PCR_HOLDOV1);
398 		if (PIC_IN_OV_RANGE(pic1)) {
399 			pic1 = 0;
400 			update_pic = B_TRUE;
401 		}
402 	}
403 
404 	if (update_pic)
405 		ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0);
406 
407 	/*
408 	 * The HV interface does not need to be used here because we are
409 	 * only resetting the OV bits and do not need to set the HT bit.
410 	 */
411 	DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
412 	ultra_setpcr(pcr);
413 
414 	return (overflow);
415 }
416 
417 /*ARGSUSED*/
418 static int
419 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
420     uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
421 {
422 	ni2_pcbe_config_t	*cfg;
423 	ni2_pcbe_config_t	*other_config;
424 	ni2_event_t		*evp;
425 	int			i;
426 	uint32_t		evsel;
427 
428 	/*
429 	 * If we've been handed an existing configuration, we need only preset
430 	 * the counter value.
431 	 */
432 	if (*data != NULL) {
433 		cfg = *data;
434 		cfg->pcbe_pic = (uint32_t)preset;
435 		return (0);
436 	}
437 
438 	if (picnum > 1)
439 		return (CPC_INVALID_PICNUM);
440 
441 	if ((evp = find_event(event)) == NULL)
442 		return (CPC_INVALID_EVENT);
443 
444 	evsel = evp->emask;
445 
446 	for (i = 0; i < nattrs; i++) {
447 		if (strcmp(attrs[i].ka_name, "hpriv") == 0) {
448 			if (attrs[i].ka_val != 0)
449 				flags |= CPC_COUNT_HV;
450 		} else if (strcmp(attrs[i].ka_name, "emask") == 0) {
451 			if ((attrs[i].ka_val | evp->emask_valid) !=
452 			    evp->emask_valid)
453 				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
454 			evsel |= attrs[i].ka_val;
455 		} else
456 			return (CPC_INVALID_ATTRIBUTE);
457 	}
458 
459 	/*
460 	 * Find other requests that will be programmed with this one, and ensure
461 	 * the flags don't conflict.
462 	 */
463 	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
464 	    (other_config->pcbe_flags != flags))
465 		return (CPC_CONFLICTING_REQS);
466 
467 	/*
468 	 * If the hpriv attribute is present, make sure we have
469 	 * access to hyperprivileged events before continuing with
470 	 * this configuration. If we can set the ht bit in the PCR
471 	 * successfully, we must have access to hyperprivileged
472 	 * events.
473 	 *
474 	 * If this is a static per-CPU configuration, the CPC
475 	 * driver ensures there can not be more than one for this
476 	 * CPU. If this is a per-LWP configuration, the driver
477 	 * ensures no static per-CPU counting is ongoing and that
478 	 * the target LWP is not already being monitored.
479 	 */
480 	if (flags & CPC_COUNT_HV) {
481 		kpreempt_disable();
482 
483 		DTRACE_PROBE1(niagara2__setpcr, uint64_t,
484 		    allstopped | CPC_NIAGARA2_PCR_HT);
485 		if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL,
486 		    allstopped | CPC_NIAGARA2_PCR_HT) != H_EOK) {
487 			kpreempt_enable();
488 			return (CPC_HV_NO_ACCESS);
489 		}
490 
491 		DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped);
492 		(void) hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, allstopped);
493 
494 		kpreempt_enable();
495 	}
496 
497 	cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP);
498 
499 	cfg->pcbe_picno = picnum;
500 	cfg->pcbe_evsel = evsel;
501 	cfg->pcbe_flags = flags;
502 	cfg->pcbe_pic = (uint32_t)preset;
503 
504 	*data = cfg;
505 	return (0);
506 }
507 
508 static void
509 ni2_pcbe_program(void *token)
510 {
511 	ni2_pcbe_config_t	*pic0;
512 	ni2_pcbe_config_t	*pic1;
513 	ni2_pcbe_config_t	*tmp;
514 	ni2_pcbe_config_t	nullcfg = { 1, 0, 0, 0 };
515 	uint64_t		pcr;
516 	uint64_t		curpic;
517 	uint64_t		toe;
518 
519 	/* enable trap-on-event for pic0 and pic1 */
520 	toe = (CPC_NIAGARA2_PCR_TOE0 | CPC_NIAGARA2_PCR_TOE1);
521 
522 	if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) ==
523 	    NULL)
524 		panic("ni2_pcbe: token %p has no configs", token);
525 
526 	if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) {
527 		pic1 = &nullcfg;
528 		nullcfg.pcbe_flags = pic0->pcbe_flags;
529 		toe = CPC_NIAGARA2_PCR_TOE0; /* enable trap-on-event for pic0 */
530 	}
531 
532 	if (pic0->pcbe_picno != 0) {
533 		/*
534 		 * pic0 is counter 1, so if we need the null config it should
535 		 * be counter 0.
536 		 */
537 		nullcfg.pcbe_picno = 0;
538 		tmp = pic0;
539 		pic0 = pic1;
540 		pic1 = tmp;
541 		toe = CPC_NIAGARA2_PCR_TOE1; /* enable trap-on-event for pic1 */
542 	}
543 
544 	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
545 		panic("%s: bad config on token %p\n", ni2_impl_name, token);
546 
547 	/*
548 	 * UltraSPARC does not allow pic0 to be configured differently
549 	 * from pic1. If the flags on these two configurations are
550 	 * different, they are incompatible. This condition should be
551 	 * caught at configure time.
552 	 */
553 	ASSERT(pic0->pcbe_flags == pic1->pcbe_flags);
554 
555 	ni2_pcbe_allstop();
556 
557 	ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) |
558 	    (uint64_t)pic0->pcbe_pic);
559 
560 	pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_NIAGARA2_PCR_PIC0_SHIFT;
561 	pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) <<
562 	    CPC_NIAGARA2_PCR_PIC1_SHIFT;
563 
564 	if (pic0->pcbe_flags & CPC_COUNT_USER)
565 		pcr |= (1ull << CPC_NIAGARA2_PCR_UT_SHIFT);
566 	if (pic0->pcbe_flags & CPC_COUNT_SYSTEM)
567 		pcr |= (1ull << CPC_NIAGARA2_PCR_ST_SHIFT);
568 	if (pic0->pcbe_flags & CPC_COUNT_HV)
569 		pcr |= (1ull << CPC_NIAGARA2_PCR_HT_SHIFT);
570 	pcr |= toe;
571 
572 	DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
573 
574 	if (pic0->pcbe_flags & CPC_COUNT_HV) {
575 		/*
576 		 * The ht bit in the PCR is only writable in
577 		 * hyperprivileged mode. So if we are counting
578 		 * hpriv events, we must use the HV interface
579 		 * hv_niagara_setperf to set the PCR. If this
580 		 * fails, assume we no longer have access to
581 		 * hpriv events.
582 		 */
583 		if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, pcr) != H_EOK) {
584 			kcpc_invalidate_config(token);
585 			return;
586 		}
587 	} else
588 		/* Set the PCR with no hpriv event counting enabled. */
589 		ultra_setpcr(pcr);
590 
591 	ni2_cpc_counting[CPU->cpu_id] = B_TRUE;
592 
593 	/*
594 	 * On UltraSPARC, only read-to-read counts are accurate. We cannot
595 	 * expect the value we wrote into the PIC, above, to be there after
596 	 * starting the counter. We must sample the counter value now and use
597 	 * that as the baseline for future samples.
598 	 */
599 	curpic = ultra_getpic();
600 	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
601 	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
602 
603 	DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic);
604 }
605 
606 static void
607 ni2_pcbe_allstop(void)
608 {
609 	/*
610 	 * We use the HV interface here because if we were counting
611 	 * hyperprivileged events, we must reset the PCR.ht bit to stop
612 	 * the counting. In the event that this HV call fails, we fall
613 	 * back on ultra_setpcr which does not have write access to the
614 	 * ht bit.
615 	 */
616 	if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, allstopped) != H_EOK)
617 		ultra_setpcr(allstopped);
618 
619 	ni2_cpc_counting[CPU->cpu_id] = B_FALSE;
620 }
621 
622 static void
623 ni2_pcbe_sample(void *token)
624 {
625 	uint64_t		curpic;
626 	int64_t			diff;
627 	uint64_t		*pic0_data;
628 	uint64_t		*pic1_data;
629 	uint64_t		*dtmp;
630 	uint64_t		tmp;
631 	uint64_t		pcr;
632 	ni2_pcbe_config_t	*pic0;
633 	ni2_pcbe_config_t	*pic1;
634 	ni2_pcbe_config_t	nullcfg = { 1, 0, 0, 0 };
635 	ni2_pcbe_config_t	*ctmp;
636 
637 	curpic = ultra_getpic();
638 	DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic);
639 
640 	if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
641 		panic("%s: token %p has no configs", ni2_impl_name, token);
642 
643 	if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
644 		pic1 = &nullcfg;
645 		pic1_data = &tmp;
646 	}
647 
648 	if (pic0->pcbe_picno != 0) {
649 		nullcfg.pcbe_picno = 0;
650 		ctmp = pic0;
651 		pic0 = pic1;
652 		pic1 = ctmp;
653 		dtmp = pic0_data;
654 		pic0_data = pic1_data;
655 		pic1_data = dtmp;
656 	}
657 
658 	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
659 		panic("%s: bad config on token %p\n", ni2_impl_name, token);
660 
661 
662 	if (pic0->pcbe_flags & CPC_COUNT_HV) {
663 		/*
664 		 * If the hpriv attribute is present, but the HT bit
665 		 * is not set in the PCR, access to hyperprivileged
666 		 * events must have been revoked. Only perform this
667 		 * check if counting is not stopped.
668 		 */
669 #ifdef N2_1x_CPC_WORKAROUNDS
670 		if (!ni2_1x_perf_workarounds) {
671 #endif
672 			pcr = ultra_getpcr();
673 			DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
674 			if (ni2_cpc_counting[CPU->cpu_id] &&
675 			    !(pcr & CPC_NIAGARA2_PCR_HT)) {
676 				kcpc_invalidate_config(token);
677 				return;
678 			}
679 #ifdef N2_1x_CPC_WORKAROUNDS
680 		}
681 #endif
682 	}
683 
684 	diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic;
685 	if (diff < 0)
686 		diff += (1ll << 32);
687 	*pic0_data += diff;
688 
689 	diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic;
690 	if (diff < 0)
691 		diff += (1ll << 32);
692 	*pic1_data += diff;
693 
694 	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
695 	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
696 }
697 
698 static void
699 ni2_pcbe_free(void *config)
700 {
701 	kmem_free(config, sizeof (ni2_pcbe_config_t));
702 }
703 
704 
705 static struct modlpcbe modlpcbe = {
706 	&mod_pcbeops,
707 	"UltraSPARC T2 Performance Counters v%I%",
708 	&ni2_pcbe_ops
709 };
710 
711 static struct modlinkage modl = {
712 	MODREV_1,
713 	&modlpcbe,
714 };
715 
716 int
717 _init(void)
718 {
719 	if (ni2_pcbe_init() != 0)
720 		return (ENOTSUP);
721 	return (mod_install(&modl));
722 }
723 
724 int
725 _fini(void)
726 {
727 	return (mod_remove(&modl));
728 }
729 
730 int
731 _info(struct modinfo *mi)
732 {
733 	return (mod_info(&modl, mi));
734 }
735