xref: /titanic_41/usr/src/uts/sun4v/pcbe/niagara2_pcbe.c (revision 6a634c9dca3093f3922e4b7ab826d7bdf17bf78e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /*
26  * This file contains preset event names from the Performance Application
27  * Programming Interface v3.5 which included the following notice:
28  *
29  *                             Copyright (c) 2005,6
30  *                           Innovative Computing Labs
31  *                         Computer Science Department,
32  *                            University of Tennessee,
33  *                                 Knoxville, TN.
34  *                              All Rights Reserved.
35  *
36  *
37  * Redistribution and use in source and binary forms, with or without
38  * modification, are permitted provided that the following conditions are met:
39  *
40  *    * Redistributions of source code must retain the above copyright notice,
41  *      this list of conditions and the following disclaimer.
42  *    * Redistributions in binary form must reproduce the above copyright
43  *      notice, this list of conditions and the following disclaimer in the
44  *      documentation and/or other materials provided with the distribution.
45  *    * Neither the name of the University of Tennessee nor the names of its
46  *      contributors may be used to endorse or promote products derived from
47  *      this software without specific prior written permission.
48  *
49  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
50  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
53  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
54  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
55  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
56  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
57  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
58  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
59  * POSSIBILITY OF SUCH DAMAGE.
60  *
61  *
62  * This open source software license conforms to the BSD License template.
63  */
64 
65 /*
66  * Niagara2 Performance Counter Backend
67  */
68 
69 #include <sys/cpuvar.h>
70 #include <sys/systm.h>
71 #include <sys/archsystm.h>
72 #include <sys/cmn_err.h>
73 #include <sys/cpc_impl.h>
74 #include <sys/cpc_pcbe.h>
75 #include <sys/modctl.h>
76 #include <sys/machsystm.h>
77 #include <sys/sdt.h>
78 #include <sys/niagara2regs.h>
79 #include <sys/hsvc.h>
80 #include <sys/hypervisor_api.h>
81 #include <sys/disp.h>
82 
83 /*LINTLIBRARY*/
84 static int ni2_pcbe_init(void);
85 static uint_t ni2_pcbe_ncounters(void);
86 static const char *ni2_pcbe_impl_name(void);
87 static const char *ni2_pcbe_cpuref(void);
88 static char *ni2_pcbe_list_events(uint_t picnum);
89 static char *ni2_pcbe_list_attrs(void);
90 static uint64_t ni2_pcbe_event_coverage(char *event);
91 static uint64_t ni2_pcbe_overflow_bitmap(void);
92 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
93     uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
94     void *token);
95 static void ni2_pcbe_program(void *token);
96 static void ni2_pcbe_allstop(void);
97 static void ni2_pcbe_sample(void *token);
98 static void ni2_pcbe_free(void *config);
99 
100 extern void ultra_setpcr(uint64_t);
101 extern uint64_t ultra_getpcr(void);
102 extern void ultra_setpic(uint64_t);
103 extern uint64_t ultra_getpic(void);
104 extern uint64_t ultra_gettick(void);
105 extern char cpu_module_name[];
106 
107 #define	L2_dmiss_ld_event	0x320
108 int pcbe_l2dmiss_ovtrap_enable = 0;
109 
110 pcbe_ops_t ni2_pcbe_ops = {
111 	PCBE_VER_1,
112 	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,
113 	ni2_pcbe_ncounters,
114 	ni2_pcbe_impl_name,
115 	ni2_pcbe_cpuref,
116 	ni2_pcbe_list_events,
117 	ni2_pcbe_list_attrs,
118 	ni2_pcbe_event_coverage,
119 	ni2_pcbe_overflow_bitmap,
120 	ni2_pcbe_configure,
121 	ni2_pcbe_program,
122 	ni2_pcbe_allstop,
123 	ni2_pcbe_sample,
124 	ni2_pcbe_free
125 };
126 
127 typedef struct _ni2_pcbe_config {
128 	uint_t		pcbe_picno;	/* 0 for pic0 or 1 for pic1 */
129 	uint32_t	pcbe_evsel;	/* %pcr event code unshifted */
130 	uint32_t	pcbe_flags;	/* hpriv/user/system/priv */
131 	uint32_t	pcbe_pic;	/* unshifted raw %pic value */
132 } ni2_pcbe_config_t;
133 
134 typedef struct _ni2_event {
135 	const char	*name;
136 	const uint32_t	emask;
137 	const uint32_t	emask_valid;	/* Mask of unreserved MASK bits */
138 } ni2_event_t;
139 
140 typedef struct _ni2_generic_event {
141 	char *name;
142 	char *event;
143 } ni2_generic_event_t;
144 
145 #define	ULTRA_PCR_PRIVPIC	(UINT64_C(1) << CPC_PCR_PRIV_SHIFT)
146 #define	EV_END {NULL, 0, 0}
147 #define	GEN_EV_END {NULL, NULL}
148 
149 static const uint64_t	allstopped = (ULTRA_PCR_PRIVPIC |
150 	CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1);
151 
152 /*
153  * We update this array in the program and allstop routine. The array
154  * is checked in the sample routine to allow us to only perform the
155  * PCR.ht bit check when counting is in progress.
156  */
157 static boolean_t ni2_cpc_counting[NCPU];
158 
159 static ni2_event_t ni2_events[] = {
160 	{ "Idle_strands",			0x000, 0x00 },
161 	{ "Br_completed",			0x201, 0xff },
162 	{ "Br_taken",				0x202, 0xff },
163 	{ "Instr_FGU_arithmetic",		0x204, 0xff },
164 	{ "Instr_ld",				0x208, 0xff },
165 	{ "Instr_st",				0x210, 0xff },
166 	{ "Instr_sw",				0x220, 0xff },
167 	{ "Instr_other",			0x240, 0xff },
168 	{ "Atomics",				0x280, 0xff },
169 	{ "Instr_cnt",				0x2fd, 0xff },
170 	{ "IC_miss",				0x301, 0x33 },
171 	{ "DC_miss",				0x302, 0x33 },
172 	{ "L2_imiss",				0x310, 0x33 },
173 	{ "L2_dmiss_ld",			0x320, 0x33 },
174 	{ "ITLB_HWTW_ref_L2",			0x404, 0x3c },
175 	{ "DTLB_HWTW_ref_L2",			0x408, 0x3c },
176 	{ "ITLB_HWTW_miss_L2",			0x410, 0x3c },
177 	{ "DTLB_HWTW_miss_L2",			0x420, 0x3c },
178 	{ "Stream_ld_to_PCX",			0x501, 0x3f },
179 	{ "Stream_st_to_PCX",			0x502, 0x3f },
180 	{ "CPU_ld_to_PCX",			0x504, 0x3f },
181 	{ "CPU_ifetch_to_PCX",			0x508, 0x3f },
182 	{ "CPU_st_to_PCX",			0x510, 0x3f },
183 	{ "MMU_ld_to_PCX",			0x520, 0x3f },
184 #ifdef KT_IMPL
185 	{ "DES_3DES_op",			0x601, 0xff },
186 	{ "AES_op",				0x602, 0xff },
187 	{ "Kasumi_op",				0x604, 0xff },
188 	{ "MD5_SHA-1_SHA-256_op",		0x608, 0xff },
189 	{ "MA_op",				0x610, 0xff },
190 	{ "CRC_TCPIP_cksum",			0x620, 0xff },
191 	{ "DES_3DES_busy_cycle",		0x701, 0xff },
192 	{ "AES_busy_cycle",			0x702, 0xff },
193 	{ "Kasumi_busy_cycle",			0x704, 0xff },
194 	{ "MD5_SHA-1_SHA-256_busy_cycle",	0x708, 0xff },
195 	{ "MA_busy_cycle",			0x710, 0xff },
196 	{ "CRC_MPA_cksum",			0x720, 0xff },
197 #else
198 	{ "DES_3DES_op",			0x601, 0x3f },
199 	{ "AES_op",				0x602, 0x3f },
200 	{ "RC4_op",				0x604, 0x3f },
201 	{ "MD5_SHA-1_SHA-256_op",		0x608, 0x3f },
202 	{ "MA_op",				0x610, 0x3f },
203 	{ "CRC_TCPIP_cksum",			0x620, 0x3f },
204 	{ "DES_3DES_busy_cycle",		0x701, 0x3f },
205 	{ "AES_busy_cycle",			0x702, 0x3f },
206 	{ "RC4_busy_cycle",			0x704, 0x3f },
207 	{ "MD5_SHA-1_SHA-256_busy_cycle",	0x708, 0x3f },
208 	{ "MA_busy_cycle",			0x710, 0x3f },
209 	{ "CRC_MPA_cksum",			0x720, 0x3f },
210 #endif
211 	{ "ITLB_miss",				0xb04, 0x0c },
212 	{ "DTLB_miss",				0xb08, 0x0c },
213 	{ "TLB_miss",				0xb0c, 0x0c },
214 	EV_END
215 };
216 
217 static ni2_generic_event_t ni2_generic_events[] = {
218 	{ "PAPI_tot_ins",	"Instr_cnt" },
219 	{ "PAPI_l1_dcm",	"DC_miss" },
220 	{ "PAPI_l1_icm",	"IC_miss" },
221 	{ "PAPI_l2_icm",	"L2_imiss" },
222 	{ "PAPI_l2_ldm",	"L2_dmiss_ld" },
223 	{ "PAPI_tlb_dm",	"DTLB_miss" },
224 	{ "PAPI_tlb_im",	"ITLB_miss" },
225 	{ "PAPI_tlb_tm",	"TLB_miss" },
226 	{ "PAPI_br_tkn",	"Br_taken" },
227 	{ "PAPI_br_ins",	"Br_completed" },
228 	{ "PAPI_ld_ins",	"Instr_ld" },
229 	{ "PAPI_sr_ins",	"Instr_st" },
230 	{ "PAPI_fp_ops",	"Instr_FGU_arithmetic" },
231 	{ "PAPI_fp_ins",	"Instr_FGU_arithmetic" },
232 	GEN_EV_END
233 };
234 
235 static char		*evlist;
236 static size_t		evlist_sz;
237 static uint16_t 	pcr_pic0_mask;
238 static uint16_t 	pcr_pic1_mask;
239 
240 #define	CPU_REF_URL " Documentation for Sun processors can be found at: " \
241 			"http://www.sun.com/processors/manuals"
242 
243 #if defined(NIAGARA2_IMPL)
244 static const char	*cpu_impl_name = "UltraSPARC T2";
245 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2 User's Manual\" "
246 			"for descriptions of these events." CPU_REF_URL;
247 #elif defined(VFALLS_IMPL)
248 static const char	*cpu_impl_name = "UltraSPARC T2+";
249 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2+ User's Manual\" "
250 			"for descriptions of these events." CPU_REF_URL;
251 #elif defined(KT_IMPL)
252 static const char	*cpu_impl_name = "SPARC T3";
253 static const char *cpu_pcbe_ref = "See the \"SPARC T3 User's Manual\" "
254 			"for descriptions of these events." CPU_REF_URL;
255 #endif
256 
257 static boolean_t cpu_hsvc_available = B_TRUE;
258 
259 static int
ni2_pcbe_init(void)260 ni2_pcbe_init(void)
261 {
262 	ni2_event_t		*evp;
263 	ni2_generic_event_t	*gevp;
264 	int			status;
265 	uint64_t		cpu_hsvc_major;
266 	uint64_t		cpu_hsvc_minor;
267 #if defined(NIAGARA2_IMPL)
268 	uint64_t		hsvc_cpu_group = HSVC_GROUP_NIAGARA2_CPU;
269 	uint64_t		hsvc_cpu_major = NIAGARA2_HSVC_MAJOR;
270 #elif defined(VFALLS_IMPL)
271 	uint64_t		hsvc_cpu_group = HSVC_GROUP_VFALLS_CPU;
272 	uint64_t		hsvc_cpu_major = VFALLS_HSVC_MAJOR;
273 #elif defined(KT_IMPL)
274 	uint64_t	hsvc_cpu_group = HSVC_GROUP_KT_CPU;
275 	uint64_t	hsvc_cpu_major = KT_HSVC_MAJOR;
276 #endif
277 
278 	pcr_pic0_mask = CPC_PCR_PIC0_MASK;
279 	pcr_pic1_mask = CPC_PCR_PIC1_MASK;
280 
281 	/*
282 	 * Validate API version for Niagara2 specific hypervisor services
283 	 */
284 	status = hsvc_version(hsvc_cpu_group, &cpu_hsvc_major,
285 	    &cpu_hsvc_minor);
286 	if ((status != 0) || (cpu_hsvc_major != hsvc_cpu_major)) {
287 		cmn_err(CE_WARN, "hypervisor services not negotiated "
288 		    "or unsupported major number: group: 0x%lx major: 0x%lx "
289 		    "minor: 0x%lx errno: %d", hsvc_cpu_group,
290 		    cpu_hsvc_major, cpu_hsvc_minor, status);
291 		cpu_hsvc_available = B_FALSE;
292 	}
293 
294 	/*
295 	 * Construct event list.
296 	 *
297 	 * First pass:  Calculate size needed. We'll need an additional byte
298 	 *		for the NULL pointer during the last strcat.
299 	 *
300 	 * Second pass: Copy strings.
301 	 */
302 	for (evp = ni2_events; evp->name != NULL; evp++)
303 		evlist_sz += strlen(evp->name) + 1;
304 
305 	for (gevp = ni2_generic_events; gevp->name != NULL; gevp++)
306 		evlist_sz += strlen(gevp->name) + 1;
307 
308 	evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP);
309 	evlist[0] = '\0';
310 
311 	for (evp = ni2_events; evp->name != NULL; evp++) {
312 		(void) strcat(evlist, evp->name);
313 		(void) strcat(evlist, ",");
314 	}
315 
316 	for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) {
317 		(void) strcat(evlist, gevp->name);
318 		(void) strcat(evlist, ",");
319 	}
320 
321 	/*
322 	 * Remove trailing comma.
323 	 */
324 	evlist[evlist_sz - 1] = '\0';
325 
326 	return (0);
327 }
328 
329 static uint_t
ni2_pcbe_ncounters(void)330 ni2_pcbe_ncounters(void)
331 {
332 	return (2);
333 }
334 
335 static const char *
ni2_pcbe_impl_name(void)336 ni2_pcbe_impl_name(void)
337 {
338 	return (cpu_impl_name);
339 }
340 
341 static const char *
ni2_pcbe_cpuref(void)342 ni2_pcbe_cpuref(void)
343 {
344 	return (cpu_pcbe_ref);
345 }
346 
347 static char *
ni2_pcbe_list_events(uint_t picnum)348 ni2_pcbe_list_events(uint_t picnum)
349 {
350 	ASSERT(picnum < cpc_ncounters);
351 
352 	return (evlist);
353 }
354 
355 static char *
ni2_pcbe_list_attrs(void)356 ni2_pcbe_list_attrs(void)
357 {
358 	if (cpu_hsvc_available == B_TRUE)
359 #if defined(NIAGARA2_IMPL)
360 		return ("hpriv,emask");
361 #elif defined(VFALLS_IMPL)
362 		return ("hpriv,l2ctl,emask");
363 #elif defined(KT_IMPL)
364 		return ("hpriv,l2ctl,emask,sample");
365 #endif
366 	else
367 #if defined(KT_IMPL)
368 		return ("emask,sample");
369 #else
370 		return ("emask");
371 #endif
372 }
373 
374 static ni2_generic_event_t *
find_generic_event(char * name)375 find_generic_event(char *name)
376 {
377 	ni2_generic_event_t	*gevp;
378 
379 	for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) {
380 		if (strcmp(name, gevp->name) == 0)
381 			return (gevp);
382 	}
383 
384 	return (NULL);
385 }
386 
387 static ni2_event_t *
find_event(char * name)388 find_event(char *name)
389 {
390 	ni2_event_t		*evp;
391 
392 	for (evp = ni2_events; evp->name != NULL; evp++)
393 		if (strcmp(name, evp->name) == 0)
394 			return (evp);
395 
396 	return (NULL);
397 }
398 
399 /*ARGSUSED*/
400 static uint64_t
ni2_pcbe_event_coverage(char * event)401 ni2_pcbe_event_coverage(char *event)
402 {
403 	/*
404 	 * Check whether counter event is supported
405 	 */
406 	if (find_event(event) == NULL && find_generic_event(event) == NULL)
407 		return (0);
408 
409 	/*
410 	 * Fortunately, both pic0 and pic1 can count all events.
411 	 */
412 	return (0x3);
413 }
414 
415 static uint64_t
ni2_pcbe_overflow_bitmap(void)416 ni2_pcbe_overflow_bitmap(void)
417 {
418 	uint64_t	pcr, overflow;
419 	uint64_t	pic;
420 	uint32_t	pic0, pic1;
421 	boolean_t	update_pic = B_FALSE;
422 	boolean_t	pic_inrange = B_FALSE;
423 
424 	ASSERT(getpil() >= DISP_LEVEL);
425 	pcr = ultra_getpcr();
426 	DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
427 	overflow =  (pcr & CPC_PCR_OV0_MASK) >>
428 	    CPC_PCR_OV0_SHIFT;
429 	overflow |=  (pcr & CPC_PCR_OV1_MASK) >>
430 	    CPC_PCR_OV1_SHIFT;
431 
432 	pic = ultra_getpic();
433 	pic0 = (uint32_t)(pic & PIC0_MASK);
434 	pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK);
435 
436 	pcr |= (CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1);
437 
438 	if (overflow & 0x1) {
439 		pcr &= ~(CPC_PCR_OV0_MASK |
440 		    CPC_PCR_HOLDOV0);
441 		pic_inrange = PIC_IN_OV_RANGE(pic0);
442 #if defined(KT_IMPL)
443 		if (pcr & CPC_PCR_SAMPLE_MODE_MASK)
444 			pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic0);
445 #endif
446 		if (pic_inrange) {
447 			pic0 = 0;
448 			update_pic = B_TRUE;
449 		}
450 	}
451 
452 	if (overflow & 0x2) {
453 		pcr &= ~(CPC_PCR_OV1_MASK |
454 		    CPC_PCR_HOLDOV1);
455 		pic_inrange = PIC_IN_OV_RANGE(pic1);
456 #if defined(KT_IMPL)
457 		if (pcr & CPC_PCR_SAMPLE_MODE_MASK)
458 			pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic1);
459 #endif
460 		if (pic_inrange) {
461 			pic1 = 0;
462 			update_pic = B_TRUE;
463 		}
464 	}
465 
466 	if (update_pic)
467 		ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0);
468 
469 	/*
470 	 * The HV interface does not need to be used here because we are
471 	 * only resetting the OV bits and do not need to set the HT bit.
472 	 */
473 	DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
474 	ultra_setpcr(pcr);
475 
476 	return (overflow);
477 }
478 
479 /*ARGSUSED*/
480 static int
ni2_pcbe_configure(uint_t picnum,char * event,uint64_t preset,uint32_t flags,uint_t nattrs,kcpc_attr_t * attrs,void ** data,void * token)481 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
482     uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
483 {
484 	ni2_pcbe_config_t	*cfg;
485 	ni2_pcbe_config_t	*other_config;
486 	ni2_event_t		*evp;
487 	ni2_generic_event_t	*gevp;
488 	int			i;
489 	uint32_t		evsel;
490 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
491 	uint64_t		l2ctl = 0;
492 #endif
493 
494 	/*
495 	 * If we've been handed an existing configuration, we need only preset
496 	 * the counter value.
497 	 */
498 	if (*data != NULL) {
499 		cfg = *data;
500 		cfg->pcbe_pic = (uint32_t)preset;
501 		return (0);
502 	}
503 
504 	if (picnum > 1)
505 		return (CPC_INVALID_PICNUM);
506 
507 
508 	if ((evp = find_event(event)) == NULL) {
509 		if ((gevp = find_generic_event(event)) != NULL) {
510 			evp = find_event(gevp->event);
511 			ASSERT(evp != NULL);
512 
513 			if (nattrs > 0)
514 				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
515 		} else {
516 			return (CPC_INVALID_EVENT);
517 		}
518 	}
519 
520 	evsel = evp->emask;
521 
522 	for (i = 0; i < nattrs; i++) {
523 		if (strcmp(attrs[i].ka_name, "hpriv") == 0) {
524 			if (attrs[i].ka_val != 0)
525 				flags |= CPC_COUNT_HV;
526 		} else if (strcmp(attrs[i].ka_name, "emask") == 0) {
527 			if ((attrs[i].ka_val | evp->emask_valid) !=
528 			    evp->emask_valid)
529 				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
530 			evsel |= attrs[i].ka_val;
531 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
532 		} else if (strcmp(attrs[i].ka_name, "l2ctl") == 0) {
533 			if ((attrs[i].ka_val | L2_CTL_MASK) !=
534 			    L2_CTL_MASK)
535 				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
536 			else
537 				l2ctl = attrs[i].ka_val;
538 #endif
539 #if defined(KT_IMPL)
540 		} else if (strcmp(attrs[i].ka_name, "sample") == 0) {
541 			if (attrs[i].ka_val != 0)
542 				flags |= CPC_COUNT_SAMPLE_MODE;
543 #endif
544 		} else
545 			return (CPC_INVALID_ATTRIBUTE);
546 	}
547 
548 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
549 	/*
550 	 * Set PERF_CONTROL bits in L2_CONTROL_REG only when events have
551 	 * SL bits equal to 3.
552 	 */
553 	if ((evsel & SL_MASK) == SL3_MASK) {
554 		if ((hv_niagara_setperf(HV_L2_CTL, l2ctl)) != 0)
555 			return (CPC_HV_NO_ACCESS);
556 	}
557 #endif
558 
559 	/*
560 	 * Find other requests that will be programmed with this one, and ensure
561 	 * the flags don't conflict.
562 	 */
563 	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
564 	    (other_config->pcbe_flags != flags))
565 		return (CPC_CONFLICTING_REQS);
566 
567 	/*
568 	 * If the hpriv attribute is present, make sure we have
569 	 * access to hyperprivileged events before continuing with
570 	 * this configuration. If we can set the ht bit in the PCR
571 	 * successfully, we must have access to hyperprivileged
572 	 * events.
573 	 *
574 	 * If this is a static per-CPU configuration, the CPC
575 	 * driver ensures there can not be more than one for this
576 	 * CPU. If this is a per-LWP configuration, the driver
577 	 * ensures no static per-CPU counting is ongoing and that
578 	 * the target LWP is not already being monitored.
579 	 */
580 	if (flags & CPC_COUNT_HV) {
581 		kpreempt_disable();
582 
583 		DTRACE_PROBE1(niagara2__setpcr, uint64_t,
584 		    allstopped | CPC_PCR_HT);
585 		if (hv_niagara_setperf(HV_SPARC_CTL,
586 		    allstopped | CPC_PCR_HT) != H_EOK) {
587 			kpreempt_enable();
588 			return (CPC_HV_NO_ACCESS);
589 		}
590 
591 		DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped);
592 		(void) hv_niagara_setperf(HV_SPARC_CTL, allstopped);
593 
594 		kpreempt_enable();
595 	}
596 
597 	cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP);
598 
599 	cfg->pcbe_picno = picnum;
600 	cfg->pcbe_evsel = evsel;
601 	cfg->pcbe_flags = flags;
602 	cfg->pcbe_pic = (uint32_t)preset;
603 
604 	*data = cfg;
605 	return (0);
606 }
607 
608 static void
ni2_pcbe_program(void * token)609 ni2_pcbe_program(void *token)
610 {
611 	ni2_pcbe_config_t	*pic0;
612 	ni2_pcbe_config_t	*pic1;
613 	ni2_pcbe_config_t	*tmp;
614 	ni2_pcbe_config_t	nullcfg = { 1, 0, 0, 0 };
615 	uint64_t		pcr;
616 	uint64_t		curpic;
617 	uint64_t		toe;
618 
619 	/* enable trap-on-event for pic0 and pic1 */
620 	toe = (CPC_PCR_TOE0 | CPC_PCR_TOE1);
621 
622 	if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) ==
623 	    NULL)
624 		panic("ni2_pcbe: token %p has no configs", token);
625 
626 	if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) {
627 		pic1 = &nullcfg;
628 		nullcfg.pcbe_flags = pic0->pcbe_flags;
629 		toe = CPC_PCR_TOE0; /* enable trap-on-event for pic0 */
630 	}
631 
632 	if (pic0->pcbe_picno != 0) {
633 		/*
634 		 * pic0 is counter 1, so if we need the null config it should
635 		 * be counter 0.
636 		 */
637 		nullcfg.pcbe_picno = 0;
638 		tmp = pic0;
639 		pic0 = pic1;
640 		pic1 = tmp;
641 		toe = CPC_PCR_TOE1; /* enable trap-on-event for pic1 */
642 	}
643 
644 	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
645 		panic("%s: bad config on token %p\n", cpu_impl_name, token);
646 
647 	/*
648 	 * UltraSPARC does not allow pic0 to be configured differently
649 	 * from pic1. If the flags on these two configurations are
650 	 * different, they are incompatible. This condition should be
651 	 * caught at configure time.
652 	 */
653 	ASSERT(pic0->pcbe_flags == pic1->pcbe_flags);
654 
655 	ni2_pcbe_allstop();
656 
657 	ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) |
658 	    (uint64_t)pic0->pcbe_pic);
659 
660 	pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_PCR_PIC0_SHIFT;
661 	pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) <<
662 	    CPC_PCR_PIC1_SHIFT;
663 
664 	if (pic0->pcbe_flags & CPC_COUNT_USER)
665 		pcr |= (1ull << CPC_PCR_UT_SHIFT);
666 	if (pic0->pcbe_flags & CPC_COUNT_SYSTEM)
667 		pcr |= (1ull << CPC_PCR_ST_SHIFT);
668 	if (pic0->pcbe_flags & CPC_COUNT_HV)
669 		pcr |= (1ull << CPC_PCR_HT_SHIFT);
670 #if defined(KT_IMPL)
671 	if (pic0->pcbe_flags & CPC_COUNT_SAMPLE_MODE)
672 		pcr |= (1ull << CPC_PCR_SAMPLE_MODE_SHIFT);
673 #endif
674 
675 	if (pcbe_l2dmiss_ovtrap_enable == 0) {
676 		/*
677 		 * SW workaround for HW Erratum 108.
678 		 * Disable overflow interrupts when L2_dmiss_ld event is
679 		 * selected.
680 		 */
681 		if ((pic0->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event)
682 			toe &= ~CPC_PCR_TOE0;
683 
684 		if ((pic1->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event)
685 			toe &= ~CPC_PCR_TOE1;
686 	}
687 	pcr |= toe;
688 
689 	DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
690 
691 	if (pic0->pcbe_flags & CPC_COUNT_HV) {
692 		/*
693 		 * The ht bit in the PCR is only writable in
694 		 * hyperprivileged mode. So if we are counting
695 		 * hpriv events, we must use the HV interface
696 		 * hv_niagara_setperf to set the PCR. If this
697 		 * fails, assume we no longer have access to
698 		 * hpriv events.
699 		 */
700 		if (hv_niagara_setperf(HV_SPARC_CTL, pcr) != H_EOK) {
701 			kcpc_invalidate_config(token);
702 			return;
703 		}
704 	} else
705 		/* Set the PCR with no hpriv event counting enabled. */
706 		ultra_setpcr(pcr);
707 
708 	ni2_cpc_counting[CPU->cpu_id] = B_TRUE;
709 
710 	/*
711 	 * On UltraSPARC, only read-to-read counts are accurate. We cannot
712 	 * expect the value we wrote into the PIC, above, to be there after
713 	 * starting the counter. We must sample the counter value now and use
714 	 * that as the baseline for future samples.
715 	 */
716 	curpic = ultra_getpic();
717 	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
718 	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
719 
720 	DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic);
721 }
722 
723 static void
ni2_pcbe_allstop(void)724 ni2_pcbe_allstop(void)
725 {
726 	/*
727 	 * We use the HV interface here because if we were counting
728 	 * hyperprivileged events, we must reset the PCR.ht bit to stop
729 	 * the counting. In the event that this HV call fails, we fall
730 	 * back on ultra_setpcr which does not have write access to the
731 	 * ht bit.
732 	 */
733 	if (hv_niagara_setperf(HV_SPARC_CTL, allstopped) != H_EOK)
734 		ultra_setpcr(allstopped);
735 
736 	ni2_cpc_counting[CPU->cpu_id] = B_FALSE;
737 }
738 
739 static void
ni2_pcbe_sample(void * token)740 ni2_pcbe_sample(void *token)
741 {
742 	uint64_t		curpic;
743 	int64_t			diff;
744 	uint64_t		*pic0_data;
745 	uint64_t		*pic1_data;
746 	uint64_t		*dtmp;
747 	uint64_t		tmp;
748 	uint64_t		pcr;
749 	ni2_pcbe_config_t	*pic0;
750 	ni2_pcbe_config_t	*pic1;
751 	ni2_pcbe_config_t	nullcfg = { 1, 0, 0, 0 };
752 	ni2_pcbe_config_t	*ctmp;
753 
754 	curpic = ultra_getpic();
755 	DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic);
756 
757 	if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
758 		panic("%s: token %p has no configs", cpu_impl_name, token);
759 
760 	if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
761 		pic1 = &nullcfg;
762 		pic1_data = &tmp;
763 	}
764 
765 	if (pic0->pcbe_picno != 0) {
766 		nullcfg.pcbe_picno = 0;
767 		ctmp = pic0;
768 		pic0 = pic1;
769 		pic1 = ctmp;
770 		dtmp = pic0_data;
771 		pic0_data = pic1_data;
772 		pic1_data = dtmp;
773 	}
774 
775 	if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
776 		panic("%s: bad config on token %p\n", cpu_impl_name, token);
777 
778 
779 	if (pic0->pcbe_flags & CPC_COUNT_HV) {
780 		/*
781 		 * If the hpriv attribute is present, but the HT bit
782 		 * is not set in the PCR, access to hyperprivileged
783 		 * events must have been revoked. Only perform this
784 		 * check if counting is not stopped.
785 		 */
786 		pcr = ultra_getpcr();
787 		DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
788 		if (ni2_cpc_counting[CPU->cpu_id] &&
789 		    !(pcr & CPC_PCR_HT)) {
790 			kcpc_invalidate_config(token);
791 			return;
792 		}
793 	}
794 
795 	diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic;
796 	if (diff < 0)
797 		diff += (1ll << 32);
798 	*pic0_data += diff;
799 
800 	diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic;
801 	if (diff < 0)
802 		diff += (1ll << 32);
803 	*pic1_data += diff;
804 
805 	pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
806 	pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
807 }
808 
809 static void
ni2_pcbe_free(void * config)810 ni2_pcbe_free(void *config)
811 {
812 	kmem_free(config, sizeof (ni2_pcbe_config_t));
813 }
814 
815 
816 static struct modlpcbe modlpcbe = {
817 	&mod_pcbeops,
818 #if defined(NIAGARA2_IMPL)
819 	"UltraSPARC T2 Performance Counters",
820 #elif defined(VFALLS_IMPL)
821 	"UltraSPARC T2+ Performance Counters",
822 #elif defined(KT_IMPL)
823 	"SPARC T3 Performance Counters",
824 #endif
825 	&ni2_pcbe_ops
826 };
827 
828 static struct modlinkage modl = {
829 	MODREV_1,
830 	&modlpcbe,
831 };
832 
833 int
_init(void)834 _init(void)
835 {
836 	if (ni2_pcbe_init() != 0)
837 		return (ENOTSUP);
838 	return (mod_install(&modl));
839 }
840 
841 int
_fini(void)842 _fini(void)
843 {
844 	return (mod_remove(&modl));
845 }
846 
847 int
_info(struct modinfo * mi)848 _info(struct modinfo *mi)
849 {
850 	return (mod_info(&modl, mi));
851 }
852