1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * This file contains preset event names from the Performance Application
27 * Programming Interface v3.5 which included the following notice:
28 *
29 * Copyright (c) 2005,6
30 * Innovative Computing Labs
31 * Computer Science Department,
32 * University of Tennessee,
33 * Knoxville, TN.
34 * All Rights Reserved.
35 *
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions are met:
39 *
40 * * Redistributions of source code must retain the above copyright notice,
41 * this list of conditions and the following disclaimer.
42 * * Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * * Neither the name of the University of Tennessee nor the names of its
46 * contributors may be used to endorse or promote products derived from
47 * this software without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
50 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
53 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
59 * POSSIBILITY OF SUCH DAMAGE.
60 *
61 *
62 * This open source software license conforms to the BSD License template.
63 */
64
65 /*
66 * Niagara2 Performance Counter Backend
67 */
68
69 #include <sys/cpuvar.h>
70 #include <sys/systm.h>
71 #include <sys/archsystm.h>
72 #include <sys/cmn_err.h>
73 #include <sys/cpc_impl.h>
74 #include <sys/cpc_pcbe.h>
75 #include <sys/modctl.h>
76 #include <sys/machsystm.h>
77 #include <sys/sdt.h>
78 #include <sys/niagara2regs.h>
79 #include <sys/hsvc.h>
80 #include <sys/hypervisor_api.h>
81 #include <sys/disp.h>
82
83 /*LINTLIBRARY*/
84 static int ni2_pcbe_init(void);
85 static uint_t ni2_pcbe_ncounters(void);
86 static const char *ni2_pcbe_impl_name(void);
87 static const char *ni2_pcbe_cpuref(void);
88 static char *ni2_pcbe_list_events(uint_t picnum);
89 static char *ni2_pcbe_list_attrs(void);
90 static uint64_t ni2_pcbe_event_coverage(char *event);
91 static uint64_t ni2_pcbe_overflow_bitmap(void);
92 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
93 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
94 void *token);
95 static void ni2_pcbe_program(void *token);
96 static void ni2_pcbe_allstop(void);
97 static void ni2_pcbe_sample(void *token);
98 static void ni2_pcbe_free(void *config);
99
100 extern void ultra_setpcr(uint64_t);
101 extern uint64_t ultra_getpcr(void);
102 extern void ultra_setpic(uint64_t);
103 extern uint64_t ultra_getpic(void);
104 extern uint64_t ultra_gettick(void);
105 extern char cpu_module_name[];
106
107 #define L2_dmiss_ld_event 0x320
108 int pcbe_l2dmiss_ovtrap_enable = 0;
109
110 pcbe_ops_t ni2_pcbe_ops = {
111 PCBE_VER_1,
112 CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,
113 ni2_pcbe_ncounters,
114 ni2_pcbe_impl_name,
115 ni2_pcbe_cpuref,
116 ni2_pcbe_list_events,
117 ni2_pcbe_list_attrs,
118 ni2_pcbe_event_coverage,
119 ni2_pcbe_overflow_bitmap,
120 ni2_pcbe_configure,
121 ni2_pcbe_program,
122 ni2_pcbe_allstop,
123 ni2_pcbe_sample,
124 ni2_pcbe_free
125 };
126
127 typedef struct _ni2_pcbe_config {
128 uint_t pcbe_picno; /* 0 for pic0 or 1 for pic1 */
129 uint32_t pcbe_evsel; /* %pcr event code unshifted */
130 uint32_t pcbe_flags; /* hpriv/user/system/priv */
131 uint32_t pcbe_pic; /* unshifted raw %pic value */
132 } ni2_pcbe_config_t;
133
134 typedef struct _ni2_event {
135 const char *name;
136 const uint32_t emask;
137 const uint32_t emask_valid; /* Mask of unreserved MASK bits */
138 } ni2_event_t;
139
140 typedef struct _ni2_generic_event {
141 char *name;
142 char *event;
143 } ni2_generic_event_t;
144
145 #define ULTRA_PCR_PRIVPIC (UINT64_C(1) << CPC_PCR_PRIV_SHIFT)
146 #define EV_END {NULL, 0, 0}
147 #define GEN_EV_END {NULL, NULL}
148
149 static const uint64_t allstopped = (ULTRA_PCR_PRIVPIC |
150 CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1);
151
152 /*
153 * We update this array in the program and allstop routine. The array
154 * is checked in the sample routine to allow us to only perform the
155 * PCR.ht bit check when counting is in progress.
156 */
157 static boolean_t ni2_cpc_counting[NCPU];
158
159 static ni2_event_t ni2_events[] = {
160 { "Idle_strands", 0x000, 0x00 },
161 { "Br_completed", 0x201, 0xff },
162 { "Br_taken", 0x202, 0xff },
163 { "Instr_FGU_arithmetic", 0x204, 0xff },
164 { "Instr_ld", 0x208, 0xff },
165 { "Instr_st", 0x210, 0xff },
166 { "Instr_sw", 0x220, 0xff },
167 { "Instr_other", 0x240, 0xff },
168 { "Atomics", 0x280, 0xff },
169 { "Instr_cnt", 0x2fd, 0xff },
170 { "IC_miss", 0x301, 0x33 },
171 { "DC_miss", 0x302, 0x33 },
172 { "L2_imiss", 0x310, 0x33 },
173 { "L2_dmiss_ld", 0x320, 0x33 },
174 { "ITLB_HWTW_ref_L2", 0x404, 0x3c },
175 { "DTLB_HWTW_ref_L2", 0x408, 0x3c },
176 { "ITLB_HWTW_miss_L2", 0x410, 0x3c },
177 { "DTLB_HWTW_miss_L2", 0x420, 0x3c },
178 { "Stream_ld_to_PCX", 0x501, 0x3f },
179 { "Stream_st_to_PCX", 0x502, 0x3f },
180 { "CPU_ld_to_PCX", 0x504, 0x3f },
181 { "CPU_ifetch_to_PCX", 0x508, 0x3f },
182 { "CPU_st_to_PCX", 0x510, 0x3f },
183 { "MMU_ld_to_PCX", 0x520, 0x3f },
184 #ifdef KT_IMPL
185 { "DES_3DES_op", 0x601, 0xff },
186 { "AES_op", 0x602, 0xff },
187 { "Kasumi_op", 0x604, 0xff },
188 { "MD5_SHA-1_SHA-256_op", 0x608, 0xff },
189 { "MA_op", 0x610, 0xff },
190 { "CRC_TCPIP_cksum", 0x620, 0xff },
191 { "DES_3DES_busy_cycle", 0x701, 0xff },
192 { "AES_busy_cycle", 0x702, 0xff },
193 { "Kasumi_busy_cycle", 0x704, 0xff },
194 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0xff },
195 { "MA_busy_cycle", 0x710, 0xff },
196 { "CRC_MPA_cksum", 0x720, 0xff },
197 #else
198 { "DES_3DES_op", 0x601, 0x3f },
199 { "AES_op", 0x602, 0x3f },
200 { "RC4_op", 0x604, 0x3f },
201 { "MD5_SHA-1_SHA-256_op", 0x608, 0x3f },
202 { "MA_op", 0x610, 0x3f },
203 { "CRC_TCPIP_cksum", 0x620, 0x3f },
204 { "DES_3DES_busy_cycle", 0x701, 0x3f },
205 { "AES_busy_cycle", 0x702, 0x3f },
206 { "RC4_busy_cycle", 0x704, 0x3f },
207 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0x3f },
208 { "MA_busy_cycle", 0x710, 0x3f },
209 { "CRC_MPA_cksum", 0x720, 0x3f },
210 #endif
211 { "ITLB_miss", 0xb04, 0x0c },
212 { "DTLB_miss", 0xb08, 0x0c },
213 { "TLB_miss", 0xb0c, 0x0c },
214 EV_END
215 };
216
217 static ni2_generic_event_t ni2_generic_events[] = {
218 { "PAPI_tot_ins", "Instr_cnt" },
219 { "PAPI_l1_dcm", "DC_miss" },
220 { "PAPI_l1_icm", "IC_miss" },
221 { "PAPI_l2_icm", "L2_imiss" },
222 { "PAPI_l2_ldm", "L2_dmiss_ld" },
223 { "PAPI_tlb_dm", "DTLB_miss" },
224 { "PAPI_tlb_im", "ITLB_miss" },
225 { "PAPI_tlb_tm", "TLB_miss" },
226 { "PAPI_br_tkn", "Br_taken" },
227 { "PAPI_br_ins", "Br_completed" },
228 { "PAPI_ld_ins", "Instr_ld" },
229 { "PAPI_sr_ins", "Instr_st" },
230 { "PAPI_fp_ops", "Instr_FGU_arithmetic" },
231 { "PAPI_fp_ins", "Instr_FGU_arithmetic" },
232 GEN_EV_END
233 };
234
235 static char *evlist;
236 static size_t evlist_sz;
237 static uint16_t pcr_pic0_mask;
238 static uint16_t pcr_pic1_mask;
239
240 #define CPU_REF_URL " Documentation for Sun processors can be found at: " \
241 "http://www.sun.com/processors/manuals"
242
243 #if defined(NIAGARA2_IMPL)
244 static const char *cpu_impl_name = "UltraSPARC T2";
245 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2 User's Manual\" "
246 "for descriptions of these events." CPU_REF_URL;
247 #elif defined(VFALLS_IMPL)
248 static const char *cpu_impl_name = "UltraSPARC T2+";
249 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2+ User's Manual\" "
250 "for descriptions of these events." CPU_REF_URL;
251 #elif defined(KT_IMPL)
252 static const char *cpu_impl_name = "SPARC T3";
253 static const char *cpu_pcbe_ref = "See the \"SPARC T3 User's Manual\" "
254 "for descriptions of these events." CPU_REF_URL;
255 #endif
256
257 static boolean_t cpu_hsvc_available = B_TRUE;
258
259 static int
ni2_pcbe_init(void)260 ni2_pcbe_init(void)
261 {
262 ni2_event_t *evp;
263 ni2_generic_event_t *gevp;
264 int status;
265 uint64_t cpu_hsvc_major;
266 uint64_t cpu_hsvc_minor;
267 #if defined(NIAGARA2_IMPL)
268 uint64_t hsvc_cpu_group = HSVC_GROUP_NIAGARA2_CPU;
269 uint64_t hsvc_cpu_major = NIAGARA2_HSVC_MAJOR;
270 #elif defined(VFALLS_IMPL)
271 uint64_t hsvc_cpu_group = HSVC_GROUP_VFALLS_CPU;
272 uint64_t hsvc_cpu_major = VFALLS_HSVC_MAJOR;
273 #elif defined(KT_IMPL)
274 uint64_t hsvc_cpu_group = HSVC_GROUP_KT_CPU;
275 uint64_t hsvc_cpu_major = KT_HSVC_MAJOR;
276 #endif
277
278 pcr_pic0_mask = CPC_PCR_PIC0_MASK;
279 pcr_pic1_mask = CPC_PCR_PIC1_MASK;
280
281 /*
282 * Validate API version for Niagara2 specific hypervisor services
283 */
284 status = hsvc_version(hsvc_cpu_group, &cpu_hsvc_major,
285 &cpu_hsvc_minor);
286 if ((status != 0) || (cpu_hsvc_major != hsvc_cpu_major)) {
287 cmn_err(CE_WARN, "hypervisor services not negotiated "
288 "or unsupported major number: group: 0x%lx major: 0x%lx "
289 "minor: 0x%lx errno: %d", hsvc_cpu_group,
290 cpu_hsvc_major, cpu_hsvc_minor, status);
291 cpu_hsvc_available = B_FALSE;
292 }
293
294 /*
295 * Construct event list.
296 *
297 * First pass: Calculate size needed. We'll need an additional byte
298 * for the NULL pointer during the last strcat.
299 *
300 * Second pass: Copy strings.
301 */
302 for (evp = ni2_events; evp->name != NULL; evp++)
303 evlist_sz += strlen(evp->name) + 1;
304
305 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++)
306 evlist_sz += strlen(gevp->name) + 1;
307
308 evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP);
309 evlist[0] = '\0';
310
311 for (evp = ni2_events; evp->name != NULL; evp++) {
312 (void) strcat(evlist, evp->name);
313 (void) strcat(evlist, ",");
314 }
315
316 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) {
317 (void) strcat(evlist, gevp->name);
318 (void) strcat(evlist, ",");
319 }
320
321 /*
322 * Remove trailing comma.
323 */
324 evlist[evlist_sz - 1] = '\0';
325
326 return (0);
327 }
328
329 static uint_t
ni2_pcbe_ncounters(void)330 ni2_pcbe_ncounters(void)
331 {
332 return (2);
333 }
334
335 static const char *
ni2_pcbe_impl_name(void)336 ni2_pcbe_impl_name(void)
337 {
338 return (cpu_impl_name);
339 }
340
341 static const char *
ni2_pcbe_cpuref(void)342 ni2_pcbe_cpuref(void)
343 {
344 return (cpu_pcbe_ref);
345 }
346
347 static char *
ni2_pcbe_list_events(uint_t picnum)348 ni2_pcbe_list_events(uint_t picnum)
349 {
350 ASSERT(picnum < cpc_ncounters);
351
352 return (evlist);
353 }
354
355 static char *
ni2_pcbe_list_attrs(void)356 ni2_pcbe_list_attrs(void)
357 {
358 if (cpu_hsvc_available == B_TRUE)
359 #if defined(NIAGARA2_IMPL)
360 return ("hpriv,emask");
361 #elif defined(VFALLS_IMPL)
362 return ("hpriv,l2ctl,emask");
363 #elif defined(KT_IMPL)
364 return ("hpriv,l2ctl,emask,sample");
365 #endif
366 else
367 #if defined(KT_IMPL)
368 return ("emask,sample");
369 #else
370 return ("emask");
371 #endif
372 }
373
374 static ni2_generic_event_t *
find_generic_event(char * name)375 find_generic_event(char *name)
376 {
377 ni2_generic_event_t *gevp;
378
379 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) {
380 if (strcmp(name, gevp->name) == 0)
381 return (gevp);
382 }
383
384 return (NULL);
385 }
386
387 static ni2_event_t *
find_event(char * name)388 find_event(char *name)
389 {
390 ni2_event_t *evp;
391
392 for (evp = ni2_events; evp->name != NULL; evp++)
393 if (strcmp(name, evp->name) == 0)
394 return (evp);
395
396 return (NULL);
397 }
398
399 /*ARGSUSED*/
400 static uint64_t
ni2_pcbe_event_coverage(char * event)401 ni2_pcbe_event_coverage(char *event)
402 {
403 /*
404 * Check whether counter event is supported
405 */
406 if (find_event(event) == NULL && find_generic_event(event) == NULL)
407 return (0);
408
409 /*
410 * Fortunately, both pic0 and pic1 can count all events.
411 */
412 return (0x3);
413 }
414
415 static uint64_t
ni2_pcbe_overflow_bitmap(void)416 ni2_pcbe_overflow_bitmap(void)
417 {
418 uint64_t pcr, overflow;
419 uint64_t pic;
420 uint32_t pic0, pic1;
421 boolean_t update_pic = B_FALSE;
422 boolean_t pic_inrange = B_FALSE;
423
424 ASSERT(getpil() >= DISP_LEVEL);
425 pcr = ultra_getpcr();
426 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
427 overflow = (pcr & CPC_PCR_OV0_MASK) >>
428 CPC_PCR_OV0_SHIFT;
429 overflow |= (pcr & CPC_PCR_OV1_MASK) >>
430 CPC_PCR_OV1_SHIFT;
431
432 pic = ultra_getpic();
433 pic0 = (uint32_t)(pic & PIC0_MASK);
434 pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK);
435
436 pcr |= (CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1);
437
438 if (overflow & 0x1) {
439 pcr &= ~(CPC_PCR_OV0_MASK |
440 CPC_PCR_HOLDOV0);
441 pic_inrange = PIC_IN_OV_RANGE(pic0);
442 #if defined(KT_IMPL)
443 if (pcr & CPC_PCR_SAMPLE_MODE_MASK)
444 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic0);
445 #endif
446 if (pic_inrange) {
447 pic0 = 0;
448 update_pic = B_TRUE;
449 }
450 }
451
452 if (overflow & 0x2) {
453 pcr &= ~(CPC_PCR_OV1_MASK |
454 CPC_PCR_HOLDOV1);
455 pic_inrange = PIC_IN_OV_RANGE(pic1);
456 #if defined(KT_IMPL)
457 if (pcr & CPC_PCR_SAMPLE_MODE_MASK)
458 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic1);
459 #endif
460 if (pic_inrange) {
461 pic1 = 0;
462 update_pic = B_TRUE;
463 }
464 }
465
466 if (update_pic)
467 ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0);
468
469 /*
470 * The HV interface does not need to be used here because we are
471 * only resetting the OV bits and do not need to set the HT bit.
472 */
473 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
474 ultra_setpcr(pcr);
475
476 return (overflow);
477 }
478
479 /*ARGSUSED*/
480 static int
ni2_pcbe_configure(uint_t picnum,char * event,uint64_t preset,uint32_t flags,uint_t nattrs,kcpc_attr_t * attrs,void ** data,void * token)481 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
482 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
483 {
484 ni2_pcbe_config_t *cfg;
485 ni2_pcbe_config_t *other_config;
486 ni2_event_t *evp;
487 ni2_generic_event_t *gevp;
488 int i;
489 uint32_t evsel;
490 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
491 uint64_t l2ctl = 0;
492 #endif
493
494 /*
495 * If we've been handed an existing configuration, we need only preset
496 * the counter value.
497 */
498 if (*data != NULL) {
499 cfg = *data;
500 cfg->pcbe_pic = (uint32_t)preset;
501 return (0);
502 }
503
504 if (picnum > 1)
505 return (CPC_INVALID_PICNUM);
506
507
508 if ((evp = find_event(event)) == NULL) {
509 if ((gevp = find_generic_event(event)) != NULL) {
510 evp = find_event(gevp->event);
511 ASSERT(evp != NULL);
512
513 if (nattrs > 0)
514 return (CPC_ATTRIBUTE_OUT_OF_RANGE);
515 } else {
516 return (CPC_INVALID_EVENT);
517 }
518 }
519
520 evsel = evp->emask;
521
522 for (i = 0; i < nattrs; i++) {
523 if (strcmp(attrs[i].ka_name, "hpriv") == 0) {
524 if (attrs[i].ka_val != 0)
525 flags |= CPC_COUNT_HV;
526 } else if (strcmp(attrs[i].ka_name, "emask") == 0) {
527 if ((attrs[i].ka_val | evp->emask_valid) !=
528 evp->emask_valid)
529 return (CPC_ATTRIBUTE_OUT_OF_RANGE);
530 evsel |= attrs[i].ka_val;
531 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
532 } else if (strcmp(attrs[i].ka_name, "l2ctl") == 0) {
533 if ((attrs[i].ka_val | L2_CTL_MASK) !=
534 L2_CTL_MASK)
535 return (CPC_ATTRIBUTE_OUT_OF_RANGE);
536 else
537 l2ctl = attrs[i].ka_val;
538 #endif
539 #if defined(KT_IMPL)
540 } else if (strcmp(attrs[i].ka_name, "sample") == 0) {
541 if (attrs[i].ka_val != 0)
542 flags |= CPC_COUNT_SAMPLE_MODE;
543 #endif
544 } else
545 return (CPC_INVALID_ATTRIBUTE);
546 }
547
548 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
549 /*
550 * Set PERF_CONTROL bits in L2_CONTROL_REG only when events have
551 * SL bits equal to 3.
552 */
553 if ((evsel & SL_MASK) == SL3_MASK) {
554 if ((hv_niagara_setperf(HV_L2_CTL, l2ctl)) != 0)
555 return (CPC_HV_NO_ACCESS);
556 }
557 #endif
558
559 /*
560 * Find other requests that will be programmed with this one, and ensure
561 * the flags don't conflict.
562 */
563 if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
564 (other_config->pcbe_flags != flags))
565 return (CPC_CONFLICTING_REQS);
566
567 /*
568 * If the hpriv attribute is present, make sure we have
569 * access to hyperprivileged events before continuing with
570 * this configuration. If we can set the ht bit in the PCR
571 * successfully, we must have access to hyperprivileged
572 * events.
573 *
574 * If this is a static per-CPU configuration, the CPC
575 * driver ensures there can not be more than one for this
576 * CPU. If this is a per-LWP configuration, the driver
577 * ensures no static per-CPU counting is ongoing and that
578 * the target LWP is not already being monitored.
579 */
580 if (flags & CPC_COUNT_HV) {
581 kpreempt_disable();
582
583 DTRACE_PROBE1(niagara2__setpcr, uint64_t,
584 allstopped | CPC_PCR_HT);
585 if (hv_niagara_setperf(HV_SPARC_CTL,
586 allstopped | CPC_PCR_HT) != H_EOK) {
587 kpreempt_enable();
588 return (CPC_HV_NO_ACCESS);
589 }
590
591 DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped);
592 (void) hv_niagara_setperf(HV_SPARC_CTL, allstopped);
593
594 kpreempt_enable();
595 }
596
597 cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP);
598
599 cfg->pcbe_picno = picnum;
600 cfg->pcbe_evsel = evsel;
601 cfg->pcbe_flags = flags;
602 cfg->pcbe_pic = (uint32_t)preset;
603
604 *data = cfg;
605 return (0);
606 }
607
608 static void
ni2_pcbe_program(void * token)609 ni2_pcbe_program(void *token)
610 {
611 ni2_pcbe_config_t *pic0;
612 ni2_pcbe_config_t *pic1;
613 ni2_pcbe_config_t *tmp;
614 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 };
615 uint64_t pcr;
616 uint64_t curpic;
617 uint64_t toe;
618
619 /* enable trap-on-event for pic0 and pic1 */
620 toe = (CPC_PCR_TOE0 | CPC_PCR_TOE1);
621
622 if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) ==
623 NULL)
624 panic("ni2_pcbe: token %p has no configs", token);
625
626 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) {
627 pic1 = &nullcfg;
628 nullcfg.pcbe_flags = pic0->pcbe_flags;
629 toe = CPC_PCR_TOE0; /* enable trap-on-event for pic0 */
630 }
631
632 if (pic0->pcbe_picno != 0) {
633 /*
634 * pic0 is counter 1, so if we need the null config it should
635 * be counter 0.
636 */
637 nullcfg.pcbe_picno = 0;
638 tmp = pic0;
639 pic0 = pic1;
640 pic1 = tmp;
641 toe = CPC_PCR_TOE1; /* enable trap-on-event for pic1 */
642 }
643
644 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
645 panic("%s: bad config on token %p\n", cpu_impl_name, token);
646
647 /*
648 * UltraSPARC does not allow pic0 to be configured differently
649 * from pic1. If the flags on these two configurations are
650 * different, they are incompatible. This condition should be
651 * caught at configure time.
652 */
653 ASSERT(pic0->pcbe_flags == pic1->pcbe_flags);
654
655 ni2_pcbe_allstop();
656
657 ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) |
658 (uint64_t)pic0->pcbe_pic);
659
660 pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_PCR_PIC0_SHIFT;
661 pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) <<
662 CPC_PCR_PIC1_SHIFT;
663
664 if (pic0->pcbe_flags & CPC_COUNT_USER)
665 pcr |= (1ull << CPC_PCR_UT_SHIFT);
666 if (pic0->pcbe_flags & CPC_COUNT_SYSTEM)
667 pcr |= (1ull << CPC_PCR_ST_SHIFT);
668 if (pic0->pcbe_flags & CPC_COUNT_HV)
669 pcr |= (1ull << CPC_PCR_HT_SHIFT);
670 #if defined(KT_IMPL)
671 if (pic0->pcbe_flags & CPC_COUNT_SAMPLE_MODE)
672 pcr |= (1ull << CPC_PCR_SAMPLE_MODE_SHIFT);
673 #endif
674
675 if (pcbe_l2dmiss_ovtrap_enable == 0) {
676 /*
677 * SW workaround for HW Erratum 108.
678 * Disable overflow interrupts when L2_dmiss_ld event is
679 * selected.
680 */
681 if ((pic0->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event)
682 toe &= ~CPC_PCR_TOE0;
683
684 if ((pic1->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event)
685 toe &= ~CPC_PCR_TOE1;
686 }
687 pcr |= toe;
688
689 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr);
690
691 if (pic0->pcbe_flags & CPC_COUNT_HV) {
692 /*
693 * The ht bit in the PCR is only writable in
694 * hyperprivileged mode. So if we are counting
695 * hpriv events, we must use the HV interface
696 * hv_niagara_setperf to set the PCR. If this
697 * fails, assume we no longer have access to
698 * hpriv events.
699 */
700 if (hv_niagara_setperf(HV_SPARC_CTL, pcr) != H_EOK) {
701 kcpc_invalidate_config(token);
702 return;
703 }
704 } else
705 /* Set the PCR with no hpriv event counting enabled. */
706 ultra_setpcr(pcr);
707
708 ni2_cpc_counting[CPU->cpu_id] = B_TRUE;
709
710 /*
711 * On UltraSPARC, only read-to-read counts are accurate. We cannot
712 * expect the value we wrote into the PIC, above, to be there after
713 * starting the counter. We must sample the counter value now and use
714 * that as the baseline for future samples.
715 */
716 curpic = ultra_getpic();
717 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
718 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
719
720 DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic);
721 }
722
723 static void
ni2_pcbe_allstop(void)724 ni2_pcbe_allstop(void)
725 {
726 /*
727 * We use the HV interface here because if we were counting
728 * hyperprivileged events, we must reset the PCR.ht bit to stop
729 * the counting. In the event that this HV call fails, we fall
730 * back on ultra_setpcr which does not have write access to the
731 * ht bit.
732 */
733 if (hv_niagara_setperf(HV_SPARC_CTL, allstopped) != H_EOK)
734 ultra_setpcr(allstopped);
735
736 ni2_cpc_counting[CPU->cpu_id] = B_FALSE;
737 }
738
739 static void
ni2_pcbe_sample(void * token)740 ni2_pcbe_sample(void *token)
741 {
742 uint64_t curpic;
743 int64_t diff;
744 uint64_t *pic0_data;
745 uint64_t *pic1_data;
746 uint64_t *dtmp;
747 uint64_t tmp;
748 uint64_t pcr;
749 ni2_pcbe_config_t *pic0;
750 ni2_pcbe_config_t *pic1;
751 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 };
752 ni2_pcbe_config_t *ctmp;
753
754 curpic = ultra_getpic();
755 DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic);
756
757 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
758 panic("%s: token %p has no configs", cpu_impl_name, token);
759
760 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
761 pic1 = &nullcfg;
762 pic1_data = &tmp;
763 }
764
765 if (pic0->pcbe_picno != 0) {
766 nullcfg.pcbe_picno = 0;
767 ctmp = pic0;
768 pic0 = pic1;
769 pic1 = ctmp;
770 dtmp = pic0_data;
771 pic0_data = pic1_data;
772 pic1_data = dtmp;
773 }
774
775 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1)
776 panic("%s: bad config on token %p\n", cpu_impl_name, token);
777
778
779 if (pic0->pcbe_flags & CPC_COUNT_HV) {
780 /*
781 * If the hpriv attribute is present, but the HT bit
782 * is not set in the PCR, access to hyperprivileged
783 * events must have been revoked. Only perform this
784 * check if counting is not stopped.
785 */
786 pcr = ultra_getpcr();
787 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr);
788 if (ni2_cpc_counting[CPU->cpu_id] &&
789 !(pcr & CPC_PCR_HT)) {
790 kcpc_invalidate_config(token);
791 return;
792 }
793 }
794
795 diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic;
796 if (diff < 0)
797 diff += (1ll << 32);
798 *pic0_data += diff;
799
800 diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic;
801 if (diff < 0)
802 diff += (1ll << 32);
803 *pic1_data += diff;
804
805 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK);
806 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT);
807 }
808
809 static void
ni2_pcbe_free(void * config)810 ni2_pcbe_free(void *config)
811 {
812 kmem_free(config, sizeof (ni2_pcbe_config_t));
813 }
814
815
816 static struct modlpcbe modlpcbe = {
817 &mod_pcbeops,
818 #if defined(NIAGARA2_IMPL)
819 "UltraSPARC T2 Performance Counters",
820 #elif defined(VFALLS_IMPL)
821 "UltraSPARC T2+ Performance Counters",
822 #elif defined(KT_IMPL)
823 "SPARC T3 Performance Counters",
824 #endif
825 &ni2_pcbe_ops
826 };
827
828 static struct modlinkage modl = {
829 MODREV_1,
830 &modlpcbe,
831 };
832
833 int
_init(void)834 _init(void)
835 {
836 if (ni2_pcbe_init() != 0)
837 return (ENOTSUP);
838 return (mod_install(&modl));
839 }
840
841 int
_fini(void)842 _fini(void)
843 {
844 return (mod_remove(&modl));
845 }
846
847 int
_info(struct modinfo * mi)848 _info(struct modinfo *mi)
849 {
850 return (mod_info(&modl, mi));
851 }
852