1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * Niagara2 Performance Counter Backend 28 */ 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #include <sys/cpuvar.h> 33 #include <sys/systm.h> 34 #include <sys/archsystm.h> 35 #include <sys/cmn_err.h> 36 #include <sys/cpc_impl.h> 37 #include <sys/cpc_pcbe.h> 38 #include <sys/modctl.h> 39 #include <sys/machsystm.h> 40 #include <sys/sdt.h> 41 #include <sys/niagara2regs.h> 42 #include <sys/hsvc.h> 43 #include <sys/hypervisor_api.h> 44 #include <sys/disp.h> 45 46 static int ni2_pcbe_init(void); 47 static uint_t ni2_pcbe_ncounters(void); 48 static const char *ni2_pcbe_impl_name(void); 49 static const char *ni2_pcbe_cpuref(void); 50 static char *ni2_pcbe_list_events(uint_t picnum); 51 static char *ni2_pcbe_list_attrs(void); 52 static uint64_t ni2_pcbe_event_coverage(char *event); 53 static uint64_t ni2_pcbe_overflow_bitmap(void); 54 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, 55 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data, 56 void *token); 57 static void ni2_pcbe_program(void *token); 58 static void ni2_pcbe_allstop(void); 59 static void ni2_pcbe_sample(void *token); 60 static void ni2_pcbe_free(void *config); 61 62 extern void ultra_setpcr(uint64_t); 63 extern uint64_t ultra_getpcr(void); 64 extern void ultra_setpic(uint64_t); 65 extern uint64_t ultra_getpic(void); 66 extern uint64_t ultra_gettick(void); 67 extern char cpu_module_name[]; 68 69 #ifdef N2_1x_CPC_WORKAROUNDS 70 extern uint64_t ni2_1x_perf_workarounds; 71 #endif 72 73 pcbe_ops_t ni2_pcbe_ops = { 74 PCBE_VER_1, 75 CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE, 76 ni2_pcbe_ncounters, 77 ni2_pcbe_impl_name, 78 ni2_pcbe_cpuref, 79 ni2_pcbe_list_events, 80 ni2_pcbe_list_attrs, 81 ni2_pcbe_event_coverage, 82 ni2_pcbe_overflow_bitmap, 83 ni2_pcbe_configure, 84 ni2_pcbe_program, 85 ni2_pcbe_allstop, 86 ni2_pcbe_sample, 87 ni2_pcbe_free 88 }; 89 90 typedef struct _ni2_pcbe_config { 91 uint_t pcbe_picno; /* 0 for pic0 or 1 for pic1 */ 92 uint32_t pcbe_evsel; /* %pcr event code unshifted */ 93 uint32_t pcbe_flags; /* hpriv/user/system/priv */ 94 uint32_t pcbe_pic; /* unshifted raw %pic value */ 95 } ni2_pcbe_config_t; 96 97 typedef struct _ni2_event { 98 const char *name; 99 const uint32_t emask; 100 const uint32_t emask_valid; /* Mask of unreserved MASK bits */ 101 } ni2_event_t; 102 103 #define ULTRA_PCR_PRIVPIC (UINT64_C(1) << CPC_NIAGARA2_PCR_PRIV_SHIFT) 104 #define EV_END {NULL, 0, 0} 105 106 static const uint64_t allstopped = (ULTRA_PCR_PRIVPIC | 107 CPC_NIAGARA2_PCR_HOLDOV0 | CPC_NIAGARA2_PCR_HOLDOV1); 108 109 /* 110 * We update this array in the program and allstop routine. The array 111 * is checked in the sample routine to allow us to only perform the 112 * PCR.ht bit check when counting is in progress. 113 */ 114 static boolean_t ni2_cpc_counting[NCPU]; 115 116 #ifdef N2_1x_CPC_WORKAROUNDS 117 static ni2_event_t ni2_1x_events[] = { 118 { "Idle_strands", 0x000, 0x00 }, 119 { "Br_completed", 0x201, 0x7f }, 120 { "Br_taken", 0x202, 0x7f }, 121 { "Instr_FGU_arithmetic", 0x204, 0x7f }, 122 { "Instr_ld", 0x208, 0x7f }, 123 { "Instr_st", 0x210, 0x7f }, 124 { "Instr_sw", 0x220, 0x7f }, 125 { "Instr_other", 0x240, 0x7f }, 126 { "Instr_cnt", 0x27d, 0x7f }, 127 { "IC_miss", 0x301, 0x3f }, 128 { "DC_miss", 0x302, 0x3f }, 129 { "ITLB_miss", 0x304, 0x3f }, 130 { "DTLB_miss", 0x308, 0x3f }, 131 { "L2_imiss", 0x310, 0x3f }, 132 { "L2_dmiss_ld", 0x320, 0x3f }, 133 { "ITLB_HWTW_ref_L2", 0x404, 0x3c }, 134 { "DTLB_HWTW_ref_L2", 0x408, 0x3c }, 135 { "ITLB_HWTW_miss_L2", 0x410, 0x3c }, 136 { "DTLB_HWTW_miss_L2", 0x420, 0x3c }, 137 { "Stream_ld_to_PCX", 0x501, 0x3f }, 138 { "Stream_st_to_PCX", 0x502, 0x3f }, 139 { "CPU_ld_to_PCX", 0x504, 0x3f }, 140 { "CPU_ifetch_to_PCX", 0x508, 0x3f }, 141 { "CPU_st_to_PCX", 0x510, 0x3f }, 142 { "MMU_ld_to_PCX", 0x520, 0x3f }, 143 { "DES_3DES_op", 0x601, 0x3f }, 144 { "AES_op", 0x602, 0x3f }, 145 { "RC4_op", 0x604, 0x3f }, 146 { "MD5_SHA-1_SHA-256_op", 0x608, 0x3f }, 147 { "MA_op", 0x610, 0x3f }, 148 { "CRC_TCPIP_cksum", 0x620, 0x3f }, 149 { "DES_3DES_busy_cycle", 0x701, 0x3f }, 150 { "AES_busy_cycle", 0x702, 0x3f }, 151 { "RC4_busy_cycle", 0x704, 0x3f }, 152 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0x3f }, 153 { "MA_busy_cycle", 0x710, 0x3f }, 154 { "CRC_MPA_cksum", 0x720, 0x3f }, 155 EV_END 156 }; 157 #endif 158 159 static ni2_event_t ni2_2x_events[] = { 160 { "Idle_strands", 0x000, 0x00 }, 161 { "Br_completed", 0x201, 0xff }, 162 { "Br_taken", 0x202, 0xff }, 163 { "Instr_FGU_arithmetic", 0x204, 0xff }, 164 { "Instr_ld", 0x208, 0xff }, 165 { "Instr_st", 0x210, 0xff }, 166 { "Instr_sw", 0x220, 0xff }, 167 { "Instr_other", 0x240, 0xff }, 168 { "Atomics", 0x280, 0xff }, 169 { "Instr_cnt", 0x2fd, 0xff }, 170 { "IC_miss", 0x301, 0x33 }, 171 { "DC_miss", 0x302, 0x33 }, 172 { "L2_imiss", 0x310, 0x33 }, 173 { "L2_dmiss_ld", 0x320, 0x33 }, 174 { "ITLB_HWTW_ref_L2", 0x404, 0x3c }, 175 { "DTLB_HWTW_ref_L2", 0x408, 0x3c }, 176 { "ITLB_HWTW_miss_L2", 0x410, 0x3c }, 177 { "DTLB_HWTW_miss_L2", 0x420, 0x3c }, 178 { "Stream_ld_to_PCX", 0x501, 0x3f }, 179 { "Stream_st_to_PCX", 0x502, 0x3f }, 180 { "CPU_ld_to_PCX", 0x504, 0x3f }, 181 { "CPU_ifetch_to_PCX", 0x508, 0x3f }, 182 { "CPU_st_to_PCX", 0x510, 0x3f }, 183 { "MMU_ld_to_PCX", 0x520, 0x3f }, 184 { "DES_3DES_op", 0x601, 0x3f }, 185 { "AES_op", 0x602, 0x3f }, 186 { "RC4_op", 0x604, 0x3f }, 187 { "MD5_SHA-1_SHA-256_op", 0x608, 0x3f }, 188 { "MA_op", 0x610, 0x3f }, 189 { "CRC_TCPIP_cksum", 0x620, 0x3f }, 190 { "DES_3DES_busy_cycle", 0x701, 0x3f }, 191 { "AES_busy_cycle", 0x702, 0x3f }, 192 { "RC4_busy_cycle", 0x704, 0x3f }, 193 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0x3f }, 194 { "MA_busy_cycle", 0x710, 0x3f }, 195 { "CRC_MPA_cksum", 0x720, 0x3f }, 196 { "ITLB_miss", 0xb04, 0x0c }, 197 { "DTLB_miss", 0xb08, 0x0c }, 198 { "TLB_miss", 0xb0c, 0x0c }, 199 EV_END 200 }; 201 202 static ni2_event_t *ni2_events = ni2_2x_events; 203 static const char *ni2_impl_name = "UltraSPARC T2"; 204 static char *evlist; 205 static size_t evlist_sz; 206 static uint16_t pcr_pic0_mask; 207 static uint16_t pcr_pic1_mask; 208 209 #define CPU_REF_URL " Documentation for Sun processors can be found at: " \ 210 "http://www.sun.com/processors/manuals" 211 212 static const char *niagara2_cpuref = "See the \"UltraSPARC T2 User's Manual\" " 213 "for descriptions of these events." CPU_REF_URL; 214 215 static boolean_t niagara2_hsvc_available = B_TRUE; 216 217 static int 218 ni2_pcbe_init(void) 219 { 220 ni2_event_t *evp; 221 int status; 222 uint64_t niagara2_hsvc_major; 223 uint64_t niagara2_hsvc_minor; 224 225 pcr_pic0_mask = CPC_NIAGARA2_PCR_PIC0_MASK; 226 pcr_pic1_mask = CPC_NIAGARA2_PCR_PIC1_MASK; 227 228 /* 229 * Validate API version for Niagara2 specific hypervisor services 230 */ 231 status = hsvc_version(HSVC_GROUP_NIAGARA2_CPU, &niagara2_hsvc_major, 232 &niagara2_hsvc_minor); 233 if ((status != 0) || (niagara2_hsvc_major != NIAGARA2_HSVC_MAJOR)) { 234 cmn_err(CE_WARN, "hypervisor services not negotiated " 235 "or unsupported major number: group: 0x%x major: 0x%lx " 236 "minor: 0x%lx errno: %d", HSVC_GROUP_NIAGARA2_CPU, 237 niagara2_hsvc_major, niagara2_hsvc_minor, status); 238 niagara2_hsvc_available = B_FALSE; 239 } 240 #ifdef N2_1x_CPC_WORKAROUNDS 241 if (ni2_1x_perf_workarounds) 242 ni2_events = ni2_1x_events; 243 #endif 244 /* 245 * Construct event list. 246 * 247 * First pass: Calculate size needed. We'll need an additional byte 248 * for the NULL pointer during the last strcat. 249 * 250 * Second pass: Copy strings. 251 */ 252 for (evp = ni2_events; evp->name != NULL; evp++) 253 evlist_sz += strlen(evp->name) + 1; 254 255 evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP); 256 evlist[0] = '\0'; 257 258 for (evp = ni2_events; evp->name != NULL; evp++) { 259 (void) strcat(evlist, evp->name); 260 (void) strcat(evlist, ","); 261 } 262 /* 263 * Remove trailing comma. 264 */ 265 evlist[evlist_sz - 1] = '\0'; 266 267 return (0); 268 } 269 270 static uint_t 271 ni2_pcbe_ncounters(void) 272 { 273 return (2); 274 } 275 276 static const char * 277 ni2_pcbe_impl_name(void) 278 { 279 return (ni2_impl_name); 280 } 281 282 static const char * 283 ni2_pcbe_cpuref(void) 284 { 285 return (niagara2_cpuref); 286 } 287 288 static char * 289 ni2_pcbe_list_events(uint_t picnum) 290 { 291 ASSERT(picnum < cpc_ncounters); 292 293 return (evlist); 294 } 295 296 static char * 297 ni2_pcbe_list_attrs(void) 298 { 299 if (niagara2_hsvc_available == B_TRUE) 300 return ("hpriv,emask"); 301 else 302 return ("emask"); 303 } 304 305 static ni2_event_t * 306 find_event(char *name) 307 { 308 ni2_event_t *evp; 309 310 for (evp = ni2_events; evp->name != NULL; evp++) 311 if (strcmp(name, evp->name) == 0) 312 return (evp); 313 314 return (NULL); 315 } 316 317 /*ARGSUSED*/ 318 static uint64_t 319 ni2_pcbe_event_coverage(char *event) 320 { 321 /* 322 * Fortunately, both pic0 and pic1 can count all events. 323 */ 324 return (0x3); 325 } 326 327 #ifdef N2_1x_CPC_WORKAROUNDS 328 uint64_t ni2_ov_tstamp[NCPU]; /* last overflow time stamp */ 329 uint64_t ni2_ov_spurious_range = 1000000; /* 1 msec at 1GHz */ 330 #endif 331 332 static uint64_t 333 ni2_pcbe_overflow_bitmap(void) 334 { 335 uint64_t pcr, overflow; 336 uint64_t pic; 337 uint32_t pic0, pic1; 338 boolean_t update_pic = B_FALSE; 339 340 ASSERT(getpil() >= DISP_LEVEL); 341 pcr = ultra_getpcr(); 342 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 343 overflow = (pcr & CPC_NIAGARA2_PCR_OV0_MASK) >> 344 CPC_NIAGARA2_PCR_OV0_SHIFT; 345 overflow |= (pcr & CPC_NIAGARA2_PCR_OV1_MASK) >> 346 CPC_NIAGARA2_PCR_OV1_SHIFT; 347 348 pic = ultra_getpic(); 349 pic0 = (uint32_t)(pic & PIC0_MASK); 350 pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK); 351 352 #ifdef N2_1x_CPC_WORKAROUNDS 353 if (ni2_1x_perf_workarounds) { 354 uint64_t tstamp; 355 processorid_t cpun; 356 357 /* 358 * Niagara2 1.x silicon can generate a duplicate overflow 359 * trap per event. If we take an overflow trap with no 360 * counters overflowing, return a non-zero bitmask with no 361 * OV bit set for supported counter so that the framework 362 * can ignore this trap. 363 */ 364 cpun = CPU->cpu_id; 365 tstamp = ultra_gettick(); 366 if (overflow) 367 ni2_ov_tstamp[cpun] = tstamp; 368 else if (tstamp < (ni2_ov_tstamp[cpun] + ni2_ov_spurious_range)) 369 overflow |= 1ULL << 63; 370 371 /* 372 * In Niagara2 1.x silicon, PMU doesn't set OV bit for 373 * precise events. So, if we take a trap with the counter 374 * within the overflow range and the OV bit is not set, we 375 * assume OV bit should have been set. 376 */ 377 if (PIC_IN_OV_RANGE(pic0)) 378 overflow |= 0x1; 379 if (PIC_IN_OV_RANGE(pic1)) 380 overflow |= 0x2; 381 } 382 #endif 383 384 pcr |= (CPC_NIAGARA2_PCR_HOLDOV0 | CPC_NIAGARA2_PCR_HOLDOV1); 385 386 if (overflow & 0x1) { 387 pcr &= ~(CPC_NIAGARA2_PCR_OV0_MASK | 388 CPC_NIAGARA2_PCR_HOLDOV0); 389 if (PIC_IN_OV_RANGE(pic0)) { 390 pic0 = 0; 391 update_pic = B_TRUE; 392 } 393 } 394 395 if (overflow & 0x2) { 396 pcr &= ~(CPC_NIAGARA2_PCR_OV1_MASK | 397 CPC_NIAGARA2_PCR_HOLDOV1); 398 if (PIC_IN_OV_RANGE(pic1)) { 399 pic1 = 0; 400 update_pic = B_TRUE; 401 } 402 } 403 404 if (update_pic) 405 ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0); 406 407 /* 408 * The HV interface does not need to be used here because we are 409 * only resetting the OV bits and do not need to set the HT bit. 410 */ 411 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 412 ultra_setpcr(pcr); 413 414 return (overflow); 415 } 416 417 /*ARGSUSED*/ 418 static int 419 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags, 420 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token) 421 { 422 ni2_pcbe_config_t *cfg; 423 ni2_pcbe_config_t *other_config; 424 ni2_event_t *evp; 425 int i; 426 uint32_t evsel; 427 428 /* 429 * If we've been handed an existing configuration, we need only preset 430 * the counter value. 431 */ 432 if (*data != NULL) { 433 cfg = *data; 434 cfg->pcbe_pic = (uint32_t)preset; 435 return (0); 436 } 437 438 if (picnum > 1) 439 return (CPC_INVALID_PICNUM); 440 441 if ((evp = find_event(event)) == NULL) 442 return (CPC_INVALID_EVENT); 443 444 evsel = evp->emask; 445 446 for (i = 0; i < nattrs; i++) { 447 if (strcmp(attrs[i].ka_name, "hpriv") == 0) { 448 if (attrs[i].ka_val != 0) 449 flags |= CPC_COUNT_HV; 450 } else if (strcmp(attrs[i].ka_name, "emask") == 0) { 451 if ((attrs[i].ka_val | evp->emask_valid) != 452 evp->emask_valid) 453 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 454 evsel |= attrs[i].ka_val; 455 } else 456 return (CPC_INVALID_ATTRIBUTE); 457 } 458 459 /* 460 * Find other requests that will be programmed with this one, and ensure 461 * the flags don't conflict. 462 */ 463 if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) && 464 (other_config->pcbe_flags != flags)) 465 return (CPC_CONFLICTING_REQS); 466 467 /* 468 * If the hpriv attribute is present, make sure we have 469 * access to hyperprivileged events before continuing with 470 * this configuration. If we can set the ht bit in the PCR 471 * successfully, we must have access to hyperprivileged 472 * events. 473 * 474 * If this is a static per-CPU configuration, the CPC 475 * driver ensures there can not be more than one for this 476 * CPU. If this is a per-LWP configuration, the driver 477 * ensures no static per-CPU counting is ongoing and that 478 * the target LWP is not already being monitored. 479 */ 480 if (flags & CPC_COUNT_HV) { 481 kpreempt_disable(); 482 483 DTRACE_PROBE1(niagara2__setpcr, uint64_t, 484 allstopped | CPC_NIAGARA2_PCR_HT); 485 if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, 486 allstopped | CPC_NIAGARA2_PCR_HT) != H_EOK) { 487 kpreempt_enable(); 488 return (CPC_HV_NO_ACCESS); 489 } 490 491 DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped); 492 (void) hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, allstopped); 493 494 kpreempt_enable(); 495 } 496 497 cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP); 498 499 cfg->pcbe_picno = picnum; 500 cfg->pcbe_evsel = evsel; 501 cfg->pcbe_flags = flags; 502 cfg->pcbe_pic = (uint32_t)preset; 503 504 *data = cfg; 505 return (0); 506 } 507 508 static void 509 ni2_pcbe_program(void *token) 510 { 511 ni2_pcbe_config_t *pic0; 512 ni2_pcbe_config_t *pic1; 513 ni2_pcbe_config_t *tmp; 514 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 515 uint64_t pcr; 516 uint64_t curpic; 517 uint64_t toe; 518 519 /* enable trap-on-event for pic0 and pic1 */ 520 toe = (CPC_NIAGARA2_PCR_TOE0 | CPC_NIAGARA2_PCR_TOE1); 521 522 if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) == 523 NULL) 524 panic("ni2_pcbe: token %p has no configs", token); 525 526 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) { 527 pic1 = &nullcfg; 528 nullcfg.pcbe_flags = pic0->pcbe_flags; 529 toe = CPC_NIAGARA2_PCR_TOE0; /* enable trap-on-event for pic0 */ 530 } 531 532 if (pic0->pcbe_picno != 0) { 533 /* 534 * pic0 is counter 1, so if we need the null config it should 535 * be counter 0. 536 */ 537 nullcfg.pcbe_picno = 0; 538 tmp = pic0; 539 pic0 = pic1; 540 pic1 = tmp; 541 toe = CPC_NIAGARA2_PCR_TOE1; /* enable trap-on-event for pic1 */ 542 } 543 544 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 545 panic("%s: bad config on token %p\n", ni2_impl_name, token); 546 547 /* 548 * UltraSPARC does not allow pic0 to be configured differently 549 * from pic1. If the flags on these two configurations are 550 * different, they are incompatible. This condition should be 551 * caught at configure time. 552 */ 553 ASSERT(pic0->pcbe_flags == pic1->pcbe_flags); 554 555 ni2_pcbe_allstop(); 556 557 ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) | 558 (uint64_t)pic0->pcbe_pic); 559 560 pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_NIAGARA2_PCR_PIC0_SHIFT; 561 pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) << 562 CPC_NIAGARA2_PCR_PIC1_SHIFT; 563 564 if (pic0->pcbe_flags & CPC_COUNT_USER) 565 pcr |= (1ull << CPC_NIAGARA2_PCR_UT_SHIFT); 566 if (pic0->pcbe_flags & CPC_COUNT_SYSTEM) 567 pcr |= (1ull << CPC_NIAGARA2_PCR_ST_SHIFT); 568 if (pic0->pcbe_flags & CPC_COUNT_HV) 569 pcr |= (1ull << CPC_NIAGARA2_PCR_HT_SHIFT); 570 pcr |= toe; 571 572 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 573 574 if (pic0->pcbe_flags & CPC_COUNT_HV) { 575 /* 576 * The ht bit in the PCR is only writable in 577 * hyperprivileged mode. So if we are counting 578 * hpriv events, we must use the HV interface 579 * hv_niagara_setperf to set the PCR. If this 580 * fails, assume we no longer have access to 581 * hpriv events. 582 */ 583 if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, pcr) != H_EOK) { 584 kcpc_invalidate_config(token); 585 return; 586 } 587 } else 588 /* Set the PCR with no hpriv event counting enabled. */ 589 ultra_setpcr(pcr); 590 591 ni2_cpc_counting[CPU->cpu_id] = B_TRUE; 592 593 /* 594 * On UltraSPARC, only read-to-read counts are accurate. We cannot 595 * expect the value we wrote into the PIC, above, to be there after 596 * starting the counter. We must sample the counter value now and use 597 * that as the baseline for future samples. 598 */ 599 curpic = ultra_getpic(); 600 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 601 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 602 603 DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic); 604 } 605 606 static void 607 ni2_pcbe_allstop(void) 608 { 609 /* 610 * We use the HV interface here because if we were counting 611 * hyperprivileged events, we must reset the PCR.ht bit to stop 612 * the counting. In the event that this HV call fails, we fall 613 * back on ultra_setpcr which does not have write access to the 614 * ht bit. 615 */ 616 if (hv_niagara_setperf(HV_NIAGARA_SPARC_CTL, allstopped) != H_EOK) 617 ultra_setpcr(allstopped); 618 619 ni2_cpc_counting[CPU->cpu_id] = B_FALSE; 620 } 621 622 static void 623 ni2_pcbe_sample(void *token) 624 { 625 uint64_t curpic; 626 int64_t diff; 627 uint64_t *pic0_data; 628 uint64_t *pic1_data; 629 uint64_t *dtmp; 630 uint64_t tmp; 631 uint64_t pcr; 632 ni2_pcbe_config_t *pic0; 633 ni2_pcbe_config_t *pic1; 634 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 635 ni2_pcbe_config_t *ctmp; 636 637 curpic = ultra_getpic(); 638 DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic); 639 640 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL) 641 panic("%s: token %p has no configs", ni2_impl_name, token); 642 643 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) { 644 pic1 = &nullcfg; 645 pic1_data = &tmp; 646 } 647 648 if (pic0->pcbe_picno != 0) { 649 nullcfg.pcbe_picno = 0; 650 ctmp = pic0; 651 pic0 = pic1; 652 pic1 = ctmp; 653 dtmp = pic0_data; 654 pic0_data = pic1_data; 655 pic1_data = dtmp; 656 } 657 658 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 659 panic("%s: bad config on token %p\n", ni2_impl_name, token); 660 661 662 if (pic0->pcbe_flags & CPC_COUNT_HV) { 663 /* 664 * If the hpriv attribute is present, but the HT bit 665 * is not set in the PCR, access to hyperprivileged 666 * events must have been revoked. Only perform this 667 * check if counting is not stopped. 668 */ 669 #ifdef N2_1x_CPC_WORKAROUNDS 670 if (!ni2_1x_perf_workarounds) { 671 #endif 672 pcr = ultra_getpcr(); 673 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 674 if (ni2_cpc_counting[CPU->cpu_id] && 675 !(pcr & CPC_NIAGARA2_PCR_HT)) { 676 kcpc_invalidate_config(token); 677 return; 678 } 679 #ifdef N2_1x_CPC_WORKAROUNDS 680 } 681 #endif 682 } 683 684 diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic; 685 if (diff < 0) 686 diff += (1ll << 32); 687 *pic0_data += diff; 688 689 diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic; 690 if (diff < 0) 691 diff += (1ll << 32); 692 *pic1_data += diff; 693 694 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 695 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 696 } 697 698 static void 699 ni2_pcbe_free(void *config) 700 { 701 kmem_free(config, sizeof (ni2_pcbe_config_t)); 702 } 703 704 705 static struct modlpcbe modlpcbe = { 706 &mod_pcbeops, 707 "UltraSPARC T2 Performance Counters v%I%", 708 &ni2_pcbe_ops 709 }; 710 711 static struct modlinkage modl = { 712 MODREV_1, 713 &modlpcbe, 714 }; 715 716 int 717 _init(void) 718 { 719 if (ni2_pcbe_init() != 0) 720 return (ENOTSUP); 721 return (mod_install(&modl)); 722 } 723 724 int 725 _fini(void) 726 { 727 return (mod_remove(&modl)); 728 } 729 730 int 731 _info(struct modinfo *mi) 732 { 733 return (mod_info(&modl, mi)); 734 } 735