1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * This file contains preset event names from the Performance Application 27 * Programming Interface v3.5 which included the following notice: 28 * 29 * Copyright (c) 2005,6 30 * Innovative Computing Labs 31 * Computer Science Department, 32 * University of Tennessee, 33 * Knoxville, TN. 34 * All Rights Reserved. 35 * 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions are met: 39 * 40 * * Redistributions of source code must retain the above copyright notice, 41 * this list of conditions and the following disclaimer. 42 * * Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * * Neither the name of the University of Tennessee nor the names of its 46 * contributors may be used to endorse or promote products derived from 47 * this software without specific prior written permission. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 50 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 52 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 53 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 * 61 * 62 * This open source software license conforms to the BSD License template. 63 */ 64 65 /* 66 * Niagara2 Performance Counter Backend 67 */ 68 69 #include <sys/cpuvar.h> 70 #include <sys/systm.h> 71 #include <sys/archsystm.h> 72 #include <sys/cmn_err.h> 73 #include <sys/cpc_impl.h> 74 #include <sys/cpc_pcbe.h> 75 #include <sys/modctl.h> 76 #include <sys/machsystm.h> 77 #include <sys/sdt.h> 78 #include <sys/niagara2regs.h> 79 #include <sys/hsvc.h> 80 #include <sys/hypervisor_api.h> 81 #include <sys/disp.h> 82 83 /*LINTLIBRARY*/ 84 static int ni2_pcbe_init(void); 85 static uint_t ni2_pcbe_ncounters(void); 86 static const char *ni2_pcbe_impl_name(void); 87 static const char *ni2_pcbe_cpuref(void); 88 static char *ni2_pcbe_list_events(uint_t picnum); 89 static char *ni2_pcbe_list_attrs(void); 90 static uint64_t ni2_pcbe_event_coverage(char *event); 91 static uint64_t ni2_pcbe_overflow_bitmap(void); 92 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, 93 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data, 94 void *token); 95 static void ni2_pcbe_program(void *token); 96 static void ni2_pcbe_allstop(void); 97 static void ni2_pcbe_sample(void *token); 98 static void ni2_pcbe_free(void *config); 99 100 extern void ultra_setpcr(uint64_t); 101 extern uint64_t ultra_getpcr(void); 102 extern void ultra_setpic(uint64_t); 103 extern uint64_t ultra_getpic(void); 104 extern uint64_t ultra_gettick(void); 105 extern char cpu_module_name[]; 106 107 #define L2_dmiss_ld_event 0x320 108 int pcbe_l2dmiss_ovtrap_enable = 0; 109 110 pcbe_ops_t ni2_pcbe_ops = { 111 PCBE_VER_1, 112 CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE, 113 ni2_pcbe_ncounters, 114 ni2_pcbe_impl_name, 115 ni2_pcbe_cpuref, 116 ni2_pcbe_list_events, 117 ni2_pcbe_list_attrs, 118 ni2_pcbe_event_coverage, 119 ni2_pcbe_overflow_bitmap, 120 ni2_pcbe_configure, 121 ni2_pcbe_program, 122 ni2_pcbe_allstop, 123 ni2_pcbe_sample, 124 ni2_pcbe_free 125 }; 126 127 typedef struct _ni2_pcbe_config { 128 uint_t pcbe_picno; /* 0 for pic0 or 1 for pic1 */ 129 uint32_t pcbe_evsel; /* %pcr event code unshifted */ 130 uint32_t pcbe_flags; /* hpriv/user/system/priv */ 131 uint32_t pcbe_pic; /* unshifted raw %pic value */ 132 } ni2_pcbe_config_t; 133 134 typedef struct _ni2_event { 135 const char *name; 136 const uint32_t emask; 137 const uint32_t emask_valid; /* Mask of unreserved MASK bits */ 138 } ni2_event_t; 139 140 typedef struct _ni2_generic_event { 141 char *name; 142 char *event; 143 } ni2_generic_event_t; 144 145 #define ULTRA_PCR_PRIVPIC (UINT64_C(1) << CPC_PCR_PRIV_SHIFT) 146 #define EV_END {NULL, 0, 0} 147 #define GEN_EV_END {NULL, NULL} 148 149 static const uint64_t allstopped = (ULTRA_PCR_PRIVPIC | 150 CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1); 151 152 /* 153 * We update this array in the program and allstop routine. The array 154 * is checked in the sample routine to allow us to only perform the 155 * PCR.ht bit check when counting is in progress. 156 */ 157 static boolean_t ni2_cpc_counting[NCPU]; 158 159 static ni2_event_t ni2_events[] = { 160 { "Idle_strands", 0x000, 0x00 }, 161 { "Br_completed", 0x201, 0xff }, 162 { "Br_taken", 0x202, 0xff }, 163 { "Instr_FGU_arithmetic", 0x204, 0xff }, 164 { "Instr_ld", 0x208, 0xff }, 165 { "Instr_st", 0x210, 0xff }, 166 { "Instr_sw", 0x220, 0xff }, 167 { "Instr_other", 0x240, 0xff }, 168 { "Atomics", 0x280, 0xff }, 169 { "Instr_cnt", 0x2fd, 0xff }, 170 { "IC_miss", 0x301, 0x33 }, 171 { "DC_miss", 0x302, 0x33 }, 172 { "L2_imiss", 0x310, 0x33 }, 173 { "L2_dmiss_ld", 0x320, 0x33 }, 174 { "ITLB_HWTW_ref_L2", 0x404, 0x3c }, 175 { "DTLB_HWTW_ref_L2", 0x408, 0x3c }, 176 { "ITLB_HWTW_miss_L2", 0x410, 0x3c }, 177 { "DTLB_HWTW_miss_L2", 0x420, 0x3c }, 178 { "Stream_ld_to_PCX", 0x501, 0x3f }, 179 { "Stream_st_to_PCX", 0x502, 0x3f }, 180 { "CPU_ld_to_PCX", 0x504, 0x3f }, 181 { "CPU_ifetch_to_PCX", 0x508, 0x3f }, 182 { "CPU_st_to_PCX", 0x510, 0x3f }, 183 { "MMU_ld_to_PCX", 0x520, 0x3f }, 184 #ifdef KT_IMPL 185 { "DES_3DES_op", 0x601, 0xff }, 186 { "AES_op", 0x602, 0xff }, 187 { "Kasumi_op", 0x604, 0xff }, 188 { "MD5_SHA-1_SHA-256_op", 0x608, 0xff }, 189 { "MA_op", 0x610, 0xff }, 190 { "CRC_TCPIP_cksum", 0x620, 0xff }, 191 { "DES_3DES_busy_cycle", 0x701, 0xff }, 192 { "AES_busy_cycle", 0x702, 0xff }, 193 { "Kasumi_busy_cycle", 0x704, 0xff }, 194 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0xff }, 195 { "MA_busy_cycle", 0x710, 0xff }, 196 { "CRC_MPA_cksum", 0x720, 0xff }, 197 #else 198 { "DES_3DES_op", 0x601, 0x3f }, 199 { "AES_op", 0x602, 0x3f }, 200 { "RC4_op", 0x604, 0x3f }, 201 { "MD5_SHA-1_SHA-256_op", 0x608, 0x3f }, 202 { "MA_op", 0x610, 0x3f }, 203 { "CRC_TCPIP_cksum", 0x620, 0x3f }, 204 { "DES_3DES_busy_cycle", 0x701, 0x3f }, 205 { "AES_busy_cycle", 0x702, 0x3f }, 206 { "RC4_busy_cycle", 0x704, 0x3f }, 207 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0x3f }, 208 { "MA_busy_cycle", 0x710, 0x3f }, 209 { "CRC_MPA_cksum", 0x720, 0x3f }, 210 #endif 211 { "ITLB_miss", 0xb04, 0x0c }, 212 { "DTLB_miss", 0xb08, 0x0c }, 213 { "TLB_miss", 0xb0c, 0x0c }, 214 EV_END 215 }; 216 217 static ni2_generic_event_t ni2_generic_events[] = { 218 { "PAPI_tot_ins", "Instr_cnt" }, 219 { "PAPI_l1_dcm", "DC_miss" }, 220 { "PAPI_l1_icm", "IC_miss" }, 221 { "PAPI_l2_icm", "L2_imiss" }, 222 { "PAPI_l2_ldm", "L2_dmiss_ld" }, 223 { "PAPI_tlb_dm", "DTLB_miss" }, 224 { "PAPI_tlb_im", "ITLB_miss" }, 225 { "PAPI_tlb_tm", "TLB_miss" }, 226 { "PAPI_br_tkn", "Br_taken" }, 227 { "PAPI_br_ins", "Br_completed" }, 228 { "PAPI_ld_ins", "Instr_ld" }, 229 { "PAPI_sr_ins", "Instr_st" }, 230 { "PAPI_fp_ops", "Instr_FGU_arithmetic" }, 231 { "PAPI_fp_ins", "Instr_FGU_arithmetic" }, 232 GEN_EV_END 233 }; 234 235 static char *evlist; 236 static size_t evlist_sz; 237 static uint16_t pcr_pic0_mask; 238 static uint16_t pcr_pic1_mask; 239 240 #define CPU_REF_URL " Documentation for Sun processors can be found at: " \ 241 "http://www.sun.com/processors/manuals" 242 243 #if defined(NIAGARA2_IMPL) 244 static const char *cpu_impl_name = "UltraSPARC T2"; 245 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2 User's Manual\" " 246 "for descriptions of these events." CPU_REF_URL; 247 #elif defined(VFALLS_IMPL) 248 static const char *cpu_impl_name = "UltraSPARC T2+"; 249 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2+ User's Manual\" " 250 "for descriptions of these events." CPU_REF_URL; 251 #elif defined(KT_IMPL) 252 static const char *cpu_impl_name = "UltraSPARC T3"; 253 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T3 User's Manual\" " 254 "for descriptions of these events." CPU_REF_URL; 255 #endif 256 257 static boolean_t cpu_hsvc_available = B_TRUE; 258 259 static int 260 ni2_pcbe_init(void) 261 { 262 ni2_event_t *evp; 263 ni2_generic_event_t *gevp; 264 int status; 265 uint64_t cpu_hsvc_major; 266 uint64_t cpu_hsvc_minor; 267 #if defined(NIAGARA2_IMPL) 268 uint64_t hsvc_cpu_group = HSVC_GROUP_NIAGARA2_CPU; 269 uint64_t hsvc_cpu_major = NIAGARA2_HSVC_MAJOR; 270 #elif defined(VFALLS_IMPL) 271 uint64_t hsvc_cpu_group = HSVC_GROUP_VFALLS_CPU; 272 uint64_t hsvc_cpu_major = VFALLS_HSVC_MAJOR; 273 #elif defined(KT_IMPL) 274 uint64_t hsvc_cpu_group = HSVC_GROUP_KT_CPU; 275 uint64_t hsvc_cpu_major = KT_HSVC_MAJOR; 276 #endif 277 278 pcr_pic0_mask = CPC_PCR_PIC0_MASK; 279 pcr_pic1_mask = CPC_PCR_PIC1_MASK; 280 281 /* 282 * Validate API version for Niagara2 specific hypervisor services 283 */ 284 status = hsvc_version(hsvc_cpu_group, &cpu_hsvc_major, 285 &cpu_hsvc_minor); 286 if ((status != 0) || (cpu_hsvc_major != hsvc_cpu_major)) { 287 cmn_err(CE_WARN, "hypervisor services not negotiated " 288 "or unsupported major number: group: 0x%lx major: 0x%lx " 289 "minor: 0x%lx errno: %d", hsvc_cpu_group, 290 cpu_hsvc_major, cpu_hsvc_minor, status); 291 cpu_hsvc_available = B_FALSE; 292 } 293 294 /* 295 * Construct event list. 296 * 297 * First pass: Calculate size needed. We'll need an additional byte 298 * for the NULL pointer during the last strcat. 299 * 300 * Second pass: Copy strings. 301 */ 302 for (evp = ni2_events; evp->name != NULL; evp++) 303 evlist_sz += strlen(evp->name) + 1; 304 305 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) 306 evlist_sz += strlen(gevp->name) + 1; 307 308 evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP); 309 evlist[0] = '\0'; 310 311 for (evp = ni2_events; evp->name != NULL; evp++) { 312 (void) strcat(evlist, evp->name); 313 (void) strcat(evlist, ","); 314 } 315 316 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) { 317 (void) strcat(evlist, gevp->name); 318 (void) strcat(evlist, ","); 319 } 320 321 /* 322 * Remove trailing comma. 323 */ 324 evlist[evlist_sz - 1] = '\0'; 325 326 return (0); 327 } 328 329 static uint_t 330 ni2_pcbe_ncounters(void) 331 { 332 return (2); 333 } 334 335 static const char * 336 ni2_pcbe_impl_name(void) 337 { 338 return (cpu_impl_name); 339 } 340 341 static const char * 342 ni2_pcbe_cpuref(void) 343 { 344 return (cpu_pcbe_ref); 345 } 346 347 static char * 348 ni2_pcbe_list_events(uint_t picnum) 349 { 350 ASSERT(picnum < cpc_ncounters); 351 352 return (evlist); 353 } 354 355 static char * 356 ni2_pcbe_list_attrs(void) 357 { 358 if (cpu_hsvc_available == B_TRUE) 359 #if defined(NIAGARA2_IMPL) 360 return ("hpriv,emask"); 361 #elif defined(VFALLS_IMPL) 362 return ("hpriv,l2ctl,emask"); 363 #elif defined(KT_IMPL) 364 return ("hpriv,l2ctl,emask,sample"); 365 #endif 366 else 367 #if defined(KT_IMPL) 368 return ("emask,sample"); 369 #else 370 return ("emask"); 371 #endif 372 } 373 374 static ni2_generic_event_t * 375 find_generic_event(char *name) 376 { 377 ni2_generic_event_t *gevp; 378 379 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) { 380 if (strcmp(name, gevp->name) == 0) 381 return (gevp); 382 } 383 384 return (NULL); 385 } 386 387 static ni2_event_t * 388 find_event(char *name) 389 { 390 ni2_event_t *evp; 391 392 for (evp = ni2_events; evp->name != NULL; evp++) 393 if (strcmp(name, evp->name) == 0) 394 return (evp); 395 396 return (NULL); 397 } 398 399 /*ARGSUSED*/ 400 static uint64_t 401 ni2_pcbe_event_coverage(char *event) 402 { 403 /* 404 * Check whether counter event is supported 405 */ 406 if (find_event(event) == NULL && find_generic_event(event) == NULL) 407 return (0); 408 409 /* 410 * Fortunately, both pic0 and pic1 can count all events. 411 */ 412 return (0x3); 413 } 414 415 static uint64_t 416 ni2_pcbe_overflow_bitmap(void) 417 { 418 uint64_t pcr, overflow; 419 uint64_t pic; 420 uint32_t pic0, pic1; 421 boolean_t update_pic = B_FALSE; 422 boolean_t pic_inrange = B_FALSE; 423 424 ASSERT(getpil() >= DISP_LEVEL); 425 pcr = ultra_getpcr(); 426 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 427 overflow = (pcr & CPC_PCR_OV0_MASK) >> 428 CPC_PCR_OV0_SHIFT; 429 overflow |= (pcr & CPC_PCR_OV1_MASK) >> 430 CPC_PCR_OV1_SHIFT; 431 432 pic = ultra_getpic(); 433 pic0 = (uint32_t)(pic & PIC0_MASK); 434 pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK); 435 436 pcr |= (CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1); 437 438 if (overflow & 0x1) { 439 pcr &= ~(CPC_PCR_OV0_MASK | 440 CPC_PCR_HOLDOV0); 441 pic_inrange = PIC_IN_OV_RANGE(pic0); 442 #if defined(KT_IMPL) 443 if (pcr & CPC_PCR_SAMPLE_MODE_MASK) 444 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic0); 445 #endif 446 if (pic_inrange) { 447 pic0 = 0; 448 update_pic = B_TRUE; 449 } 450 } 451 452 if (overflow & 0x2) { 453 pcr &= ~(CPC_PCR_OV1_MASK | 454 CPC_PCR_HOLDOV1); 455 pic_inrange = PIC_IN_OV_RANGE(pic1); 456 #if defined(KT_IMPL) 457 if (pcr & CPC_PCR_SAMPLE_MODE_MASK) 458 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic1); 459 #endif 460 if (pic_inrange) { 461 pic1 = 0; 462 update_pic = B_TRUE; 463 } 464 } 465 466 if (update_pic) 467 ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0); 468 469 /* 470 * The HV interface does not need to be used here because we are 471 * only resetting the OV bits and do not need to set the HT bit. 472 */ 473 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 474 ultra_setpcr(pcr); 475 476 return (overflow); 477 } 478 479 /*ARGSUSED*/ 480 static int 481 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags, 482 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token) 483 { 484 ni2_pcbe_config_t *cfg; 485 ni2_pcbe_config_t *other_config; 486 ni2_event_t *evp; 487 ni2_generic_event_t *gevp; 488 int i; 489 uint32_t evsel; 490 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 491 uint64_t l2ctl = 0; 492 #endif 493 494 /* 495 * If we've been handed an existing configuration, we need only preset 496 * the counter value. 497 */ 498 if (*data != NULL) { 499 cfg = *data; 500 cfg->pcbe_pic = (uint32_t)preset; 501 return (0); 502 } 503 504 if (picnum > 1) 505 return (CPC_INVALID_PICNUM); 506 507 508 if ((evp = find_event(event)) == NULL) { 509 if ((gevp = find_generic_event(event)) != NULL) { 510 evp = find_event(gevp->event); 511 ASSERT(evp != NULL); 512 513 if (nattrs > 0) 514 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 515 } else { 516 return (CPC_INVALID_EVENT); 517 } 518 } 519 520 evsel = evp->emask; 521 522 for (i = 0; i < nattrs; i++) { 523 if (strcmp(attrs[i].ka_name, "hpriv") == 0) { 524 if (attrs[i].ka_val != 0) 525 flags |= CPC_COUNT_HV; 526 } else if (strcmp(attrs[i].ka_name, "emask") == 0) { 527 if ((attrs[i].ka_val | evp->emask_valid) != 528 evp->emask_valid) 529 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 530 evsel |= attrs[i].ka_val; 531 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 532 } else if (strcmp(attrs[i].ka_name, "l2ctl") == 0) { 533 if ((attrs[i].ka_val | L2_CTL_MASK) != 534 L2_CTL_MASK) 535 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 536 else 537 l2ctl = attrs[i].ka_val; 538 #endif 539 #if defined(KT_IMPL) 540 } else if (strcmp(attrs[i].ka_name, "sample") == 0) { 541 if (attrs[i].ka_val != 0) 542 flags |= CPC_COUNT_SAMPLE_MODE; 543 #endif 544 } else 545 return (CPC_INVALID_ATTRIBUTE); 546 } 547 548 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 549 /* 550 * Set PERF_CONTROL bits in L2_CONTROL_REG only when events have 551 * SL bits equal to 3. 552 */ 553 if ((evsel & SL_MASK) == SL3_MASK) { 554 if ((hv_niagara_setperf(HV_L2_CTL, l2ctl)) != 0) 555 return (CPC_HV_NO_ACCESS); 556 } 557 #endif 558 559 /* 560 * Find other requests that will be programmed with this one, and ensure 561 * the flags don't conflict. 562 */ 563 if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) && 564 (other_config->pcbe_flags != flags)) 565 return (CPC_CONFLICTING_REQS); 566 567 /* 568 * If the hpriv attribute is present, make sure we have 569 * access to hyperprivileged events before continuing with 570 * this configuration. If we can set the ht bit in the PCR 571 * successfully, we must have access to hyperprivileged 572 * events. 573 * 574 * If this is a static per-CPU configuration, the CPC 575 * driver ensures there can not be more than one for this 576 * CPU. If this is a per-LWP configuration, the driver 577 * ensures no static per-CPU counting is ongoing and that 578 * the target LWP is not already being monitored. 579 */ 580 if (flags & CPC_COUNT_HV) { 581 kpreempt_disable(); 582 583 DTRACE_PROBE1(niagara2__setpcr, uint64_t, 584 allstopped | CPC_PCR_HT); 585 if (hv_niagara_setperf(HV_SPARC_CTL, 586 allstopped | CPC_PCR_HT) != H_EOK) { 587 kpreempt_enable(); 588 return (CPC_HV_NO_ACCESS); 589 } 590 591 DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped); 592 (void) hv_niagara_setperf(HV_SPARC_CTL, allstopped); 593 594 kpreempt_enable(); 595 } 596 597 cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP); 598 599 cfg->pcbe_picno = picnum; 600 cfg->pcbe_evsel = evsel; 601 cfg->pcbe_flags = flags; 602 cfg->pcbe_pic = (uint32_t)preset; 603 604 *data = cfg; 605 return (0); 606 } 607 608 static void 609 ni2_pcbe_program(void *token) 610 { 611 ni2_pcbe_config_t *pic0; 612 ni2_pcbe_config_t *pic1; 613 ni2_pcbe_config_t *tmp; 614 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 615 uint64_t pcr; 616 uint64_t curpic; 617 uint64_t toe; 618 619 /* enable trap-on-event for pic0 and pic1 */ 620 toe = (CPC_PCR_TOE0 | CPC_PCR_TOE1); 621 622 if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) == 623 NULL) 624 panic("ni2_pcbe: token %p has no configs", token); 625 626 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) { 627 pic1 = &nullcfg; 628 nullcfg.pcbe_flags = pic0->pcbe_flags; 629 toe = CPC_PCR_TOE0; /* enable trap-on-event for pic0 */ 630 } 631 632 if (pic0->pcbe_picno != 0) { 633 /* 634 * pic0 is counter 1, so if we need the null config it should 635 * be counter 0. 636 */ 637 nullcfg.pcbe_picno = 0; 638 tmp = pic0; 639 pic0 = pic1; 640 pic1 = tmp; 641 toe = CPC_PCR_TOE1; /* enable trap-on-event for pic1 */ 642 } 643 644 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 645 panic("%s: bad config on token %p\n", cpu_impl_name, token); 646 647 /* 648 * UltraSPARC does not allow pic0 to be configured differently 649 * from pic1. If the flags on these two configurations are 650 * different, they are incompatible. This condition should be 651 * caught at configure time. 652 */ 653 ASSERT(pic0->pcbe_flags == pic1->pcbe_flags); 654 655 ni2_pcbe_allstop(); 656 657 ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) | 658 (uint64_t)pic0->pcbe_pic); 659 660 pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_PCR_PIC0_SHIFT; 661 pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) << 662 CPC_PCR_PIC1_SHIFT; 663 664 if (pic0->pcbe_flags & CPC_COUNT_USER) 665 pcr |= (1ull << CPC_PCR_UT_SHIFT); 666 if (pic0->pcbe_flags & CPC_COUNT_SYSTEM) 667 pcr |= (1ull << CPC_PCR_ST_SHIFT); 668 if (pic0->pcbe_flags & CPC_COUNT_HV) 669 pcr |= (1ull << CPC_PCR_HT_SHIFT); 670 #if defined(KT_IMPL) 671 if (pic0->pcbe_flags & CPC_COUNT_SAMPLE_MODE) 672 pcr |= (1ull << CPC_PCR_SAMPLE_MODE_SHIFT); 673 #endif 674 675 if (pcbe_l2dmiss_ovtrap_enable == 0) { 676 /* 677 * SW workaround for HW Erratum 108. 678 * Disable overflow interrupts when L2_dmiss_ld event is 679 * selected. 680 */ 681 if ((pic0->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event) 682 toe &= ~CPC_PCR_TOE0; 683 684 if ((pic1->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event) 685 toe &= ~CPC_PCR_TOE1; 686 } 687 pcr |= toe; 688 689 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 690 691 if (pic0->pcbe_flags & CPC_COUNT_HV) { 692 /* 693 * The ht bit in the PCR is only writable in 694 * hyperprivileged mode. So if we are counting 695 * hpriv events, we must use the HV interface 696 * hv_niagara_setperf to set the PCR. If this 697 * fails, assume we no longer have access to 698 * hpriv events. 699 */ 700 if (hv_niagara_setperf(HV_SPARC_CTL, pcr) != H_EOK) { 701 kcpc_invalidate_config(token); 702 return; 703 } 704 } else 705 /* Set the PCR with no hpriv event counting enabled. */ 706 ultra_setpcr(pcr); 707 708 ni2_cpc_counting[CPU->cpu_id] = B_TRUE; 709 710 /* 711 * On UltraSPARC, only read-to-read counts are accurate. We cannot 712 * expect the value we wrote into the PIC, above, to be there after 713 * starting the counter. We must sample the counter value now and use 714 * that as the baseline for future samples. 715 */ 716 curpic = ultra_getpic(); 717 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 718 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 719 720 DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic); 721 } 722 723 static void 724 ni2_pcbe_allstop(void) 725 { 726 /* 727 * We use the HV interface here because if we were counting 728 * hyperprivileged events, we must reset the PCR.ht bit to stop 729 * the counting. In the event that this HV call fails, we fall 730 * back on ultra_setpcr which does not have write access to the 731 * ht bit. 732 */ 733 if (hv_niagara_setperf(HV_SPARC_CTL, allstopped) != H_EOK) 734 ultra_setpcr(allstopped); 735 736 ni2_cpc_counting[CPU->cpu_id] = B_FALSE; 737 } 738 739 static void 740 ni2_pcbe_sample(void *token) 741 { 742 uint64_t curpic; 743 int64_t diff; 744 uint64_t *pic0_data; 745 uint64_t *pic1_data; 746 uint64_t *dtmp; 747 uint64_t tmp; 748 uint64_t pcr; 749 ni2_pcbe_config_t *pic0; 750 ni2_pcbe_config_t *pic1; 751 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 752 ni2_pcbe_config_t *ctmp; 753 754 curpic = ultra_getpic(); 755 DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic); 756 757 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL) 758 panic("%s: token %p has no configs", cpu_impl_name, token); 759 760 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) { 761 pic1 = &nullcfg; 762 pic1_data = &tmp; 763 } 764 765 if (pic0->pcbe_picno != 0) { 766 nullcfg.pcbe_picno = 0; 767 ctmp = pic0; 768 pic0 = pic1; 769 pic1 = ctmp; 770 dtmp = pic0_data; 771 pic0_data = pic1_data; 772 pic1_data = dtmp; 773 } 774 775 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 776 panic("%s: bad config on token %p\n", cpu_impl_name, token); 777 778 779 if (pic0->pcbe_flags & CPC_COUNT_HV) { 780 /* 781 * If the hpriv attribute is present, but the HT bit 782 * is not set in the PCR, access to hyperprivileged 783 * events must have been revoked. Only perform this 784 * check if counting is not stopped. 785 */ 786 pcr = ultra_getpcr(); 787 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 788 if (ni2_cpc_counting[CPU->cpu_id] && 789 !(pcr & CPC_PCR_HT)) { 790 kcpc_invalidate_config(token); 791 return; 792 } 793 } 794 795 diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic; 796 if (diff < 0) 797 diff += (1ll << 32); 798 *pic0_data += diff; 799 800 diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic; 801 if (diff < 0) 802 diff += (1ll << 32); 803 *pic1_data += diff; 804 805 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 806 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 807 } 808 809 static void 810 ni2_pcbe_free(void *config) 811 { 812 kmem_free(config, sizeof (ni2_pcbe_config_t)); 813 } 814 815 816 static struct modlpcbe modlpcbe = { 817 &mod_pcbeops, 818 #if defined(NIAGARA2_IMPL) 819 "UltraSPARC T2 Performance Counters", 820 #elif defined(VFALLS_IMPL) 821 "UltraSPARC T2+ Performance Counters", 822 #elif defined(KT_IMPL) 823 "UltraSPARC T3 Performance Counters", 824 #endif 825 &ni2_pcbe_ops 826 }; 827 828 static struct modlinkage modl = { 829 MODREV_1, 830 &modlpcbe, 831 }; 832 833 int 834 _init(void) 835 { 836 if (ni2_pcbe_init() != 0) 837 return (ENOTSUP); 838 return (mod_install(&modl)); 839 } 840 841 int 842 _fini(void) 843 { 844 return (mod_remove(&modl)); 845 } 846 847 int 848 _info(struct modinfo *mi) 849 { 850 return (mod_info(&modl, mi)); 851 } 852