1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * This file contains preset event names from the Performance Application 28 * Programming Interface v3.5 which included the following notice: 29 * 30 * Copyright (c) 2005,6 31 * Innovative Computing Labs 32 * Computer Science Department, 33 * University of Tennessee, 34 * Knoxville, TN. 35 * All Rights Reserved. 36 * 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions are met: 40 * 41 * * Redistributions of source code must retain the above copyright notice, 42 * this list of conditions and the following disclaimer. 43 * * Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * * Neither the name of the University of Tennessee nor the names of its 47 * contributors may be used to endorse or promote products derived from 48 * this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 51 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 54 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 60 * POSSIBILITY OF SUCH DAMAGE. 61 * 62 * 63 * This open source software license conforms to the BSD License template. 64 */ 65 66 /* 67 * Niagara2 Performance Counter Backend 68 */ 69 70 #include <sys/cpuvar.h> 71 #include <sys/systm.h> 72 #include <sys/archsystm.h> 73 #include <sys/cmn_err.h> 74 #include <sys/cpc_impl.h> 75 #include <sys/cpc_pcbe.h> 76 #include <sys/modctl.h> 77 #include <sys/machsystm.h> 78 #include <sys/sdt.h> 79 #include <sys/niagara2regs.h> 80 #include <sys/hsvc.h> 81 #include <sys/hypervisor_api.h> 82 #include <sys/disp.h> 83 84 /*LINTLIBRARY*/ 85 static int ni2_pcbe_init(void); 86 static uint_t ni2_pcbe_ncounters(void); 87 static const char *ni2_pcbe_impl_name(void); 88 static const char *ni2_pcbe_cpuref(void); 89 static char *ni2_pcbe_list_events(uint_t picnum); 90 static char *ni2_pcbe_list_attrs(void); 91 static uint64_t ni2_pcbe_event_coverage(char *event); 92 static uint64_t ni2_pcbe_overflow_bitmap(void); 93 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, 94 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data, 95 void *token); 96 static void ni2_pcbe_program(void *token); 97 static void ni2_pcbe_allstop(void); 98 static void ni2_pcbe_sample(void *token); 99 static void ni2_pcbe_free(void *config); 100 101 extern void ultra_setpcr(uint64_t); 102 extern uint64_t ultra_getpcr(void); 103 extern void ultra_setpic(uint64_t); 104 extern uint64_t ultra_getpic(void); 105 extern uint64_t ultra_gettick(void); 106 extern char cpu_module_name[]; 107 108 #define L2_dmiss_ld_event 0x320 109 int pcbe_l2dmiss_ovtrap_enable = 0; 110 111 pcbe_ops_t ni2_pcbe_ops = { 112 PCBE_VER_1, 113 CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE, 114 ni2_pcbe_ncounters, 115 ni2_pcbe_impl_name, 116 ni2_pcbe_cpuref, 117 ni2_pcbe_list_events, 118 ni2_pcbe_list_attrs, 119 ni2_pcbe_event_coverage, 120 ni2_pcbe_overflow_bitmap, 121 ni2_pcbe_configure, 122 ni2_pcbe_program, 123 ni2_pcbe_allstop, 124 ni2_pcbe_sample, 125 ni2_pcbe_free 126 }; 127 128 typedef struct _ni2_pcbe_config { 129 uint_t pcbe_picno; /* 0 for pic0 or 1 for pic1 */ 130 uint32_t pcbe_evsel; /* %pcr event code unshifted */ 131 uint32_t pcbe_flags; /* hpriv/user/system/priv */ 132 uint32_t pcbe_pic; /* unshifted raw %pic value */ 133 } ni2_pcbe_config_t; 134 135 typedef struct _ni2_event { 136 const char *name; 137 const uint32_t emask; 138 const uint32_t emask_valid; /* Mask of unreserved MASK bits */ 139 } ni2_event_t; 140 141 typedef struct _ni2_generic_event { 142 char *name; 143 char *event; 144 } ni2_generic_event_t; 145 146 #define ULTRA_PCR_PRIVPIC (UINT64_C(1) << CPC_PCR_PRIV_SHIFT) 147 #define EV_END {NULL, 0, 0} 148 #define GEN_EV_END {NULL, NULL} 149 150 static const uint64_t allstopped = (ULTRA_PCR_PRIVPIC | 151 CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1); 152 153 /* 154 * We update this array in the program and allstop routine. The array 155 * is checked in the sample routine to allow us to only perform the 156 * PCR.ht bit check when counting is in progress. 157 */ 158 static boolean_t ni2_cpc_counting[NCPU]; 159 160 static ni2_event_t ni2_events[] = { 161 { "Idle_strands", 0x000, 0x00 }, 162 { "Br_completed", 0x201, 0xff }, 163 { "Br_taken", 0x202, 0xff }, 164 { "Instr_FGU_arithmetic", 0x204, 0xff }, 165 { "Instr_ld", 0x208, 0xff }, 166 { "Instr_st", 0x210, 0xff }, 167 { "Instr_sw", 0x220, 0xff }, 168 { "Instr_other", 0x240, 0xff }, 169 { "Atomics", 0x280, 0xff }, 170 { "Instr_cnt", 0x2fd, 0xff }, 171 { "IC_miss", 0x301, 0x33 }, 172 { "DC_miss", 0x302, 0x33 }, 173 { "L2_imiss", 0x310, 0x33 }, 174 { "L2_dmiss_ld", 0x320, 0x33 }, 175 { "ITLB_HWTW_ref_L2", 0x404, 0x3c }, 176 { "DTLB_HWTW_ref_L2", 0x408, 0x3c }, 177 { "ITLB_HWTW_miss_L2", 0x410, 0x3c }, 178 { "DTLB_HWTW_miss_L2", 0x420, 0x3c }, 179 { "Stream_ld_to_PCX", 0x501, 0x3f }, 180 { "Stream_st_to_PCX", 0x502, 0x3f }, 181 { "CPU_ld_to_PCX", 0x504, 0x3f }, 182 { "CPU_ifetch_to_PCX", 0x508, 0x3f }, 183 { "CPU_st_to_PCX", 0x510, 0x3f }, 184 { "MMU_ld_to_PCX", 0x520, 0x3f }, 185 #ifdef KT_IMPL 186 { "DES_3DES_op", 0x601, 0xff }, 187 { "AES_op", 0x602, 0xff }, 188 { "Kasumi_op", 0x604, 0xff }, 189 { "MD5_SHA-1_SHA-256_op", 0x608, 0xff }, 190 { "MA_op", 0x610, 0xff }, 191 { "CRC_TCPIP_cksum", 0x620, 0xff }, 192 { "DES_3DES_busy_cycle", 0x701, 0xff }, 193 { "AES_busy_cycle", 0x702, 0xff }, 194 { "Kasumi_busy_cycle", 0x704, 0xff }, 195 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0xff }, 196 { "MA_busy_cycle", 0x710, 0xff }, 197 { "CRC_MPA_cksum", 0x720, 0xff }, 198 #else 199 { "DES_3DES_op", 0x601, 0x3f }, 200 { "AES_op", 0x602, 0x3f }, 201 { "RC4_op", 0x604, 0x3f }, 202 { "MD5_SHA-1_SHA-256_op", 0x608, 0x3f }, 203 { "MA_op", 0x610, 0x3f }, 204 { "CRC_TCPIP_cksum", 0x620, 0x3f }, 205 { "DES_3DES_busy_cycle", 0x701, 0x3f }, 206 { "AES_busy_cycle", 0x702, 0x3f }, 207 { "RC4_busy_cycle", 0x704, 0x3f }, 208 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0x3f }, 209 { "MA_busy_cycle", 0x710, 0x3f }, 210 { "CRC_MPA_cksum", 0x720, 0x3f }, 211 #endif 212 { "ITLB_miss", 0xb04, 0x0c }, 213 { "DTLB_miss", 0xb08, 0x0c }, 214 { "TLB_miss", 0xb0c, 0x0c }, 215 EV_END 216 }; 217 218 static ni2_generic_event_t ni2_generic_events[] = { 219 { "PAPI_tot_ins", "Instr_cnt" }, 220 { "PAPI_l1_dcm", "DC_miss" }, 221 { "PAPI_l1_icm", "IC_miss" }, 222 { "PAPI_l2_icm", "L2_imiss" }, 223 { "PAPI_l2_ldm", "L2_dmiss_ld" }, 224 { "PAPI_tlb_dm", "DTLB_miss" }, 225 { "PAPI_tlb_im", "ITLB_miss" }, 226 { "PAPI_tlb_tm", "TLB_miss" }, 227 { "PAPI_br_tkn", "Br_taken" }, 228 { "PAPI_br_ins", "Br_completed" }, 229 { "PAPI_ld_ins", "Instr_ld" }, 230 { "PAPI_sr_ins", "Instr_st" }, 231 { "PAPI_fp_ops", "Instr_FGU_arithmetic" }, 232 { "PAPI_fp_ins", "Instr_FGU_arithmetic" }, 233 GEN_EV_END 234 }; 235 236 static char *evlist; 237 static size_t evlist_sz; 238 static uint16_t pcr_pic0_mask; 239 static uint16_t pcr_pic1_mask; 240 241 #define CPU_REF_URL " Documentation for Sun processors can be found at: " \ 242 "http://www.sun.com/processors/manuals" 243 244 #if defined(NIAGARA2_IMPL) 245 static const char *cpu_impl_name = "UltraSPARC T2"; 246 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2 User's Manual\" " 247 "for descriptions of these events." CPU_REF_URL; 248 #elif defined(VFALLS_IMPL) 249 static const char *cpu_impl_name = "UltraSPARC T2+"; 250 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2+ User's Manual\" " 251 "for descriptions of these events." CPU_REF_URL; 252 #elif defined(KT_IMPL) 253 static const char *cpu_impl_name = "UltraSPARC T3"; 254 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T3 User's Manual\" " 255 "for descriptions of these events." CPU_REF_URL; 256 #endif 257 258 static boolean_t cpu_hsvc_available = B_TRUE; 259 260 static int 261 ni2_pcbe_init(void) 262 { 263 ni2_event_t *evp; 264 ni2_generic_event_t *gevp; 265 int status; 266 uint64_t cpu_hsvc_major; 267 uint64_t cpu_hsvc_minor; 268 #if defined(NIAGARA2_IMPL) 269 uint64_t hsvc_cpu_group = HSVC_GROUP_NIAGARA2_CPU; 270 uint64_t hsvc_cpu_major = NIAGARA2_HSVC_MAJOR; 271 #elif defined(VFALLS_IMPL) 272 uint64_t hsvc_cpu_group = HSVC_GROUP_VFALLS_CPU; 273 uint64_t hsvc_cpu_major = VFALLS_HSVC_MAJOR; 274 #elif defined(KT_IMPL) 275 uint64_t hsvc_cpu_group = HSVC_GROUP_KT_CPU; 276 uint64_t hsvc_cpu_major = KT_HSVC_MAJOR; 277 #endif 278 279 pcr_pic0_mask = CPC_PCR_PIC0_MASK; 280 pcr_pic1_mask = CPC_PCR_PIC1_MASK; 281 282 /* 283 * Validate API version for Niagara2 specific hypervisor services 284 */ 285 status = hsvc_version(hsvc_cpu_group, &cpu_hsvc_major, 286 &cpu_hsvc_minor); 287 if ((status != 0) || (cpu_hsvc_major != hsvc_cpu_major)) { 288 cmn_err(CE_WARN, "hypervisor services not negotiated " 289 "or unsupported major number: group: 0x%lx major: 0x%lx " 290 "minor: 0x%lx errno: %d", hsvc_cpu_group, 291 cpu_hsvc_major, cpu_hsvc_minor, status); 292 cpu_hsvc_available = B_FALSE; 293 } 294 295 /* 296 * Construct event list. 297 * 298 * First pass: Calculate size needed. We'll need an additional byte 299 * for the NULL pointer during the last strcat. 300 * 301 * Second pass: Copy strings. 302 */ 303 for (evp = ni2_events; evp->name != NULL; evp++) 304 evlist_sz += strlen(evp->name) + 1; 305 306 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) 307 evlist_sz += strlen(gevp->name) + 1; 308 309 evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP); 310 evlist[0] = '\0'; 311 312 for (evp = ni2_events; evp->name != NULL; evp++) { 313 (void) strcat(evlist, evp->name); 314 (void) strcat(evlist, ","); 315 } 316 317 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) { 318 (void) strcat(evlist, gevp->name); 319 (void) strcat(evlist, ","); 320 } 321 322 /* 323 * Remove trailing comma. 324 */ 325 evlist[evlist_sz - 1] = '\0'; 326 327 return (0); 328 } 329 330 static uint_t 331 ni2_pcbe_ncounters(void) 332 { 333 return (2); 334 } 335 336 static const char * 337 ni2_pcbe_impl_name(void) 338 { 339 return (cpu_impl_name); 340 } 341 342 static const char * 343 ni2_pcbe_cpuref(void) 344 { 345 return (cpu_pcbe_ref); 346 } 347 348 static char * 349 ni2_pcbe_list_events(uint_t picnum) 350 { 351 ASSERT(picnum < cpc_ncounters); 352 353 return (evlist); 354 } 355 356 static char * 357 ni2_pcbe_list_attrs(void) 358 { 359 if (cpu_hsvc_available == B_TRUE) 360 #if defined(NIAGARA2_IMPL) 361 return ("hpriv,emask"); 362 #elif defined(VFALLS_IMPL) 363 return ("hpriv,l2ctl,emask"); 364 #elif defined(KT_IMPL) 365 return ("hpriv,l2ctl,emask,sample"); 366 #endif 367 else 368 #if defined(KT_IMPL) 369 return ("emask,sample"); 370 #else 371 return ("emask"); 372 #endif 373 } 374 375 static ni2_generic_event_t * 376 find_generic_event(char *name) 377 { 378 ni2_generic_event_t *gevp; 379 380 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) { 381 if (strcmp(name, gevp->name) == 0) 382 return (gevp); 383 } 384 385 return (NULL); 386 } 387 388 static ni2_event_t * 389 find_event(char *name) 390 { 391 ni2_event_t *evp; 392 393 for (evp = ni2_events; evp->name != NULL; evp++) 394 if (strcmp(name, evp->name) == 0) 395 return (evp); 396 397 return (NULL); 398 } 399 400 /*ARGSUSED*/ 401 static uint64_t 402 ni2_pcbe_event_coverage(char *event) 403 { 404 /* 405 * Check whether counter event is supported 406 */ 407 if (find_event(event) == NULL && find_generic_event(event) == NULL) 408 return (0); 409 410 /* 411 * Fortunately, both pic0 and pic1 can count all events. 412 */ 413 return (0x3); 414 } 415 416 static uint64_t 417 ni2_pcbe_overflow_bitmap(void) 418 { 419 uint64_t pcr, overflow; 420 uint64_t pic; 421 uint32_t pic0, pic1; 422 boolean_t update_pic = B_FALSE; 423 boolean_t pic_inrange = B_FALSE; 424 425 ASSERT(getpil() >= DISP_LEVEL); 426 pcr = ultra_getpcr(); 427 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 428 overflow = (pcr & CPC_PCR_OV0_MASK) >> 429 CPC_PCR_OV0_SHIFT; 430 overflow |= (pcr & CPC_PCR_OV1_MASK) >> 431 CPC_PCR_OV1_SHIFT; 432 433 pic = ultra_getpic(); 434 pic0 = (uint32_t)(pic & PIC0_MASK); 435 pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK); 436 437 pcr |= (CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1); 438 439 if (overflow & 0x1) { 440 pcr &= ~(CPC_PCR_OV0_MASK | 441 CPC_PCR_HOLDOV0); 442 pic_inrange = PIC_IN_OV_RANGE(pic0); 443 #if defined(KT_IMPL) 444 if (pcr & CPC_PCR_SAMPLE_MODE_MASK) 445 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic0); 446 #endif 447 if (pic_inrange) { 448 pic0 = 0; 449 update_pic = B_TRUE; 450 } 451 } 452 453 if (overflow & 0x2) { 454 pcr &= ~(CPC_PCR_OV1_MASK | 455 CPC_PCR_HOLDOV1); 456 pic_inrange = PIC_IN_OV_RANGE(pic1); 457 #if defined(KT_IMPL) 458 if (pcr & CPC_PCR_SAMPLE_MODE_MASK) 459 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic1); 460 #endif 461 if (pic_inrange) { 462 pic1 = 0; 463 update_pic = B_TRUE; 464 } 465 } 466 467 if (update_pic) 468 ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0); 469 470 /* 471 * The HV interface does not need to be used here because we are 472 * only resetting the OV bits and do not need to set the HT bit. 473 */ 474 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 475 ultra_setpcr(pcr); 476 477 return (overflow); 478 } 479 480 /*ARGSUSED*/ 481 static int 482 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags, 483 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token) 484 { 485 ni2_pcbe_config_t *cfg; 486 ni2_pcbe_config_t *other_config; 487 ni2_event_t *evp; 488 ni2_generic_event_t *gevp; 489 int i; 490 uint32_t evsel; 491 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 492 uint64_t l2ctl = 0; 493 #endif 494 495 /* 496 * If we've been handed an existing configuration, we need only preset 497 * the counter value. 498 */ 499 if (*data != NULL) { 500 cfg = *data; 501 cfg->pcbe_pic = (uint32_t)preset; 502 return (0); 503 } 504 505 if (picnum > 1) 506 return (CPC_INVALID_PICNUM); 507 508 509 if ((evp = find_event(event)) == NULL) { 510 if ((gevp = find_generic_event(event)) != NULL) { 511 evp = find_event(gevp->event); 512 ASSERT(evp != NULL); 513 514 if (nattrs > 0) 515 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 516 } else { 517 return (CPC_INVALID_EVENT); 518 } 519 } 520 521 evsel = evp->emask; 522 523 for (i = 0; i < nattrs; i++) { 524 if (strcmp(attrs[i].ka_name, "hpriv") == 0) { 525 if (attrs[i].ka_val != 0) 526 flags |= CPC_COUNT_HV; 527 } else if (strcmp(attrs[i].ka_name, "emask") == 0) { 528 if ((attrs[i].ka_val | evp->emask_valid) != 529 evp->emask_valid) 530 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 531 evsel |= attrs[i].ka_val; 532 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 533 } else if (strcmp(attrs[i].ka_name, "l2ctl") == 0) { 534 if ((attrs[i].ka_val | L2_CTL_MASK) != 535 L2_CTL_MASK) 536 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 537 else 538 l2ctl = attrs[i].ka_val; 539 #endif 540 #if defined(KT_IMPL) 541 } else if (strcmp(attrs[i].ka_name, "sample") == 0) { 542 if (attrs[i].ka_val != 0) 543 flags |= CPC_COUNT_SAMPLE_MODE; 544 #endif 545 } else 546 return (CPC_INVALID_ATTRIBUTE); 547 } 548 549 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 550 /* 551 * Set PERF_CONTROL bits in L2_CONTROL_REG only when events have 552 * SL bits equal to 3. 553 */ 554 if ((evsel & SL3_MASK) == SL3_MASK) { 555 if ((hv_niagara_setperf(HV_L2_CTL, l2ctl)) != 0) 556 return (CPC_HV_NO_ACCESS); 557 } 558 #endif 559 560 /* 561 * Find other requests that will be programmed with this one, and ensure 562 * the flags don't conflict. 563 */ 564 if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) && 565 (other_config->pcbe_flags != flags)) 566 return (CPC_CONFLICTING_REQS); 567 568 /* 569 * If the hpriv attribute is present, make sure we have 570 * access to hyperprivileged events before continuing with 571 * this configuration. If we can set the ht bit in the PCR 572 * successfully, we must have access to hyperprivileged 573 * events. 574 * 575 * If this is a static per-CPU configuration, the CPC 576 * driver ensures there can not be more than one for this 577 * CPU. If this is a per-LWP configuration, the driver 578 * ensures no static per-CPU counting is ongoing and that 579 * the target LWP is not already being monitored. 580 */ 581 if (flags & CPC_COUNT_HV) { 582 kpreempt_disable(); 583 584 DTRACE_PROBE1(niagara2__setpcr, uint64_t, 585 allstopped | CPC_PCR_HT); 586 if (hv_niagara_setperf(HV_SPARC_CTL, 587 allstopped | CPC_PCR_HT) != H_EOK) { 588 kpreempt_enable(); 589 return (CPC_HV_NO_ACCESS); 590 } 591 592 DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped); 593 (void) hv_niagara_setperf(HV_SPARC_CTL, allstopped); 594 595 kpreempt_enable(); 596 } 597 598 cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP); 599 600 cfg->pcbe_picno = picnum; 601 cfg->pcbe_evsel = evsel; 602 cfg->pcbe_flags = flags; 603 cfg->pcbe_pic = (uint32_t)preset; 604 605 *data = cfg; 606 return (0); 607 } 608 609 static void 610 ni2_pcbe_program(void *token) 611 { 612 ni2_pcbe_config_t *pic0; 613 ni2_pcbe_config_t *pic1; 614 ni2_pcbe_config_t *tmp; 615 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 616 uint64_t pcr; 617 uint64_t curpic; 618 uint64_t toe; 619 620 /* enable trap-on-event for pic0 and pic1 */ 621 toe = (CPC_PCR_TOE0 | CPC_PCR_TOE1); 622 623 if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) == 624 NULL) 625 panic("ni2_pcbe: token %p has no configs", token); 626 627 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) { 628 pic1 = &nullcfg; 629 nullcfg.pcbe_flags = pic0->pcbe_flags; 630 toe = CPC_PCR_TOE0; /* enable trap-on-event for pic0 */ 631 } 632 633 if (pic0->pcbe_picno != 0) { 634 /* 635 * pic0 is counter 1, so if we need the null config it should 636 * be counter 0. 637 */ 638 nullcfg.pcbe_picno = 0; 639 tmp = pic0; 640 pic0 = pic1; 641 pic1 = tmp; 642 toe = CPC_PCR_TOE1; /* enable trap-on-event for pic1 */ 643 } 644 645 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 646 panic("%s: bad config on token %p\n", cpu_impl_name, token); 647 648 /* 649 * UltraSPARC does not allow pic0 to be configured differently 650 * from pic1. If the flags on these two configurations are 651 * different, they are incompatible. This condition should be 652 * caught at configure time. 653 */ 654 ASSERT(pic0->pcbe_flags == pic1->pcbe_flags); 655 656 ni2_pcbe_allstop(); 657 658 ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) | 659 (uint64_t)pic0->pcbe_pic); 660 661 pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_PCR_PIC0_SHIFT; 662 pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) << 663 CPC_PCR_PIC1_SHIFT; 664 665 if (pic0->pcbe_flags & CPC_COUNT_USER) 666 pcr |= (1ull << CPC_PCR_UT_SHIFT); 667 if (pic0->pcbe_flags & CPC_COUNT_SYSTEM) 668 pcr |= (1ull << CPC_PCR_ST_SHIFT); 669 if (pic0->pcbe_flags & CPC_COUNT_HV) 670 pcr |= (1ull << CPC_PCR_HT_SHIFT); 671 #if defined(KT_IMPL) 672 if (pic0->pcbe_flags & CPC_COUNT_SAMPLE_MODE) 673 pcr |= (1ull << CPC_PCR_SAMPLE_MODE_SHIFT); 674 #endif 675 676 if (pcbe_l2dmiss_ovtrap_enable == 0) { 677 /* 678 * SW workaround for HW Erratum 108. 679 * Disable overflow interrupts when L2_dmiss_ld event is 680 * selected. 681 */ 682 if ((pic0->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event) 683 toe &= ~CPC_PCR_TOE0; 684 685 if ((pic1->pcbe_evsel & L2_dmiss_ld_event) == L2_dmiss_ld_event) 686 toe &= ~CPC_PCR_TOE1; 687 } 688 pcr |= toe; 689 690 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 691 692 if (pic0->pcbe_flags & CPC_COUNT_HV) { 693 /* 694 * The ht bit in the PCR is only writable in 695 * hyperprivileged mode. So if we are counting 696 * hpriv events, we must use the HV interface 697 * hv_niagara_setperf to set the PCR. If this 698 * fails, assume we no longer have access to 699 * hpriv events. 700 */ 701 if (hv_niagara_setperf(HV_SPARC_CTL, pcr) != H_EOK) { 702 kcpc_invalidate_config(token); 703 return; 704 } 705 } else 706 /* Set the PCR with no hpriv event counting enabled. */ 707 ultra_setpcr(pcr); 708 709 ni2_cpc_counting[CPU->cpu_id] = B_TRUE; 710 711 /* 712 * On UltraSPARC, only read-to-read counts are accurate. We cannot 713 * expect the value we wrote into the PIC, above, to be there after 714 * starting the counter. We must sample the counter value now and use 715 * that as the baseline for future samples. 716 */ 717 curpic = ultra_getpic(); 718 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 719 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 720 721 DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic); 722 } 723 724 static void 725 ni2_pcbe_allstop(void) 726 { 727 /* 728 * We use the HV interface here because if we were counting 729 * hyperprivileged events, we must reset the PCR.ht bit to stop 730 * the counting. In the event that this HV call fails, we fall 731 * back on ultra_setpcr which does not have write access to the 732 * ht bit. 733 */ 734 if (hv_niagara_setperf(HV_SPARC_CTL, allstopped) != H_EOK) 735 ultra_setpcr(allstopped); 736 737 ni2_cpc_counting[CPU->cpu_id] = B_FALSE; 738 } 739 740 static void 741 ni2_pcbe_sample(void *token) 742 { 743 uint64_t curpic; 744 int64_t diff; 745 uint64_t *pic0_data; 746 uint64_t *pic1_data; 747 uint64_t *dtmp; 748 uint64_t tmp; 749 uint64_t pcr; 750 ni2_pcbe_config_t *pic0; 751 ni2_pcbe_config_t *pic1; 752 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 753 ni2_pcbe_config_t *ctmp; 754 755 curpic = ultra_getpic(); 756 DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic); 757 758 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL) 759 panic("%s: token %p has no configs", cpu_impl_name, token); 760 761 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) { 762 pic1 = &nullcfg; 763 pic1_data = &tmp; 764 } 765 766 if (pic0->pcbe_picno != 0) { 767 nullcfg.pcbe_picno = 0; 768 ctmp = pic0; 769 pic0 = pic1; 770 pic1 = ctmp; 771 dtmp = pic0_data; 772 pic0_data = pic1_data; 773 pic1_data = dtmp; 774 } 775 776 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 777 panic("%s: bad config on token %p\n", cpu_impl_name, token); 778 779 780 if (pic0->pcbe_flags & CPC_COUNT_HV) { 781 /* 782 * If the hpriv attribute is present, but the HT bit 783 * is not set in the PCR, access to hyperprivileged 784 * events must have been revoked. Only perform this 785 * check if counting is not stopped. 786 */ 787 pcr = ultra_getpcr(); 788 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 789 if (ni2_cpc_counting[CPU->cpu_id] && 790 !(pcr & CPC_PCR_HT)) { 791 kcpc_invalidate_config(token); 792 return; 793 } 794 } 795 796 diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic; 797 if (diff < 0) 798 diff += (1ll << 32); 799 *pic0_data += diff; 800 801 diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic; 802 if (diff < 0) 803 diff += (1ll << 32); 804 *pic1_data += diff; 805 806 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 807 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 808 } 809 810 static void 811 ni2_pcbe_free(void *config) 812 { 813 kmem_free(config, sizeof (ni2_pcbe_config_t)); 814 } 815 816 817 static struct modlpcbe modlpcbe = { 818 &mod_pcbeops, 819 #if defined(NIAGARA2_IMPL) 820 "UltraSPARC T2 Performance Counters", 821 #elif defined(VFALLS_IMPL) 822 "UltraSPARC T2+ Performance Counters", 823 #elif defined(KT_IMPL) 824 "UltraSPARC T3 Performance Counters", 825 #endif 826 &ni2_pcbe_ops 827 }; 828 829 static struct modlinkage modl = { 830 MODREV_1, 831 &modlpcbe, 832 }; 833 834 int 835 _init(void) 836 { 837 if (ni2_pcbe_init() != 0) 838 return (ENOTSUP); 839 return (mod_install(&modl)); 840 } 841 842 int 843 _fini(void) 844 { 845 return (mod_remove(&modl)); 846 } 847 848 int 849 _info(struct modinfo *mi) 850 { 851 return (mod_info(&modl, mi)); 852 } 853