1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * This file contains preset event names from the Performance Application 28 * Programming Interface v3.5 which included the following notice: 29 * 30 * Copyright (c) 2005,6 31 * Innovative Computing Labs 32 * Computer Science Department, 33 * University of Tennessee, 34 * Knoxville, TN. 35 * All Rights Reserved. 36 * 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions are met: 40 * 41 * * Redistributions of source code must retain the above copyright notice, 42 * this list of conditions and the following disclaimer. 43 * * Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * * Neither the name of the University of Tennessee nor the names of its 47 * contributors may be used to endorse or promote products derived from 48 * this software without specific prior written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 51 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 53 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 54 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 60 * POSSIBILITY OF SUCH DAMAGE. 61 * 62 * 63 * This open source software license conforms to the BSD License template. 64 */ 65 66 /* 67 * Niagara2 Performance Counter Backend 68 */ 69 70 #include <sys/cpuvar.h> 71 #include <sys/systm.h> 72 #include <sys/archsystm.h> 73 #include <sys/cmn_err.h> 74 #include <sys/cpc_impl.h> 75 #include <sys/cpc_pcbe.h> 76 #include <sys/modctl.h> 77 #include <sys/machsystm.h> 78 #include <sys/sdt.h> 79 #include <sys/niagara2regs.h> 80 #include <sys/hsvc.h> 81 #include <sys/hypervisor_api.h> 82 #include <sys/disp.h> 83 84 /*LINTLIBRARY*/ 85 static int ni2_pcbe_init(void); 86 static uint_t ni2_pcbe_ncounters(void); 87 static const char *ni2_pcbe_impl_name(void); 88 static const char *ni2_pcbe_cpuref(void); 89 static char *ni2_pcbe_list_events(uint_t picnum); 90 static char *ni2_pcbe_list_attrs(void); 91 static uint64_t ni2_pcbe_event_coverage(char *event); 92 static uint64_t ni2_pcbe_overflow_bitmap(void); 93 static int ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, 94 uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data, 95 void *token); 96 static void ni2_pcbe_program(void *token); 97 static void ni2_pcbe_allstop(void); 98 static void ni2_pcbe_sample(void *token); 99 static void ni2_pcbe_free(void *config); 100 101 extern void ultra_setpcr(uint64_t); 102 extern uint64_t ultra_getpcr(void); 103 extern void ultra_setpic(uint64_t); 104 extern uint64_t ultra_getpic(void); 105 extern uint64_t ultra_gettick(void); 106 extern char cpu_module_name[]; 107 108 pcbe_ops_t ni2_pcbe_ops = { 109 PCBE_VER_1, 110 CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE, 111 ni2_pcbe_ncounters, 112 ni2_pcbe_impl_name, 113 ni2_pcbe_cpuref, 114 ni2_pcbe_list_events, 115 ni2_pcbe_list_attrs, 116 ni2_pcbe_event_coverage, 117 ni2_pcbe_overflow_bitmap, 118 ni2_pcbe_configure, 119 ni2_pcbe_program, 120 ni2_pcbe_allstop, 121 ni2_pcbe_sample, 122 ni2_pcbe_free 123 }; 124 125 typedef struct _ni2_pcbe_config { 126 uint_t pcbe_picno; /* 0 for pic0 or 1 for pic1 */ 127 uint32_t pcbe_evsel; /* %pcr event code unshifted */ 128 uint32_t pcbe_flags; /* hpriv/user/system/priv */ 129 uint32_t pcbe_pic; /* unshifted raw %pic value */ 130 } ni2_pcbe_config_t; 131 132 typedef struct _ni2_event { 133 const char *name; 134 const uint32_t emask; 135 const uint32_t emask_valid; /* Mask of unreserved MASK bits */ 136 } ni2_event_t; 137 138 typedef struct _ni2_generic_event { 139 char *name; 140 char *event; 141 } ni2_generic_event_t; 142 143 #define ULTRA_PCR_PRIVPIC (UINT64_C(1) << CPC_PCR_PRIV_SHIFT) 144 #define EV_END {NULL, 0, 0} 145 #define GEN_EV_END {NULL, NULL} 146 147 static const uint64_t allstopped = (ULTRA_PCR_PRIVPIC | 148 CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1); 149 150 /* 151 * We update this array in the program and allstop routine. The array 152 * is checked in the sample routine to allow us to only perform the 153 * PCR.ht bit check when counting is in progress. 154 */ 155 static boolean_t ni2_cpc_counting[NCPU]; 156 157 static ni2_event_t ni2_events[] = { 158 { "Idle_strands", 0x000, 0x00 }, 159 { "Br_completed", 0x201, 0xff }, 160 { "Br_taken", 0x202, 0xff }, 161 { "Instr_FGU_arithmetic", 0x204, 0xff }, 162 { "Instr_ld", 0x208, 0xff }, 163 { "Instr_st", 0x210, 0xff }, 164 { "Instr_sw", 0x220, 0xff }, 165 { "Instr_other", 0x240, 0xff }, 166 { "Atomics", 0x280, 0xff }, 167 { "Instr_cnt", 0x2fd, 0xff }, 168 { "IC_miss", 0x301, 0x33 }, 169 { "DC_miss", 0x302, 0x33 }, 170 { "L2_imiss", 0x310, 0x33 }, 171 { "L2_dmiss_ld", 0x320, 0x33 }, 172 { "ITLB_HWTW_ref_L2", 0x404, 0x3c }, 173 { "DTLB_HWTW_ref_L2", 0x408, 0x3c }, 174 { "ITLB_HWTW_miss_L2", 0x410, 0x3c }, 175 { "DTLB_HWTW_miss_L2", 0x420, 0x3c }, 176 { "Stream_ld_to_PCX", 0x501, 0x3f }, 177 { "Stream_st_to_PCX", 0x502, 0x3f }, 178 { "CPU_ld_to_PCX", 0x504, 0x3f }, 179 { "CPU_ifetch_to_PCX", 0x508, 0x3f }, 180 { "CPU_st_to_PCX", 0x510, 0x3f }, 181 { "MMU_ld_to_PCX", 0x520, 0x3f }, 182 #ifdef KT_IMPL 183 { "DES_3DES_op", 0x601, 0xff }, 184 { "AES_op", 0x602, 0xff }, 185 { "Kasumi_op", 0x604, 0xff }, 186 { "MD5_SHA-1_SHA-256_op", 0x608, 0xff }, 187 { "MA_op", 0x610, 0xff }, 188 { "CRC_TCPIP_cksum", 0x620, 0xff }, 189 { "DES_3DES_busy_cycle", 0x701, 0xff }, 190 { "AES_busy_cycle", 0x702, 0xff }, 191 { "Kasumi_busy_cycle", 0x704, 0xff }, 192 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0xff }, 193 { "MA_busy_cycle", 0x710, 0xff }, 194 { "CRC_MPA_cksum", 0x720, 0xff }, 195 #else 196 { "DES_3DES_op", 0x601, 0x3f }, 197 { "AES_op", 0x602, 0x3f }, 198 { "RC4_op", 0x604, 0x3f }, 199 { "MD5_SHA-1_SHA-256_op", 0x608, 0x3f }, 200 { "MA_op", 0x610, 0x3f }, 201 { "CRC_TCPIP_cksum", 0x620, 0x3f }, 202 { "DES_3DES_busy_cycle", 0x701, 0x3f }, 203 { "AES_busy_cycle", 0x702, 0x3f }, 204 { "RC4_busy_cycle", 0x704, 0x3f }, 205 { "MD5_SHA-1_SHA-256_busy_cycle", 0x708, 0x3f }, 206 { "MA_busy_cycle", 0x710, 0x3f }, 207 { "CRC_MPA_cksum", 0x720, 0x3f }, 208 #endif 209 { "ITLB_miss", 0xb04, 0x0c }, 210 { "DTLB_miss", 0xb08, 0x0c }, 211 { "TLB_miss", 0xb0c, 0x0c }, 212 EV_END 213 }; 214 215 static ni2_generic_event_t ni2_generic_events[] = { 216 { "PAPI_tot_ins", "Instr_cnt" }, 217 { "PAPI_l1_dcm", "DC_miss" }, 218 { "PAPI_l1_icm", "IC_miss" }, 219 { "PAPI_l2_icm", "L2_imiss" }, 220 { "PAPI_l2_ldm", "L2_dmiss_ld" }, 221 { "PAPI_tlb_dm", "DTLB_miss" }, 222 { "PAPI_tlb_im", "ITLB_miss" }, 223 { "PAPI_tlb_tm", "TLB_miss" }, 224 { "PAPI_br_tkn", "Br_taken" }, 225 { "PAPI_br_ins", "Br_completed" }, 226 { "PAPI_ld_ins", "Instr_ld" }, 227 { "PAPI_sr_ins", "Instr_st" }, 228 { "PAPI_fp_ops", "Instr_FGU_arithmetic" }, 229 { "PAPI_fp_ins", "Instr_FGU_arithmetic" }, 230 GEN_EV_END 231 }; 232 233 static char *evlist; 234 static size_t evlist_sz; 235 static uint16_t pcr_pic0_mask; 236 static uint16_t pcr_pic1_mask; 237 238 #define CPU_REF_URL " Documentation for Sun processors can be found at: " \ 239 "http://www.sun.com/processors/manuals" 240 241 #if defined(NIAGARA2_IMPL) 242 static const char *cpu_impl_name = "UltraSPARC T2"; 243 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2 User's Manual\" " 244 "for descriptions of these events." CPU_REF_URL; 245 #elif defined(VFALLS_IMPL) 246 static const char *cpu_impl_name = "UltraSPARC T2+"; 247 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T2+ User's Manual\" " 248 "for descriptions of these events." CPU_REF_URL; 249 #elif defined(KT_IMPL) 250 static const char *cpu_impl_name = "UltraSPARC T3"; 251 static const char *cpu_pcbe_ref = "See the \"UltraSPARC T3 User's Manual\" " 252 "for descriptions of these events." CPU_REF_URL; 253 #endif 254 255 static boolean_t cpu_hsvc_available = B_TRUE; 256 257 static int 258 ni2_pcbe_init(void) 259 { 260 ni2_event_t *evp; 261 ni2_generic_event_t *gevp; 262 int status; 263 uint64_t cpu_hsvc_major; 264 uint64_t cpu_hsvc_minor; 265 #if defined(NIAGARA2_IMPL) 266 uint64_t hsvc_cpu_group = HSVC_GROUP_NIAGARA2_CPU; 267 uint64_t hsvc_cpu_major = NIAGARA2_HSVC_MAJOR; 268 #elif defined(VFALLS_IMPL) 269 uint64_t hsvc_cpu_group = HSVC_GROUP_VFALLS_CPU; 270 uint64_t hsvc_cpu_major = VFALLS_HSVC_MAJOR; 271 #elif defined(KT_IMPL) 272 uint64_t hsvc_cpu_group = HSVC_GROUP_KT_CPU; 273 uint64_t hsvc_cpu_major = KT_HSVC_MAJOR; 274 #endif 275 276 pcr_pic0_mask = CPC_PCR_PIC0_MASK; 277 pcr_pic1_mask = CPC_PCR_PIC1_MASK; 278 279 /* 280 * Validate API version for Niagara2 specific hypervisor services 281 */ 282 status = hsvc_version(hsvc_cpu_group, &cpu_hsvc_major, 283 &cpu_hsvc_minor); 284 if ((status != 0) || (cpu_hsvc_major != hsvc_cpu_major)) { 285 cmn_err(CE_WARN, "hypervisor services not negotiated " 286 "or unsupported major number: group: 0x%lx major: 0x%lx " 287 "minor: 0x%lx errno: %d", hsvc_cpu_group, 288 cpu_hsvc_major, cpu_hsvc_minor, status); 289 cpu_hsvc_available = B_FALSE; 290 } 291 292 /* 293 * Construct event list. 294 * 295 * First pass: Calculate size needed. We'll need an additional byte 296 * for the NULL pointer during the last strcat. 297 * 298 * Second pass: Copy strings. 299 */ 300 for (evp = ni2_events; evp->name != NULL; evp++) 301 evlist_sz += strlen(evp->name) + 1; 302 303 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) 304 evlist_sz += strlen(gevp->name) + 1; 305 306 evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP); 307 evlist[0] = '\0'; 308 309 for (evp = ni2_events; evp->name != NULL; evp++) { 310 (void) strcat(evlist, evp->name); 311 (void) strcat(evlist, ","); 312 } 313 314 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) { 315 (void) strcat(evlist, gevp->name); 316 (void) strcat(evlist, ","); 317 } 318 319 /* 320 * Remove trailing comma. 321 */ 322 evlist[evlist_sz - 1] = '\0'; 323 324 return (0); 325 } 326 327 static uint_t 328 ni2_pcbe_ncounters(void) 329 { 330 return (2); 331 } 332 333 static const char * 334 ni2_pcbe_impl_name(void) 335 { 336 return (cpu_impl_name); 337 } 338 339 static const char * 340 ni2_pcbe_cpuref(void) 341 { 342 return (cpu_pcbe_ref); 343 } 344 345 static char * 346 ni2_pcbe_list_events(uint_t picnum) 347 { 348 ASSERT(picnum < cpc_ncounters); 349 350 return (evlist); 351 } 352 353 static char * 354 ni2_pcbe_list_attrs(void) 355 { 356 if (cpu_hsvc_available == B_TRUE) 357 #if defined(NIAGARA2_IMPL) 358 return ("hpriv,emask"); 359 #elif defined(VFALLS_IMPL) 360 return ("hpriv,l2ctl,emask"); 361 #elif defined(KT_IMPL) 362 return ("hpriv,l2ctl,emask,sample"); 363 #endif 364 else 365 #if defined(KT_IMPL) 366 return ("emask,sample"); 367 #else 368 return ("emask"); 369 #endif 370 } 371 372 static ni2_generic_event_t * 373 find_generic_event(char *name) 374 { 375 ni2_generic_event_t *gevp; 376 377 for (gevp = ni2_generic_events; gevp->name != NULL; gevp++) { 378 if (strcmp(name, gevp->name) == 0) 379 return (gevp); 380 } 381 382 return (NULL); 383 } 384 385 static ni2_event_t * 386 find_event(char *name) 387 { 388 ni2_event_t *evp; 389 390 for (evp = ni2_events; evp->name != NULL; evp++) 391 if (strcmp(name, evp->name) == 0) 392 return (evp); 393 394 return (NULL); 395 } 396 397 /*ARGSUSED*/ 398 static uint64_t 399 ni2_pcbe_event_coverage(char *event) 400 { 401 /* 402 * Check whether counter event is supported 403 */ 404 if (find_event(event) == NULL && find_generic_event(event) == NULL) 405 return (0); 406 407 /* 408 * Fortunately, both pic0 and pic1 can count all events. 409 */ 410 return (0x3); 411 } 412 413 static uint64_t 414 ni2_pcbe_overflow_bitmap(void) 415 { 416 uint64_t pcr, overflow; 417 uint64_t pic; 418 uint32_t pic0, pic1; 419 boolean_t update_pic = B_FALSE; 420 boolean_t pic_inrange = B_FALSE; 421 422 ASSERT(getpil() >= DISP_LEVEL); 423 pcr = ultra_getpcr(); 424 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 425 overflow = (pcr & CPC_PCR_OV0_MASK) >> 426 CPC_PCR_OV0_SHIFT; 427 overflow |= (pcr & CPC_PCR_OV1_MASK) >> 428 CPC_PCR_OV1_SHIFT; 429 430 pic = ultra_getpic(); 431 pic0 = (uint32_t)(pic & PIC0_MASK); 432 pic1 = (uint32_t)((pic >> PIC1_SHIFT) & PIC0_MASK); 433 434 pcr |= (CPC_PCR_HOLDOV0 | CPC_PCR_HOLDOV1); 435 436 if (overflow & 0x1) { 437 pcr &= ~(CPC_PCR_OV0_MASK | 438 CPC_PCR_HOLDOV0); 439 pic_inrange = PIC_IN_OV_RANGE(pic0); 440 #if defined(KT_IMPL) 441 if (pcr & CPC_PCR_SAMPLE_MODE_MASK) 442 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic0); 443 #endif 444 if (pic_inrange) { 445 pic0 = 0; 446 update_pic = B_TRUE; 447 } 448 } 449 450 if (overflow & 0x2) { 451 pcr &= ~(CPC_PCR_OV1_MASK | 452 CPC_PCR_HOLDOV1); 453 pic_inrange = PIC_IN_OV_RANGE(pic1); 454 #if defined(KT_IMPL) 455 if (pcr & CPC_PCR_SAMPLE_MODE_MASK) 456 pic_inrange = SAMPLE_PIC_IN_OV_RANGE(pic1); 457 #endif 458 if (pic_inrange) { 459 pic1 = 0; 460 update_pic = B_TRUE; 461 } 462 } 463 464 if (update_pic) 465 ultra_setpic(((uint64_t)pic1 << PIC1_SHIFT) | pic0); 466 467 /* 468 * The HV interface does not need to be used here because we are 469 * only resetting the OV bits and do not need to set the HT bit. 470 */ 471 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 472 ultra_setpcr(pcr); 473 474 return (overflow); 475 } 476 477 /*ARGSUSED*/ 478 static int 479 ni2_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags, 480 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token) 481 { 482 ni2_pcbe_config_t *cfg; 483 ni2_pcbe_config_t *other_config; 484 ni2_event_t *evp; 485 ni2_generic_event_t *gevp; 486 int i; 487 uint32_t evsel; 488 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 489 uint64_t l2ctl = 0; 490 #endif 491 492 /* 493 * If we've been handed an existing configuration, we need only preset 494 * the counter value. 495 */ 496 if (*data != NULL) { 497 cfg = *data; 498 cfg->pcbe_pic = (uint32_t)preset; 499 return (0); 500 } 501 502 if (picnum > 1) 503 return (CPC_INVALID_PICNUM); 504 505 506 if ((evp = find_event(event)) == NULL) { 507 if ((gevp = find_generic_event(event)) != NULL) { 508 evp = find_event(gevp->event); 509 ASSERT(evp != NULL); 510 511 if (nattrs > 0) 512 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 513 } else { 514 return (CPC_INVALID_EVENT); 515 } 516 } 517 518 evsel = evp->emask; 519 520 for (i = 0; i < nattrs; i++) { 521 if (strcmp(attrs[i].ka_name, "hpriv") == 0) { 522 if (attrs[i].ka_val != 0) 523 flags |= CPC_COUNT_HV; 524 } else if (strcmp(attrs[i].ka_name, "emask") == 0) { 525 if ((attrs[i].ka_val | evp->emask_valid) != 526 evp->emask_valid) 527 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 528 evsel |= attrs[i].ka_val; 529 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 530 } else if (strcmp(attrs[i].ka_name, "l2ctl") == 0) { 531 if ((attrs[i].ka_val | L2_CTL_MASK) != 532 L2_CTL_MASK) 533 return (CPC_ATTRIBUTE_OUT_OF_RANGE); 534 else 535 l2ctl = attrs[i].ka_val; 536 #endif 537 #if defined(KT_IMPL) 538 } else if (strcmp(attrs[i].ka_name, "sample") == 0) { 539 if (attrs[i].ka_val != 0) 540 flags |= CPC_COUNT_SAMPLE_MODE; 541 #endif 542 } else 543 return (CPC_INVALID_ATTRIBUTE); 544 } 545 546 #if defined(VFALLS_IMPL) || defined(KT_IMPL) 547 /* 548 * Set PERF_CONTROL bits in L2_CONTROL_REG only when events have 549 * SL bits equal to 3. 550 */ 551 if ((evsel & SL3_MASK) == SL3_MASK) { 552 if ((hv_niagara_setperf(HV_L2_CTL, l2ctl)) != 0) 553 return (CPC_HV_NO_ACCESS); 554 } 555 #endif 556 557 /* 558 * Find other requests that will be programmed with this one, and ensure 559 * the flags don't conflict. 560 */ 561 if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) && 562 (other_config->pcbe_flags != flags)) 563 return (CPC_CONFLICTING_REQS); 564 565 /* 566 * If the hpriv attribute is present, make sure we have 567 * access to hyperprivileged events before continuing with 568 * this configuration. If we can set the ht bit in the PCR 569 * successfully, we must have access to hyperprivileged 570 * events. 571 * 572 * If this is a static per-CPU configuration, the CPC 573 * driver ensures there can not be more than one for this 574 * CPU. If this is a per-LWP configuration, the driver 575 * ensures no static per-CPU counting is ongoing and that 576 * the target LWP is not already being monitored. 577 */ 578 if (flags & CPC_COUNT_HV) { 579 kpreempt_disable(); 580 581 DTRACE_PROBE1(niagara2__setpcr, uint64_t, 582 allstopped | CPC_PCR_HT); 583 if (hv_niagara_setperf(HV_SPARC_CTL, 584 allstopped | CPC_PCR_HT) != H_EOK) { 585 kpreempt_enable(); 586 return (CPC_HV_NO_ACCESS); 587 } 588 589 DTRACE_PROBE1(niagara2__setpcr, uint64_t, allstopped); 590 (void) hv_niagara_setperf(HV_SPARC_CTL, allstopped); 591 592 kpreempt_enable(); 593 } 594 595 cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP); 596 597 cfg->pcbe_picno = picnum; 598 cfg->pcbe_evsel = evsel; 599 cfg->pcbe_flags = flags; 600 cfg->pcbe_pic = (uint32_t)preset; 601 602 *data = cfg; 603 return (0); 604 } 605 606 static void 607 ni2_pcbe_program(void *token) 608 { 609 ni2_pcbe_config_t *pic0; 610 ni2_pcbe_config_t *pic1; 611 ni2_pcbe_config_t *tmp; 612 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 613 uint64_t pcr; 614 uint64_t curpic; 615 uint64_t toe; 616 617 /* enable trap-on-event for pic0 and pic1 */ 618 toe = (CPC_PCR_TOE0 | CPC_PCR_TOE1); 619 620 if ((pic0 = (ni2_pcbe_config_t *)kcpc_next_config(token, NULL, NULL)) == 621 NULL) 622 panic("ni2_pcbe: token %p has no configs", token); 623 624 if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL) { 625 pic1 = &nullcfg; 626 nullcfg.pcbe_flags = pic0->pcbe_flags; 627 toe = CPC_PCR_TOE0; /* enable trap-on-event for pic0 */ 628 } 629 630 if (pic0->pcbe_picno != 0) { 631 /* 632 * pic0 is counter 1, so if we need the null config it should 633 * be counter 0. 634 */ 635 nullcfg.pcbe_picno = 0; 636 tmp = pic0; 637 pic0 = pic1; 638 pic1 = tmp; 639 toe = CPC_PCR_TOE1; /* enable trap-on-event for pic1 */ 640 } 641 642 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 643 panic("%s: bad config on token %p\n", cpu_impl_name, token); 644 645 /* 646 * UltraSPARC does not allow pic0 to be configured differently 647 * from pic1. If the flags on these two configurations are 648 * different, they are incompatible. This condition should be 649 * caught at configure time. 650 */ 651 ASSERT(pic0->pcbe_flags == pic1->pcbe_flags); 652 653 ni2_pcbe_allstop(); 654 655 ultra_setpic(((uint64_t)pic1->pcbe_pic << PIC1_SHIFT) | 656 (uint64_t)pic0->pcbe_pic); 657 658 pcr = (pic0->pcbe_evsel & pcr_pic0_mask) << CPC_PCR_PIC0_SHIFT; 659 pcr |= (pic1->pcbe_evsel & pcr_pic1_mask) << 660 CPC_PCR_PIC1_SHIFT; 661 662 if (pic0->pcbe_flags & CPC_COUNT_USER) 663 pcr |= (1ull << CPC_PCR_UT_SHIFT); 664 if (pic0->pcbe_flags & CPC_COUNT_SYSTEM) 665 pcr |= (1ull << CPC_PCR_ST_SHIFT); 666 if (pic0->pcbe_flags & CPC_COUNT_HV) 667 pcr |= (1ull << CPC_PCR_HT_SHIFT); 668 #if defined(KT_IMPL) 669 if (pic0->pcbe_flags & CPC_COUNT_SAMPLE_MODE) 670 pcr |= (1ull << CPC_PCR_SAMPLE_MODE_SHIFT); 671 #endif 672 pcr |= toe; 673 674 DTRACE_PROBE1(niagara2__setpcr, uint64_t, pcr); 675 676 if (pic0->pcbe_flags & CPC_COUNT_HV) { 677 /* 678 * The ht bit in the PCR is only writable in 679 * hyperprivileged mode. So if we are counting 680 * hpriv events, we must use the HV interface 681 * hv_niagara_setperf to set the PCR. If this 682 * fails, assume we no longer have access to 683 * hpriv events. 684 */ 685 if (hv_niagara_setperf(HV_SPARC_CTL, pcr) != H_EOK) { 686 kcpc_invalidate_config(token); 687 return; 688 } 689 } else 690 /* Set the PCR with no hpriv event counting enabled. */ 691 ultra_setpcr(pcr); 692 693 ni2_cpc_counting[CPU->cpu_id] = B_TRUE; 694 695 /* 696 * On UltraSPARC, only read-to-read counts are accurate. We cannot 697 * expect the value we wrote into the PIC, above, to be there after 698 * starting the counter. We must sample the counter value now and use 699 * that as the baseline for future samples. 700 */ 701 curpic = ultra_getpic(); 702 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 703 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 704 705 DTRACE_PROBE1(niagara2__newpic, uint64_t, curpic); 706 } 707 708 static void 709 ni2_pcbe_allstop(void) 710 { 711 /* 712 * We use the HV interface here because if we were counting 713 * hyperprivileged events, we must reset the PCR.ht bit to stop 714 * the counting. In the event that this HV call fails, we fall 715 * back on ultra_setpcr which does not have write access to the 716 * ht bit. 717 */ 718 if (hv_niagara_setperf(HV_SPARC_CTL, allstopped) != H_EOK) 719 ultra_setpcr(allstopped); 720 721 ni2_cpc_counting[CPU->cpu_id] = B_FALSE; 722 } 723 724 static void 725 ni2_pcbe_sample(void *token) 726 { 727 uint64_t curpic; 728 int64_t diff; 729 uint64_t *pic0_data; 730 uint64_t *pic1_data; 731 uint64_t *dtmp; 732 uint64_t tmp; 733 uint64_t pcr; 734 ni2_pcbe_config_t *pic0; 735 ni2_pcbe_config_t *pic1; 736 ni2_pcbe_config_t nullcfg = { 1, 0, 0, 0 }; 737 ni2_pcbe_config_t *ctmp; 738 739 curpic = ultra_getpic(); 740 DTRACE_PROBE1(niagara2__getpic, uint64_t, curpic); 741 742 if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL) 743 panic("%s: token %p has no configs", cpu_impl_name, token); 744 745 if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) { 746 pic1 = &nullcfg; 747 pic1_data = &tmp; 748 } 749 750 if (pic0->pcbe_picno != 0) { 751 nullcfg.pcbe_picno = 0; 752 ctmp = pic0; 753 pic0 = pic1; 754 pic1 = ctmp; 755 dtmp = pic0_data; 756 pic0_data = pic1_data; 757 pic1_data = dtmp; 758 } 759 760 if (pic0->pcbe_picno != 0 || pic1->pcbe_picno != 1) 761 panic("%s: bad config on token %p\n", cpu_impl_name, token); 762 763 764 if (pic0->pcbe_flags & CPC_COUNT_HV) { 765 /* 766 * If the hpriv attribute is present, but the HT bit 767 * is not set in the PCR, access to hyperprivileged 768 * events must have been revoked. Only perform this 769 * check if counting is not stopped. 770 */ 771 pcr = ultra_getpcr(); 772 DTRACE_PROBE1(niagara2__getpcr, uint64_t, pcr); 773 if (ni2_cpc_counting[CPU->cpu_id] && 774 !(pcr & CPC_PCR_HT)) { 775 kcpc_invalidate_config(token); 776 return; 777 } 778 } 779 780 diff = (curpic & PIC0_MASK) - (uint64_t)pic0->pcbe_pic; 781 if (diff < 0) 782 diff += (1ll << 32); 783 *pic0_data += diff; 784 785 diff = (curpic >> 32) - (uint64_t)pic1->pcbe_pic; 786 if (diff < 0) 787 diff += (1ll << 32); 788 *pic1_data += diff; 789 790 pic0->pcbe_pic = (uint32_t)(curpic & PIC0_MASK); 791 pic1->pcbe_pic = (uint32_t)(curpic >> PIC1_SHIFT); 792 } 793 794 static void 795 ni2_pcbe_free(void *config) 796 { 797 kmem_free(config, sizeof (ni2_pcbe_config_t)); 798 } 799 800 801 static struct modlpcbe modlpcbe = { 802 &mod_pcbeops, 803 #if defined(NIAGARA2_IMPL) 804 "UltraSPARC T2 Performance Counters", 805 #elif defined(VFALLS_IMPL) 806 "UltraSPARC T2+ Performance Counters", 807 #elif defined(KT_IMPL) 808 "UltraSPARC T3 Performance Counters", 809 #endif 810 &ni2_pcbe_ops 811 }; 812 813 static struct modlinkage modl = { 814 MODREV_1, 815 &modlpcbe, 816 }; 817 818 int 819 _init(void) 820 { 821 if (ni2_pcbe_init() != 0) 822 return (ENOTSUP); 823 return (mod_install(&modl)); 824 } 825 826 int 827 _fini(void) 828 { 829 return (mod_remove(&modl)); 830 } 831 832 int 833 _info(struct modinfo *mi) 834 { 835 return (mod_info(&modl, mi)); 836 } 837