1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018, Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 * 29 */ 30 31 #include <sys/types.h> 32 #include <sys/errno.h> 33 #include <sys/sysctl.h> 34 #include <stddef.h> 35 #include <stdlib.h> 36 #include <limits.h> 37 #include <string.h> 38 #include <pmc.h> 39 #include <pmclog.h> 40 #include <assert.h> 41 #include <libpmcstat.h> 42 #include "pmu-events/pmu-events.h" 43 44 #if defined(__amd64__) || defined(__i386__) 45 struct pmu_alias { 46 const char *pa_alias; 47 const char *pa_name; 48 }; 49 50 typedef enum { 51 PMU_INVALID, 52 PMU_INTEL, 53 PMU_AMD, 54 } pmu_mfr_t; 55 56 static struct pmu_alias pmu_intel_alias_table[] = { 57 {"UNHALTED_CORE_CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"}, 58 {"UNHALTED-CORE-CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"}, 59 {"LLC_MISSES", "LONGEST_LAT_CACHE.MISS"}, 60 {"LLC-MISSES", "LONGEST_LAT_CACHE.MISS"}, 61 {"LLC_REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"}, 62 {"LLC-REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"}, 63 {"LLC_MISS_RHITM", "mem_load_l3_miss_retired.remote_hitm"}, 64 {"LLC-MISS-RHITM", "mem_load_l3_miss_retired.remote_hitm"}, 65 {"RESOURCE_STALL", "RESOURCE_STALLS.ANY"}, 66 {"RESOURCE_STALLS_ANY", "RESOURCE_STALLS.ANY"}, 67 {"BRANCH_INSTRUCTION_RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"}, 68 {"BRANCH-INSTRUCTION-RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"}, 69 {"BRANCH_MISSES_RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"}, 70 {"BRANCH-MISSES-RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"}, 71 {"cycles", "tsc-tsc"}, 72 {"instructions", "inst-retired.any_p"}, 73 {"branch-mispredicts", "br_misp_retired.all_branches"}, 74 {"branches", "br_inst_retired.all_branches"}, 75 {"interrupts", "hw_interrupts.received"}, 76 {"ic-misses", "frontend_retired.l1i_miss"}, 77 {NULL, NULL}, 78 }; 79 80 static struct pmu_alias pmu_amd_alias_table[] = { 81 {"UNHALTED_CORE_CYCLES", "ls_not_halted_cyc"}, 82 {"UNHALTED-CORE-CYCLES", "ls_not_halted_cyc"}, 83 {NULL, NULL}, 84 }; 85 86 87 static pmu_mfr_t 88 pmu_events_mfr(void) 89 { 90 char *buf; 91 size_t s; 92 pmu_mfr_t mfr; 93 94 if (sysctlbyname("kern.hwpmc.cpuid", (void *)NULL, &s, 95 (void *)NULL, 0) == -1) 96 return (PMU_INVALID); 97 if ((buf = malloc(s + 1)) == NULL) 98 return (PMU_INVALID); 99 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s, 100 (void *)NULL, 0) == -1) { 101 free(buf); 102 return (PMU_INVALID); 103 } 104 if (strcasestr(buf, "AuthenticAMD") != NULL) 105 mfr = PMU_AMD; 106 else if (strcasestr(buf, "GenuineIntel") != NULL) 107 mfr = PMU_INTEL; 108 else 109 mfr = PMU_INVALID; 110 free(buf); 111 return (mfr); 112 } 113 114 /* 115 * The Intel fixed mode counters are: 116 * "inst_retired.any", 117 * "cpu_clk_unhalted.thread", 118 * "cpu_clk_unhalted.thread_any", 119 * "cpu_clk_unhalted.ref_tsc", 120 * 121 */ 122 123 static const char * 124 pmu_alias_get(const char *name) 125 { 126 pmu_mfr_t mfr; 127 struct pmu_alias *pa; 128 struct pmu_alias *pmu_alias_table; 129 130 if ((mfr = pmu_events_mfr()) == PMU_INVALID) 131 return (name); 132 if (mfr == PMU_AMD) 133 pmu_alias_table = pmu_amd_alias_table; 134 else if (mfr == PMU_INTEL) 135 pmu_alias_table = pmu_intel_alias_table; 136 else 137 return (name); 138 139 for (pa = pmu_alias_table; pa->pa_alias != NULL; pa++) 140 if (strcasecmp(name, pa->pa_alias) == 0) 141 return (pa->pa_name); 142 143 return (name); 144 } 145 146 struct pmu_event_desc { 147 uint64_t ped_period; 148 uint64_t ped_offcore_rsp; 149 uint32_t ped_event; 150 uint32_t ped_frontend; 151 uint32_t ped_ldlat; 152 uint32_t ped_config1; 153 int16_t ped_umask; 154 uint8_t ped_cmask; 155 uint8_t ped_any; 156 uint8_t ped_inv; 157 uint8_t ped_edge; 158 uint8_t ped_fc_mask; 159 uint8_t ped_ch_mask; 160 }; 161 162 static const struct pmu_events_map * 163 pmu_events_map_get(const char *cpuid) 164 { 165 size_t s; 166 char buf[64]; 167 const struct pmu_events_map *pme; 168 169 if (cpuid != NULL) { 170 memcpy(buf, cpuid, 64); 171 } else { 172 if (sysctlbyname("kern.hwpmc.cpuid", (void *)NULL, &s, 173 (void *)NULL, 0) == -1) 174 return (NULL); 175 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s, 176 (void *)NULL, 0) == -1) 177 return (NULL); 178 } 179 for (pme = pmu_events_map; pme->cpuid != NULL; pme++) 180 if (strcmp(buf, pme->cpuid) == 0) 181 return (pme); 182 return (NULL); 183 } 184 185 static const struct pmu_event * 186 pmu_event_get(const char *cpuid, const char *event_name, int *idx) 187 { 188 const struct pmu_events_map *pme; 189 const struct pmu_event *pe; 190 int i; 191 192 if ((pme = pmu_events_map_get(cpuid)) == NULL) 193 return (NULL); 194 for (i = 0, pe = pme->table; pe->name || pe->desc || pe->event; pe++, i++) { 195 if (pe->name == NULL) 196 continue; 197 if (strcasecmp(pe->name, event_name) == 0) { 198 if (idx) 199 *idx = i; 200 return (pe); 201 } 202 } 203 return (NULL); 204 } 205 206 int 207 pmc_pmu_idx_get_by_event(const char *cpuid, const char *event) 208 { 209 int idx; 210 const char *realname; 211 212 realname = pmu_alias_get(event); 213 if (pmu_event_get(cpuid, realname, &idx) == NULL) 214 return (-1); 215 return (idx); 216 } 217 218 const char * 219 pmc_pmu_event_get_by_idx(const char *cpuid, int idx) 220 { 221 const struct pmu_events_map *pme; 222 223 if ((pme = pmu_events_map_get(cpuid)) == NULL) 224 return (NULL); 225 assert(pme->table[idx].name); 226 return (pme->table[idx].name); 227 } 228 229 static int 230 pmu_parse_event(struct pmu_event_desc *ped, const char *eventin) 231 { 232 char *event; 233 char *kvp, *key, *value, *r; 234 char *debug; 235 236 if ((event = strdup(eventin)) == NULL) 237 return (ENOMEM); 238 r = event; 239 bzero(ped, sizeof(*ped)); 240 ped->ped_period = DEFAULT_SAMPLE_COUNT; 241 ped->ped_umask = -1; 242 while ((kvp = strsep(&event, ",")) != NULL) { 243 key = strsep(&kvp, "="); 244 if (key == NULL) 245 abort(); 246 value = kvp; 247 if (strcmp(key, "umask") == 0) 248 ped->ped_umask = strtol(value, NULL, 16); 249 else if (strcmp(key, "event") == 0) 250 ped->ped_event = strtol(value, NULL, 16); 251 else if (strcmp(key, "period") == 0) 252 ped->ped_period = strtol(value, NULL, 10); 253 else if (strcmp(key, "offcore_rsp") == 0) 254 ped->ped_offcore_rsp = strtol(value, NULL, 16); 255 else if (strcmp(key, "any") == 0) 256 ped->ped_any = strtol(value, NULL, 10); 257 else if (strcmp(key, "cmask") == 0) 258 ped->ped_cmask = strtol(value, NULL, 10); 259 else if (strcmp(key, "inv") == 0) 260 ped->ped_inv = strtol(value, NULL, 10); 261 else if (strcmp(key, "edge") == 0) 262 ped->ped_edge = strtol(value, NULL, 10); 263 else if (strcmp(key, "frontend") == 0) 264 ped->ped_frontend = strtol(value, NULL, 16); 265 else if (strcmp(key, "ldlat") == 0) 266 ped->ped_ldlat = strtol(value, NULL, 16); 267 else if (strcmp(key, "fc_mask") == 0) 268 ped->ped_fc_mask = strtol(value, NULL, 16); 269 else if (strcmp(key, "ch_mask") == 0) 270 ped->ped_ch_mask = strtol(value, NULL, 16); 271 else if (strcmp(key, "config1") == 0) 272 ped->ped_config1 = strtol(value, NULL, 16); 273 else { 274 debug = getenv("PMUDEBUG"); 275 if (debug != NULL && strcmp(debug, "true") == 0 && value != NULL) 276 printf("unrecognized kvpair: %s:%s\n", key, value); 277 } 278 } 279 free(r); 280 return (0); 281 } 282 283 uint64_t 284 pmc_pmu_sample_rate_get(const char *event_name) 285 { 286 const struct pmu_event *pe; 287 struct pmu_event_desc ped; 288 289 event_name = pmu_alias_get(event_name); 290 if ((pe = pmu_event_get(NULL, event_name, NULL)) == NULL) 291 return (DEFAULT_SAMPLE_COUNT); 292 if (pe->alias && (pe = pmu_event_get(NULL, pe->alias, NULL)) == NULL) 293 return (DEFAULT_SAMPLE_COUNT); 294 if (pe->event == NULL) 295 return (DEFAULT_SAMPLE_COUNT); 296 if (pmu_parse_event(&ped, pe->event)) 297 return (DEFAULT_SAMPLE_COUNT); 298 return (ped.ped_period); 299 } 300 301 int 302 pmc_pmu_enabled(void) 303 { 304 305 return (pmu_events_map_get(NULL) != NULL); 306 } 307 308 void 309 pmc_pmu_print_counters(const char *event_name) 310 { 311 const struct pmu_events_map *pme; 312 const struct pmu_event *pe; 313 struct pmu_event_desc ped; 314 char *debug; 315 int do_debug; 316 317 debug = getenv("PMUDEBUG"); 318 do_debug = 0; 319 320 if (debug != NULL && strcmp(debug, "true") == 0) 321 do_debug = 1; 322 if ((pme = pmu_events_map_get(NULL)) == NULL) 323 return; 324 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 325 if (pe->name == NULL) 326 continue; 327 if (event_name != NULL && strcasestr(pe->name, event_name) == NULL) 328 continue; 329 printf("\t%s\n", pe->name); 330 if (do_debug) 331 pmu_parse_event(&ped, pe->event); 332 } 333 } 334 335 void 336 pmc_pmu_print_counter_desc(const char *ev) 337 { 338 const struct pmu_events_map *pme; 339 const struct pmu_event *pe; 340 341 if ((pme = pmu_events_map_get(NULL)) == NULL) 342 return; 343 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 344 if (pe->name == NULL) 345 continue; 346 if (strcasestr(pe->name, ev) != NULL && 347 pe->desc != NULL) 348 printf("%s:\t%s\n", pe->name, pe->desc); 349 } 350 } 351 352 void 353 pmc_pmu_print_counter_desc_long(const char *ev) 354 { 355 const struct pmu_events_map *pme; 356 const struct pmu_event *pe; 357 358 if ((pme = pmu_events_map_get(NULL)) == NULL) 359 return; 360 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 361 if (pe->name == NULL) 362 continue; 363 if (strcasestr(pe->name, ev) != NULL) { 364 if (pe->long_desc != NULL) 365 printf("%s:\n%s\n", pe->name, pe->long_desc); 366 else if (pe->desc != NULL) 367 printf("%s:\t%s\n", pe->name, pe->desc); 368 } 369 } 370 } 371 372 void 373 pmc_pmu_print_counter_full(const char *ev) 374 { 375 const struct pmu_events_map *pme; 376 const struct pmu_event *pe; 377 378 if ((pme = pmu_events_map_get(NULL)) == NULL) 379 return; 380 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 381 if (pe->name == NULL) 382 continue; 383 if (strcasestr(pe->name, ev) == NULL) 384 continue; 385 printf("name: %s\n", pe->name); 386 if (pe->long_desc != NULL) 387 printf("desc: %s\n", pe->long_desc); 388 else if (pe->desc != NULL) 389 printf("desc: %s\n", pe->desc); 390 if (pe->event != NULL) 391 printf("event: %s\n", pe->event); 392 if (pe->topic != NULL) 393 printf("topic: %s\n", pe->topic); 394 if (pe->pmu != NULL) 395 printf("pmu: %s\n", pe->pmu); 396 if (pe->unit != NULL) 397 printf("unit: %s\n", pe->unit); 398 if (pe->perpkg != NULL) 399 printf("perpkg: %s\n", pe->perpkg); 400 if (pe->metric_expr != NULL) 401 printf("metric_expr: %s\n", pe->metric_expr); 402 if (pe->metric_name != NULL) 403 printf("metric_name: %s\n", pe->metric_name); 404 if (pe->metric_group != NULL) 405 printf("metric_group: %s\n", pe->metric_group); 406 } 407 } 408 409 static int 410 pmc_pmu_amd_pmcallocate(const char *event_name __unused, struct pmc_op_pmcallocate *pm, 411 struct pmu_event_desc *ped) 412 { 413 struct pmc_md_amd_op_pmcallocate *amd; 414 415 amd = &pm->pm_md.pm_amd; 416 amd->pm_amd_config = AMD_PMC_TO_EVENTMASK(ped->ped_event); 417 if (ped->ped_umask > 0) { 418 pm->pm_caps |= PMC_CAP_QUALIFIER; 419 amd->pm_amd_config |= AMD_PMC_TO_UNITMASK(ped->ped_umask); 420 } 421 pm->pm_class = PMC_CLASS_K8; 422 423 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 || 424 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 425 (PMC_CAP_USER|PMC_CAP_SYSTEM)) 426 amd->pm_amd_config |= (AMD_PMC_USR | AMD_PMC_OS); 427 else if (pm->pm_caps & PMC_CAP_USER) 428 amd->pm_amd_config |= AMD_PMC_USR; 429 else if (pm->pm_caps & PMC_CAP_SYSTEM) 430 amd->pm_amd_config |= AMD_PMC_OS; 431 if (ped->ped_edge) 432 amd->pm_amd_config |= AMD_PMC_EDGE; 433 if (ped->ped_inv) 434 amd->pm_amd_config |= AMD_PMC_EDGE; 435 if (pm->pm_caps & PMC_CAP_INTERRUPT) 436 amd->pm_amd_config |= AMD_PMC_INT; 437 return (0); 438 } 439 440 static int 441 pmc_pmu_intel_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm, 442 struct pmu_event_desc *ped) 443 { 444 struct pmc_md_iap_op_pmcallocate *iap; 445 int isfixed; 446 447 isfixed = 0; 448 iap = &pm->pm_md.pm_iap; 449 if (strcasestr(event_name, "UNC_") == event_name || 450 strcasestr(event_name, "uncore") != NULL) { 451 pm->pm_class = PMC_CLASS_UCP; 452 pm->pm_caps |= PMC_CAP_QUALIFIER; 453 } else if ((ped->ped_umask == -1) || 454 (ped->ped_event == 0x0 && ped->ped_umask == 0x3)) { 455 pm->pm_class = PMC_CLASS_IAF; 456 } else { 457 pm->pm_class = PMC_CLASS_IAP; 458 pm->pm_caps |= PMC_CAP_QUALIFIER; 459 } 460 iap->pm_iap_config |= IAP_EVSEL(ped->ped_event); 461 if (ped->ped_umask > 0) 462 iap->pm_iap_config |= IAP_UMASK(ped->ped_umask); 463 iap->pm_iap_config |= IAP_CMASK(ped->ped_cmask); 464 iap->pm_iap_rsp = ped->ped_offcore_rsp; 465 466 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 || 467 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 468 (PMC_CAP_USER|PMC_CAP_SYSTEM)) 469 iap->pm_iap_config |= (IAP_USR | IAP_OS); 470 else if (pm->pm_caps & PMC_CAP_USER) 471 iap->pm_iap_config |= IAP_USR; 472 else if (pm->pm_caps & PMC_CAP_SYSTEM) 473 iap->pm_iap_config |= IAP_OS; 474 if (ped->ped_edge) 475 iap->pm_iap_config |= IAP_EDGE; 476 if (ped->ped_any) 477 iap->pm_iap_config |= IAP_ANY; 478 if (ped->ped_inv) 479 iap->pm_iap_config |= IAP_EDGE; 480 if (pm->pm_caps & PMC_CAP_INTERRUPT) 481 iap->pm_iap_config |= IAP_INT; 482 return (0); 483 } 484 485 int 486 pmc_pmu_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm) 487 { 488 const struct pmu_event *pe; 489 struct pmu_event_desc ped; 490 pmu_mfr_t mfr; 491 int idx = -1; 492 493 if ((mfr = pmu_events_mfr()) == PMU_INVALID) 494 return (ENOENT); 495 496 bzero(&pm->pm_md, sizeof(pm->pm_md)); 497 pm->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE); 498 event_name = pmu_alias_get(event_name); 499 if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL) 500 return (ENOENT); 501 if (pe->alias && (pe = pmu_event_get(NULL, pe->alias, &idx)) == NULL) 502 return (ENOENT); 503 assert(idx >= 0); 504 pm->pm_ev = idx; 505 506 if (pe->event == NULL) 507 return (ENOENT); 508 if (pmu_parse_event(&ped, pe->event)) 509 return (ENOENT); 510 511 if (mfr == PMU_INTEL) 512 return (pmc_pmu_intel_pmcallocate(event_name, pm, &ped)); 513 else 514 return (pmc_pmu_amd_pmcallocate(event_name, pm, &ped)); 515 } 516 517 /* 518 * Ultimately rely on AMD calling theirs the same 519 */ 520 static const char *stat_mode_cntrs[] = { 521 "cpu_clk_unhalted.thread", 522 "inst_retired.any", 523 "br_inst_retired.all_branches", 524 "br_misp_retired.all_branches", 525 "longest_lat_cache.reference", 526 "longest_lat_cache.miss", 527 }; 528 529 int 530 pmc_pmu_stat_mode(const char ***cntrs) 531 { 532 if (pmc_pmu_enabled()) { 533 *cntrs = stat_mode_cntrs; 534 return (0); 535 } 536 return (EOPNOTSUPP); 537 } 538 539 #else 540 541 uint64_t 542 pmc_pmu_sample_rate_get(const char *event_name __unused) 543 { 544 return (DEFAULT_SAMPLE_COUNT); 545 } 546 547 void 548 pmc_pmu_print_counters(const char *event_name __unused) 549 { 550 } 551 552 void 553 pmc_pmu_print_counter_desc(const char *e __unused) 554 { 555 } 556 557 void 558 pmc_pmu_print_counter_desc_long(const char *e __unused) 559 { 560 } 561 562 void 563 pmc_pmu_print_counter_full(const char *e __unused) 564 { 565 566 } 567 568 int 569 pmc_pmu_enabled(void) 570 { 571 return (0); 572 } 573 574 int 575 pmc_pmu_pmcallocate(const char *e __unused, struct pmc_op_pmcallocate *p __unused) 576 { 577 return (EOPNOTSUPP); 578 } 579 580 const char * 581 pmc_pmu_event_get_by_idx(const char *c __unused, int idx __unused) 582 { 583 return (NULL); 584 } 585 586 int 587 pmc_pmu_stat_mode(const char ***a __unused) 588 { 589 return (EOPNOTSUPP); 590 } 591 592 int 593 pmc_pmu_idx_get_by_event(const char *c __unused, const char *e __unused) 594 { 595 return (-1); 596 } 597 598 #endif 599