1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018, Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 * 29 */ 30 31 #include <sys/types.h> 32 #include <sys/errno.h> 33 #include <sys/pmc.h> 34 #include <sys/sysctl.h> 35 #include <stddef.h> 36 #include <stdlib.h> 37 #include <limits.h> 38 #include <regex.h> 39 #include <string.h> 40 #include <pmc.h> 41 #include <pmclog.h> 42 #include <assert.h> 43 #include <libpmcstat.h> 44 #include "pmu-events/pmu-events.h" 45 46 struct pmu_alias { 47 const char *pa_alias; 48 const char *pa_name; 49 }; 50 51 #if defined(__amd64__) || defined(__i386__) 52 typedef enum { 53 PMU_INVALID, 54 PMU_INTEL, 55 PMU_AMD, 56 } pmu_mfr_t; 57 58 static struct pmu_alias pmu_intel_alias_table[] = { 59 {"UNHALTED_CORE_CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"}, 60 {"UNHALTED-CORE-CYCLES", "CPU_CLK_UNHALTED.THREAD_P_ANY"}, 61 {"LLC_MISSES", "LONGEST_LAT_CACHE.MISS"}, 62 {"LLC-MISSES", "LONGEST_LAT_CACHE.MISS"}, 63 {"LLC_REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"}, 64 {"LLC-REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"}, 65 {"LLC_MISS_RHITM", "mem_load_l3_miss_retired.remote_hitm"}, 66 {"LLC-MISS-RHITM", "mem_load_l3_miss_retired.remote_hitm"}, 67 {"RESOURCE_STALL", "RESOURCE_STALLS.ANY"}, 68 {"RESOURCE_STALLS_ANY", "RESOURCE_STALLS.ANY"}, 69 {"BRANCH_INSTRUCTION_RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"}, 70 {"BRANCH-INSTRUCTION-RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"}, 71 {"BRANCH_MISSES_RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"}, 72 {"BRANCH-MISSES-RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"}, 73 {"cycles", "tsc-tsc"}, 74 {"unhalted-cycles", "CPU_CLK_UNHALTED.THREAD_P_ANY"}, 75 {"instructions", "inst_retired.any_p"}, 76 {"branch-mispredicts", "br_misp_retired.all_branches"}, 77 {"branches", "br_inst_retired.all_branches"}, 78 {"interrupts", "hw_interrupts.received"}, 79 {"ic-misses", "frontend_retired.l1i_miss"}, 80 {NULL, NULL}, 81 }; 82 83 static struct pmu_alias pmu_amd_alias_table[] = { 84 {"UNHALTED_CORE_CYCLES", "ls_not_halted_cyc"}, 85 {"UNHALTED-CORE-CYCLES", "ls_not_halted_cyc"}, 86 {NULL, NULL}, 87 }; 88 89 90 static pmu_mfr_t 91 pmu_events_mfr(void) 92 { 93 char buf[PMC_CPUID_LEN]; 94 size_t s = sizeof(buf); 95 pmu_mfr_t mfr; 96 97 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s, 98 (void *)NULL, 0) == -1) 99 return (PMU_INVALID); 100 if (strcasestr(buf, "AuthenticAMD") != NULL || 101 strcasestr(buf, "HygonGenuine") != NULL) 102 mfr = PMU_AMD; 103 else if (strcasestr(buf, "GenuineIntel") != NULL) 104 mfr = PMU_INTEL; 105 else 106 mfr = PMU_INVALID; 107 return (mfr); 108 } 109 110 /* 111 * The Intel fixed mode counters are: 112 * "inst_retired.any", 113 * "cpu_clk_unhalted.thread", 114 * "cpu_clk_unhalted.thread_any", 115 * "cpu_clk_unhalted.ref_tsc", 116 * 117 */ 118 119 static const char * 120 pmu_alias_get(const char *name) 121 { 122 pmu_mfr_t mfr; 123 struct pmu_alias *pa; 124 struct pmu_alias *pmu_alias_table; 125 126 if ((mfr = pmu_events_mfr()) == PMU_INVALID) 127 return (name); 128 if (mfr == PMU_AMD) 129 pmu_alias_table = pmu_amd_alias_table; 130 else if (mfr == PMU_INTEL) 131 pmu_alias_table = pmu_intel_alias_table; 132 else 133 return (name); 134 135 for (pa = pmu_alias_table; pa->pa_alias != NULL; pa++) 136 if (strcasecmp(name, pa->pa_alias) == 0) 137 return (pa->pa_name); 138 139 return (name); 140 } 141 142 #else 143 144 static const char * 145 pmu_alias_get(const char *name) 146 { 147 148 return (name); 149 } 150 #endif 151 152 struct pmu_event_desc { 153 uint64_t ped_period; 154 uint64_t ped_offcore_rsp; 155 uint64_t ped_l3_thread; 156 uint64_t ped_l3_slice; 157 uint32_t ped_event; 158 uint32_t ped_frontend; 159 uint32_t ped_ldlat; 160 uint32_t ped_config1; 161 int16_t ped_umask; 162 uint8_t ped_cmask; 163 uint8_t ped_any; 164 uint8_t ped_inv; 165 uint8_t ped_edge; 166 uint8_t ped_fc_mask; 167 uint8_t ped_ch_mask; 168 }; 169 170 static const struct pmu_events_map * 171 pmu_events_map_get(const char *cpuid) 172 { 173 regex_t re; 174 regmatch_t pmatch[1]; 175 char buf[PMC_CPUID_LEN]; 176 size_t s = sizeof(buf); 177 int match; 178 const struct pmu_events_map *pme; 179 180 if (cpuid != NULL) { 181 strlcpy(buf, cpuid, s); 182 } else { 183 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s, 184 (void *)NULL, 0) == -1) 185 return (NULL); 186 } 187 for (pme = pmu_events_map; pme->cpuid != NULL; pme++) { 188 if (regcomp(&re, pme->cpuid, REG_EXTENDED) != 0) { 189 printf("regex '%s' failed to compile, ignoring\n", 190 pme->cpuid); 191 continue; 192 } 193 match = regexec(&re, buf, 1, pmatch, 0); 194 regfree(&re); 195 if (match == 0) { 196 if (pmatch[0].rm_so == 0 && (buf[pmatch[0].rm_eo] == 0 197 || buf[pmatch[0].rm_eo] == '-')) 198 return (pme); 199 } 200 } 201 return (NULL); 202 } 203 204 static const struct pmu_event * 205 pmu_event_get(const char *cpuid, const char *event_name, int *idx) 206 { 207 const struct pmu_events_map *pme; 208 const struct pmu_event *pe; 209 int i; 210 211 if ((pme = pmu_events_map_get(cpuid)) == NULL) 212 return (NULL); 213 for (i = 0, pe = pme->table; pe->name || pe->desc || pe->event; pe++, i++) { 214 if (pe->name == NULL) 215 continue; 216 if (strcasecmp(pe->name, event_name) == 0) { 217 if (idx) 218 *idx = i; 219 return (pe); 220 } 221 } 222 return (NULL); 223 } 224 225 int 226 pmc_pmu_idx_get_by_event(const char *cpuid, const char *event) 227 { 228 int idx; 229 const char *realname; 230 231 realname = pmu_alias_get(event); 232 if (pmu_event_get(cpuid, realname, &idx) == NULL) 233 return (-1); 234 return (idx); 235 } 236 237 const char * 238 pmc_pmu_event_get_by_idx(const char *cpuid, int idx) 239 { 240 const struct pmu_events_map *pme; 241 242 if ((pme = pmu_events_map_get(cpuid)) == NULL) 243 return (NULL); 244 assert(pme->table[idx].name); 245 return (pme->table[idx].name); 246 } 247 248 static int 249 pmu_parse_event(struct pmu_event_desc *ped, const char *eventin) 250 { 251 char *event; 252 char *kvp, *key, *value, *r; 253 char *debug; 254 255 if ((event = strdup(eventin)) == NULL) 256 return (ENOMEM); 257 r = event; 258 bzero(ped, sizeof(*ped)); 259 ped->ped_period = DEFAULT_SAMPLE_COUNT; 260 ped->ped_umask = -1; 261 while ((kvp = strsep(&event, ",")) != NULL) { 262 key = strsep(&kvp, "="); 263 if (key == NULL) 264 abort(); 265 value = kvp; 266 if (strcmp(key, "umask") == 0) 267 ped->ped_umask = strtol(value, NULL, 16); 268 else if (strcmp(key, "event") == 0) 269 ped->ped_event = strtol(value, NULL, 16); 270 else if (strcmp(key, "period") == 0) 271 ped->ped_period = strtol(value, NULL, 10); 272 else if (strcmp(key, "offcore_rsp") == 0) 273 ped->ped_offcore_rsp = strtol(value, NULL, 16); 274 else if (strcmp(key, "any") == 0) 275 ped->ped_any = strtol(value, NULL, 10); 276 else if (strcmp(key, "cmask") == 0) 277 ped->ped_cmask = strtol(value, NULL, 10); 278 else if (strcmp(key, "inv") == 0) 279 ped->ped_inv = strtol(value, NULL, 10); 280 else if (strcmp(key, "edge") == 0) 281 ped->ped_edge = strtol(value, NULL, 10); 282 else if (strcmp(key, "frontend") == 0) 283 ped->ped_frontend = strtol(value, NULL, 16); 284 else if (strcmp(key, "ldlat") == 0) 285 ped->ped_ldlat = strtol(value, NULL, 16); 286 else if (strcmp(key, "fc_mask") == 0) 287 ped->ped_fc_mask = strtol(value, NULL, 16); 288 else if (strcmp(key, "ch_mask") == 0) 289 ped->ped_ch_mask = strtol(value, NULL, 16); 290 else if (strcmp(key, "config1") == 0) 291 ped->ped_config1 = strtol(value, NULL, 16); 292 else if (strcmp(key, "l3_thread_mask") == 0) 293 ped->ped_l3_thread = strtol(value, NULL, 16); 294 else if (strcmp(key, "l3_slice_mask") == 0) 295 ped->ped_l3_slice = strtol(value, NULL, 16); 296 else { 297 debug = getenv("PMUDEBUG"); 298 if (debug != NULL && strcmp(debug, "true") == 0 && value != NULL) 299 printf("unrecognized kvpair: %s:%s\n", key, value); 300 } 301 } 302 free(r); 303 return (0); 304 } 305 306 uint64_t 307 pmc_pmu_sample_rate_get(const char *event_name) 308 { 309 const struct pmu_event *pe; 310 struct pmu_event_desc ped; 311 312 event_name = pmu_alias_get(event_name); 313 if ((pe = pmu_event_get(NULL, event_name, NULL)) == NULL) 314 return (DEFAULT_SAMPLE_COUNT); 315 if (pe->event == NULL) 316 return (DEFAULT_SAMPLE_COUNT); 317 if (pmu_parse_event(&ped, pe->event)) 318 return (DEFAULT_SAMPLE_COUNT); 319 return (ped.ped_period); 320 } 321 322 int 323 pmc_pmu_enabled(void) 324 { 325 326 return (pmu_events_map_get(NULL) != NULL); 327 } 328 329 void 330 pmc_pmu_print_counters(const char *event_name) 331 { 332 const struct pmu_events_map *pme; 333 const struct pmu_event *pe; 334 struct pmu_event_desc ped; 335 char *debug; 336 int do_debug; 337 338 debug = getenv("PMUDEBUG"); 339 do_debug = 0; 340 341 if (debug != NULL && strcmp(debug, "true") == 0) 342 do_debug = 1; 343 if ((pme = pmu_events_map_get(NULL)) == NULL) 344 return; 345 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 346 if (pe->name == NULL) 347 continue; 348 if (event_name != NULL && strcasestr(pe->name, event_name) == NULL) 349 continue; 350 printf("\t%s\n", pe->name); 351 if (do_debug) 352 pmu_parse_event(&ped, pe->event); 353 } 354 } 355 356 void 357 pmc_pmu_print_counter_desc(const char *ev) 358 { 359 const struct pmu_events_map *pme; 360 const struct pmu_event *pe; 361 362 if ((pme = pmu_events_map_get(NULL)) == NULL) 363 return; 364 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 365 if (pe->name == NULL) 366 continue; 367 if (strcasestr(pe->name, ev) != NULL && 368 pe->desc != NULL) 369 printf("%s:\t%s\n", pe->name, pe->desc); 370 } 371 } 372 373 void 374 pmc_pmu_print_counter_desc_long(const char *ev) 375 { 376 const struct pmu_events_map *pme; 377 const struct pmu_event *pe; 378 379 if ((pme = pmu_events_map_get(NULL)) == NULL) 380 return; 381 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 382 if (pe->name == NULL) 383 continue; 384 if (strcasestr(pe->name, ev) != NULL) { 385 if (pe->long_desc != NULL) 386 printf("%s:\n%s\n", pe->name, pe->long_desc); 387 else if (pe->desc != NULL) 388 printf("%s:\t%s\n", pe->name, pe->desc); 389 } 390 } 391 } 392 393 void 394 pmc_pmu_print_counter_full(const char *ev) 395 { 396 const struct pmu_events_map *pme; 397 const struct pmu_event *pe; 398 399 if ((pme = pmu_events_map_get(NULL)) == NULL) 400 return; 401 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) { 402 if (pe->name == NULL) 403 continue; 404 if (strcasestr(pe->name, ev) == NULL) 405 continue; 406 printf("name: %s\n", pe->name); 407 if (pe->long_desc != NULL) 408 printf("desc: %s\n", pe->long_desc); 409 else if (pe->desc != NULL) 410 printf("desc: %s\n", pe->desc); 411 if (pe->event != NULL) 412 printf("event: %s\n", pe->event); 413 if (pe->topic != NULL) 414 printf("topic: %s\n", pe->topic); 415 if (pe->pmu != NULL) 416 printf("pmu: %s\n", pe->pmu); 417 if (pe->unit != NULL) 418 printf("unit: %s\n", pe->unit); 419 if (pe->perpkg != NULL) 420 printf("perpkg: %s\n", pe->perpkg); 421 if (pe->metric_expr != NULL) 422 printf("metric_expr: %s\n", pe->metric_expr); 423 if (pe->metric_name != NULL) 424 printf("metric_name: %s\n", pe->metric_name); 425 if (pe->metric_group != NULL) 426 printf("metric_group: %s\n", pe->metric_group); 427 } 428 } 429 430 #if defined(__amd64__) || defined(__i386__) 431 static int 432 pmc_pmu_amd_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm, 433 struct pmu_event_desc *ped) 434 { 435 struct pmc_md_amd_op_pmcallocate *amd; 436 const struct pmu_event *pe; 437 int idx = -1; 438 439 amd = &pm->pm_md.pm_amd; 440 if (ped->ped_umask > 0) { 441 pm->pm_caps |= PMC_CAP_QUALIFIER; 442 amd->pm_amd_config |= AMD_PMC_TO_UNITMASK(ped->ped_umask); 443 } 444 pm->pm_class = PMC_CLASS_K8; 445 pe = pmu_event_get(NULL, event_name, &idx); 446 447 if (strcmp("l3cache", pe->topic) == 0){ 448 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event); 449 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_L3_CACHE; 450 amd->pm_amd_config |= AMD_PMC_TO_L3SLICE(ped->ped_l3_slice); 451 amd->pm_amd_config |= AMD_PMC_TO_L3CORE(ped->ped_l3_thread); 452 } 453 else if (strcmp("data fabric", pe->topic) == 0){ 454 455 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK_DF(ped->ped_event); 456 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_DATA_FABRIC; 457 } 458 else{ 459 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event); 460 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_CORE; 461 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 || 462 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 463 (PMC_CAP_USER|PMC_CAP_SYSTEM)) 464 amd->pm_amd_config |= (AMD_PMC_USR | AMD_PMC_OS); 465 else if (pm->pm_caps & PMC_CAP_USER) 466 amd->pm_amd_config |= AMD_PMC_USR; 467 else if (pm->pm_caps & PMC_CAP_SYSTEM) 468 amd->pm_amd_config |= AMD_PMC_OS; 469 if (ped->ped_edge) 470 amd->pm_amd_config |= AMD_PMC_EDGE; 471 if (ped->ped_inv) 472 amd->pm_amd_config |= AMD_PMC_EDGE; 473 if (pm->pm_caps & PMC_CAP_INTERRUPT) 474 amd->pm_amd_config |= AMD_PMC_INT; 475 } 476 return (0); 477 } 478 479 static int 480 pmc_pmu_intel_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm, 481 struct pmu_event_desc *ped) 482 { 483 struct pmc_md_iap_op_pmcallocate *iap; 484 485 iap = &pm->pm_md.pm_iap; 486 if (strcasestr(event_name, "UNC_") == event_name || 487 strcasestr(event_name, "uncore") != NULL) { 488 pm->pm_class = PMC_CLASS_UCP; 489 pm->pm_caps |= PMC_CAP_QUALIFIER; 490 } else if ((ped->ped_umask == -1) || 491 (ped->ped_event == 0x0 && ped->ped_umask == 0x3)) { 492 pm->pm_class = PMC_CLASS_IAF; 493 } else { 494 pm->pm_class = PMC_CLASS_IAP; 495 pm->pm_caps |= PMC_CAP_QUALIFIER; 496 } 497 iap->pm_iap_config |= IAP_EVSEL(ped->ped_event); 498 if (ped->ped_umask > 0) 499 iap->pm_iap_config |= IAP_UMASK(ped->ped_umask); 500 iap->pm_iap_config |= IAP_CMASK(ped->ped_cmask); 501 iap->pm_iap_rsp = ped->ped_offcore_rsp; 502 503 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 || 504 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 505 (PMC_CAP_USER|PMC_CAP_SYSTEM)) 506 iap->pm_iap_config |= (IAP_USR | IAP_OS); 507 else if (pm->pm_caps & PMC_CAP_USER) 508 iap->pm_iap_config |= IAP_USR; 509 else if (pm->pm_caps & PMC_CAP_SYSTEM) 510 iap->pm_iap_config |= IAP_OS; 511 if (ped->ped_edge) 512 iap->pm_iap_config |= IAP_EDGE; 513 if (ped->ped_any) 514 iap->pm_iap_config |= IAP_ANY; 515 if (ped->ped_inv) 516 iap->pm_iap_config |= IAP_EDGE; 517 if (pm->pm_caps & PMC_CAP_INTERRUPT) 518 iap->pm_iap_config |= IAP_INT; 519 return (0); 520 } 521 522 int 523 pmc_pmu_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm) 524 { 525 const struct pmu_event *pe; 526 struct pmu_event_desc ped; 527 pmu_mfr_t mfr; 528 int idx = -1; 529 530 if ((mfr = pmu_events_mfr()) == PMU_INVALID) 531 return (ENOENT); 532 533 bzero(&pm->pm_md, sizeof(pm->pm_md)); 534 pm->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE); 535 event_name = pmu_alias_get(event_name); 536 if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL) 537 return (ENOENT); 538 assert(idx >= 0); 539 pm->pm_ev = idx; 540 541 if (pe->event == NULL) 542 return (ENOENT); 543 if (pmu_parse_event(&ped, pe->event)) 544 return (ENOENT); 545 546 if (mfr == PMU_INTEL) 547 return (pmc_pmu_intel_pmcallocate(event_name, pm, &ped)); 548 else 549 return (pmc_pmu_amd_pmcallocate(event_name, pm, &ped)); 550 } 551 552 #else 553 554 int 555 pmc_pmu_pmcallocate(const char *e __unused, struct pmc_op_pmcallocate *p __unused) 556 { 557 return (EOPNOTSUPP); 558 } 559 #endif 560