1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018, Matthew Macy
5 * Copyright (c) 2021, The FreeBSD Foundation
6 *
7 * Portions of this software were developed by Mitchell Horne
8 * under sponsorship from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 */
32
33 #include <sys/types.h>
34 #include <sys/errno.h>
35 #include <sys/pmc.h>
36 #include <sys/sysctl.h>
37 #include <stddef.h>
38 #include <stdlib.h>
39 #include <limits.h>
40 #include <regex.h>
41 #include <string.h>
42 #include <pmc.h>
43 #include <pmclog.h>
44 #include <assert.h>
45 #include <libpmcstat.h>
46 #include "pmu-events/pmu-events.h"
47
48 struct pmu_alias {
49 const char *pa_alias;
50 const char *pa_name;
51 };
52
53 #if defined(__amd64__) || defined(__i386__)
54 typedef enum {
55 PMU_INVALID,
56 PMU_INTEL,
57 PMU_AMD,
58 } pmu_mfr_t;
59
60 static struct pmu_alias pmu_intel_alias_table[] = {
61 {"UNHALTED_CORE_CYCLES", "cpu_clk_unhalted.thread"},
62 {"UNHALTED-CORE-CYCLES", "cpu_clk_unhalted.thread"},
63 {"LLC_MISSES", "LONGEST_LAT_CACHE.MISS"},
64 {"LLC-MISSES", "LONGEST_LAT_CACHE.MISS"},
65 {"LLC_REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"},
66 {"LLC-REFERENCE", "LONGEST_LAT_CACHE.REFERENCE"},
67 {"LLC_MISS_RHITM", "mem_load_l3_miss_retired.remote_hitm"},
68 {"LLC-MISS-RHITM", "mem_load_l3_miss_retired.remote_hitm"},
69 {"RESOURCE_STALL", "RESOURCE_STALLS.ANY"},
70 {"RESOURCE_STALLS_ANY", "RESOURCE_STALLS.ANY"},
71 {"BRANCH_INSTRUCTION_RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"},
72 {"BRANCH-INSTRUCTION-RETIRED", "BR_INST_RETIRED.ALL_BRANCHES"},
73 {"BRANCH_MISSES_RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"},
74 {"BRANCH-MISSES-RETIRED", "BR_MISP_RETIRED.ALL_BRANCHES"},
75 {"unhalted-cycles", "cpu_clk_unhalted.thread"},
76 {"instructions", "inst_retired.any"},
77 {"branch-mispredicts", "br_misp_retired.all_branches"},
78 {"branches", "br_inst_retired.all_branches"},
79 {"interrupts", "hw_interrupts.received"},
80 {"ic-misses", "frontend_retired.l1i_miss"},
81 {NULL, NULL},
82 };
83
84 static struct pmu_alias pmu_amd_alias_table[] = {
85 {"UNHALTED_CORE_CYCLES", "ls_not_halted_cyc"},
86 {"UNHALTED-CORE-CYCLES", "ls_not_halted_cyc"},
87 {"LLC_MISSES", "l3_comb_clstr_state.request_miss"},
88 {"LLC-MISSES", "l3_comb_clstr_state.request_miss"},
89 {"LLC_REFERENCE", "l3_request_g1.caching_l3_cache_accesses"},
90 {"LLC-REFERENCE", "l3_request_g1.caching_l3_cache_accesses"},
91 {"BRANCH_INSTRUCTION_RETIRED", "ex_ret_brn"},
92 {"BRANCH-INSTRUCTION-RETIRED", "ex_ret_brn"},
93 {"BRANCH_MISSES_RETIRED", "ex_ret_brn_misp"},
94 {"BRANCH-MISSES-RETIRED", "ex_ret_brn_misp"},
95 {"unhalted-cycles", "ls_not_halted_cyc"},
96 {"instructions", "ex_ret_instr",},
97 {"branch-mispredicts", "ex_ret_brn_misp"},
98 {"branches", "ex_ret_brn"},
99 {"interrupts", "ls_int_taken"}, /* Not on amdzen1 */
100 {NULL, NULL},
101 };
102
103
104 static pmu_mfr_t
pmu_events_mfr(void)105 pmu_events_mfr(void)
106 {
107 char buf[PMC_CPUID_LEN];
108 size_t s = sizeof(buf);
109 pmu_mfr_t mfr;
110
111 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
112 (void *)NULL, 0) == -1)
113 return (PMU_INVALID);
114 if (strcasestr(buf, "AuthenticAMD") != NULL ||
115 strcasestr(buf, "HygonGenuine") != NULL)
116 mfr = PMU_AMD;
117 else if (strcasestr(buf, "GenuineIntel") != NULL)
118 mfr = PMU_INTEL;
119 else
120 mfr = PMU_INVALID;
121 return (mfr);
122 }
123
124 static int
pmu_events_x86_family(void)125 pmu_events_x86_family(void)
126 {
127 char buf[PMC_CPUID_LEN];
128 size_t s = sizeof(buf);
129 char *cpuid, *family;
130
131 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
132 (void *)NULL, 0) == -1)
133 return (-1);
134 cpuid = &buf[0];
135
136 strsep(&cpuid, "-");
137 family = strsep(&cpuid, "-");
138
139 return (strtol(family, NULL, 10));
140 }
141
142 /*
143 * The Intel fixed mode counters are:
144 * "inst_retired.any",
145 * "cpu_clk_unhalted.thread",
146 * "cpu_clk_unhalted.thread_any",
147 * "cpu_clk_unhalted.ref_tsc",
148 *
149 */
150
151 static const char *
pmu_alias_get(const char * name)152 pmu_alias_get(const char *name)
153 {
154 pmu_mfr_t mfr;
155 struct pmu_alias *pa;
156 struct pmu_alias *pmu_alias_table;
157
158 if ((mfr = pmu_events_mfr()) == PMU_INVALID)
159 return (name);
160 if (mfr == PMU_AMD)
161 pmu_alias_table = pmu_amd_alias_table;
162 else if (mfr == PMU_INTEL)
163 pmu_alias_table = pmu_intel_alias_table;
164 else
165 return (name);
166
167 for (pa = pmu_alias_table; pa->pa_alias != NULL; pa++)
168 if (strcasecmp(name, pa->pa_alias) == 0)
169 return (pa->pa_name);
170
171 return (name);
172 }
173 #elif defined(__powerpc64__)
174
175 static const char *
pmu_alias_get(const char * name)176 pmu_alias_get(const char *name)
177 {
178 return (name);
179 }
180
181 #elif defined(__aarch64__)
182
183 static struct pmu_alias pmu_armv8_alias_table[] = {
184 {"UNHALTED_CORE_CYCLES", "CPU_CYCLES"},
185 {"UNHALTED-CORE-CYCLES", "CPU_CYCLES"},
186 {"LLC_MISSES", "LL_CACHE_MISS_RD"},
187 {"LLC-MISSES", "LL_CACHE_MISS_RD"},
188 {"LLC_REFERENCE", "LL_CACHE_RD"},
189 {"LLC-REFERENCE", "LL_CACHE_RD"},
190 {"BRANCH_INSTRUCTION_RETIRED", "BR_RETIRED"},
191 {"BRANCH-INSTRUCTION-RETIRED", "BR_RETIRED"},
192 {"BRANCH_MISSES_RETIRED", "BR_MIS_PRED_RETIRED"},
193 {"BRANCH-MISSES-RETIRED", "BR_MIS_PRED_RETIRED"},
194 {"unhalted-cycles", "CPU_CYCLES"},
195 {"instructions", "INST_RETIRED",},
196 {"branch-mispredicts", "BR_MIS_PRED_RETIRED"},
197 {"branches", "BR_RETIRED"},
198 {"interrupts", "EXC_IRQ"},
199 {NULL, NULL},
200 };
201
202 static const char *
pmu_alias_get(const char * name)203 pmu_alias_get(const char *name)
204 {
205 struct pmu_alias *pa;
206
207 for (pa = pmu_armv8_alias_table; pa->pa_alias != NULL; pa++)
208 if (strcasecmp(name, pa->pa_alias) == 0)
209 return (pa->pa_name);
210
211 return (name);
212 }
213
214 #else
215
216 static const char *
pmu_alias_get(const char * name)217 pmu_alias_get(const char *name)
218 {
219
220 return (name);
221 }
222 #endif
223
224 struct pmu_event_desc {
225 uint64_t ped_period;
226 uint64_t ped_offcore_rsp;
227 uint64_t ped_l3_thread;
228 uint64_t ped_l3_slice;
229 uint32_t ped_sourceid;
230 uint32_t ped_coreid;
231 uint32_t ped_allsources;
232 uint32_t ped_allcores;
233 uint32_t ped_rdwrmask;
234 uint32_t ped_event;
235 uint32_t ped_frontend;
236 uint32_t ped_ldlat;
237 uint32_t ped_config1;
238 int16_t ped_umask;
239 uint8_t ped_cmask;
240 uint8_t ped_any;
241 uint8_t ped_inv;
242 uint8_t ped_edge;
243 uint8_t ped_fc_mask;
244 uint8_t ped_ch_mask;
245 uint8_t ped_pebs;
246 };
247
248 static const struct pmu_events_map *
pmu_events_map_get(const char * cpuid)249 pmu_events_map_get(const char *cpuid)
250 {
251 regex_t re;
252 regmatch_t pmatch[1];
253 char buf[PMC_CPUID_LEN];
254 size_t s = sizeof(buf);
255 int match;
256 const struct pmu_events_map *pme;
257
258 if (cpuid != NULL) {
259 strlcpy(buf, cpuid, s);
260 } else {
261 if (sysctlbyname("kern.hwpmc.cpuid", buf, &s,
262 (void *)NULL, 0) == -1)
263 return (NULL);
264 }
265 for (pme = pmu_events_map; pme->cpuid != NULL; pme++) {
266 if (regcomp(&re, pme->cpuid, REG_EXTENDED) != 0) {
267 printf("regex '%s' failed to compile, ignoring\n",
268 pme->cpuid);
269 continue;
270 }
271 match = regexec(&re, buf, 1, pmatch, 0);
272 regfree(&re);
273 if (match == 0) {
274 if (pmatch[0].rm_so == 0 && (buf[pmatch[0].rm_eo] == 0
275 || buf[pmatch[0].rm_eo] == '-'))
276 return (pme);
277 }
278 }
279 return (NULL);
280 }
281
282 static const struct pmu_event *
pmu_event_get(const char * cpuid,const char * event_name,int * idx)283 pmu_event_get(const char *cpuid, const char *event_name, int *idx)
284 {
285 const struct pmu_events_map *pme;
286 const struct pmu_event *pe;
287 int i;
288
289 if ((pme = pmu_events_map_get(cpuid)) == NULL)
290 return (NULL);
291 for (i = 0, pe = pme->table; pe->name || pe->desc || pe->event; pe++, i++) {
292 if (pe->name == NULL)
293 continue;
294 if (strcasecmp(pe->name, event_name) == 0) {
295 if (idx)
296 *idx = i;
297 return (pe);
298 }
299 }
300 return (NULL);
301 }
302
303 int
pmc_pmu_idx_get_by_event(const char * cpuid,const char * event)304 pmc_pmu_idx_get_by_event(const char *cpuid, const char *event)
305 {
306 int idx;
307 const char *realname;
308
309 realname = pmu_alias_get(event);
310 if (pmu_event_get(cpuid, realname, &idx) == NULL)
311 return (-1);
312 return (idx);
313 }
314
315 const char *
pmc_pmu_event_get_by_idx(const char * cpuid,int idx)316 pmc_pmu_event_get_by_idx(const char *cpuid, int idx)
317 {
318 const struct pmu_events_map *pme;
319
320 if ((pme = pmu_events_map_get(cpuid)) == NULL)
321 return (NULL);
322 assert(pme->table[idx].name);
323 return (pme->table[idx].name);
324 }
325
326 static int
pmu_parse_event(struct pmu_event_desc * ped,const char * eventin)327 pmu_parse_event(struct pmu_event_desc *ped, const char *eventin)
328 {
329 char *event;
330 char *kvp, *key, *value, *r;
331 char *debug;
332
333 if ((event = strdup(eventin)) == NULL)
334 return (ENOMEM);
335 r = event;
336 bzero(ped, sizeof(*ped));
337 ped->ped_period = DEFAULT_SAMPLE_COUNT;
338 ped->ped_umask = -1;
339 while ((kvp = strsep(&event, ",")) != NULL) {
340 key = strsep(&kvp, "=");
341 if (key == NULL)
342 abort();
343 value = kvp;
344 if (strcmp(key, "umask") == 0)
345 ped->ped_umask = strtol(value, NULL, 16);
346 else if (strcmp(key, "event") == 0)
347 ped->ped_event = strtol(value, NULL, 16);
348 else if (strcmp(key, "period") == 0)
349 ped->ped_period = strtol(value, NULL, 10);
350 else if (strcmp(key, "offcore_rsp") == 0)
351 ped->ped_offcore_rsp = strtol(value, NULL, 16);
352 else if (strcmp(key, "any") == 0)
353 ped->ped_any = strtol(value, NULL, 10);
354 else if (strcmp(key, "cmask") == 0)
355 ped->ped_cmask = strtol(value, NULL, 10);
356 else if (strcmp(key, "inv") == 0)
357 ped->ped_inv = strtol(value, NULL, 10);
358 else if (strcmp(key, "edge") == 0)
359 ped->ped_edge = strtol(value, NULL, 10);
360 else if (strcmp(key, "frontend") == 0)
361 ped->ped_frontend = strtol(value, NULL, 16);
362 else if (strcmp(key, "ldlat") == 0)
363 ped->ped_ldlat = strtol(value, NULL, 16);
364 else if (strcmp(key, "fc_mask") == 0)
365 ped->ped_fc_mask = strtol(value, NULL, 16);
366 else if (strcmp(key, "ch_mask") == 0)
367 ped->ped_ch_mask = strtol(value, NULL, 16);
368 else if (strcmp(key, "config1") == 0)
369 ped->ped_config1 = strtol(value, NULL, 16);
370 else if (strcmp(key, "l3_thread_mask") == 0)
371 ped->ped_l3_thread = strtol(value, NULL, 16);
372 else if (strcmp(key, "l3_slice_mask") == 0)
373 ped->ped_l3_slice = strtol(value, NULL, 16);
374 else if (strcmp(key, "sourceid") == 0)
375 ped->ped_sourceid = strtol(value, NULL, 0);
376 else if (strcmp(key, "coreid") == 0)
377 ped->ped_coreid = strtol(value, NULL, 0);
378 else if (strcmp(key, "allcores") == 0)
379 ped->ped_allcores = strtol(value, NULL, 0);
380 else if (strcmp(key, "allsources") == 0)
381 ped->ped_allsources = strtol(value, NULL, 0);
382 else if (strcmp(key, "rdwrmask") == 0)
383 ped->ped_rdwrmask = strtol(value, NULL, 0);
384 else if (strcmp(key, "pebs") == 0)
385 ped->ped_pebs = strtol(value, NULL, 10);
386 else {
387 debug = getenv("PMUDEBUG");
388 if (debug != NULL && strcmp(debug, "true") == 0 && value != NULL)
389 printf("unrecognized kvpair: %s:%s\n", key, value);
390 }
391 }
392 free(r);
393 return (0);
394 }
395
396 uint64_t
pmc_pmu_sample_rate_get(const char * event_name)397 pmc_pmu_sample_rate_get(const char *event_name)
398 {
399 const struct pmu_event *pe;
400 struct pmu_event_desc ped;
401
402 event_name = pmu_alias_get(event_name);
403 if ((pe = pmu_event_get(NULL, event_name, NULL)) == NULL)
404 return (DEFAULT_SAMPLE_COUNT);
405 if (pe->event == NULL)
406 return (DEFAULT_SAMPLE_COUNT);
407 if (pmu_parse_event(&ped, pe->event))
408 return (DEFAULT_SAMPLE_COUNT);
409 return (ped.ped_period);
410 }
411
412 int
pmc_pmu_enabled(void)413 pmc_pmu_enabled(void)
414 {
415
416 return (pmu_events_map_get(NULL) != NULL);
417 }
418
419 void
pmc_pmu_print_counters(const char * event_name)420 pmc_pmu_print_counters(const char *event_name)
421 {
422 const struct pmu_events_map *pme;
423 const struct pmu_event *pe;
424 struct pmu_event_desc ped;
425 char *debug;
426 int do_debug;
427
428 debug = getenv("PMUDEBUG");
429 do_debug = 0;
430
431 if (debug != NULL && strcmp(debug, "true") == 0)
432 do_debug = 1;
433 if ((pme = pmu_events_map_get(NULL)) == NULL)
434 return;
435 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
436 if (pe->name == NULL)
437 continue;
438 if (event_name != NULL && strcasestr(pe->name, event_name) == NULL)
439 continue;
440 printf("\t%s\n", pe->name);
441 if (do_debug)
442 pmu_parse_event(&ped, pe->event);
443 }
444 }
445
446 void
pmc_pmu_print_counter_desc(const char * ev)447 pmc_pmu_print_counter_desc(const char *ev)
448 {
449 const struct pmu_events_map *pme;
450 const struct pmu_event *pe;
451
452 if ((pme = pmu_events_map_get(NULL)) == NULL)
453 return;
454 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
455 if (pe->name == NULL)
456 continue;
457 if (strcasestr(pe->name, ev) != NULL &&
458 pe->desc != NULL)
459 printf("%s:\t%s\n", pe->name, pe->desc);
460 }
461 }
462
463 void
pmc_pmu_print_counter_desc_long(const char * ev)464 pmc_pmu_print_counter_desc_long(const char *ev)
465 {
466 const struct pmu_events_map *pme;
467 const struct pmu_event *pe;
468
469 if ((pme = pmu_events_map_get(NULL)) == NULL)
470 return;
471 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
472 if (pe->name == NULL)
473 continue;
474 if (strcasestr(pe->name, ev) != NULL) {
475 if (pe->long_desc != NULL)
476 printf("%s:\n%s\n", pe->name, pe->long_desc);
477 else if (pe->desc != NULL)
478 printf("%s:\t%s\n", pe->name, pe->desc);
479 }
480 }
481 }
482
483 void
pmc_pmu_print_counter_full(const char * ev)484 pmc_pmu_print_counter_full(const char *ev)
485 {
486 const struct pmu_events_map *pme;
487 const struct pmu_event *pe;
488
489 if ((pme = pmu_events_map_get(NULL)) == NULL)
490 return;
491 for (pe = pme->table; pe->name || pe->desc || pe->event; pe++) {
492 if (pe->name == NULL)
493 continue;
494 if (strcasestr(pe->name, ev) == NULL)
495 continue;
496 if (pe != pme->table)
497 printf("\n");
498 printf("name: %s\n", pe->name);
499 if (pe->long_desc != NULL)
500 printf("desc: %s\n", pe->long_desc);
501 else if (pe->desc != NULL)
502 printf("desc: %s\n", pe->desc);
503 if (pe->event != NULL)
504 printf("event: %s\n", pe->event);
505 if (pe->topic != NULL)
506 printf("topic: %s\n", pe->topic);
507 if (pe->pmu != NULL)
508 printf("pmu: %s\n", pe->pmu);
509 if (pe->unit != NULL)
510 printf("unit: %s\n", pe->unit);
511 if (pe->perpkg != NULL)
512 printf("perpkg: %s\n", pe->perpkg);
513 if (pe->metric_expr != NULL)
514 printf("metric_expr: %s\n", pe->metric_expr);
515 if (pe->metric_name != NULL)
516 printf("metric_name: %s\n", pe->metric_name);
517 if (pe->metric_group != NULL)
518 printf("metric_group: %s\n", pe->metric_group);
519 }
520 }
521
522 #if defined(__amd64__) || defined(__i386__)
523 static int
pmc_pmu_amd_pmcallocate(const char * event_name,struct pmc_op_pmcallocate * pm,struct pmu_event_desc * ped)524 pmc_pmu_amd_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm,
525 struct pmu_event_desc *ped)
526 {
527 int cpu_family;
528 struct pmc_md_amd_op_pmcallocate *amd;
529 const struct pmu_event *pe;
530 int idx = -1;
531
532 cpu_family = pmu_events_x86_family();
533
534 amd = &pm->pm_md.pm_amd;
535 if (ped->ped_umask > 0) {
536 pm->pm_caps |= PMC_CAP_QUALIFIER;
537 }
538 pm->pm_class = PMC_CLASS_K8;
539 pe = pmu_event_get(NULL, event_name, &idx);
540
541 if (pe->pmu == NULL) {
542 amd->pm_amd_config |= AMD_PMC_TO_EVENTMASK(ped->ped_event);
543 amd->pm_amd_config |= AMD_PMC_TO_UNITMASK(ped->ped_umask);
544 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_CORE;
545 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 ||
546 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) ==
547 (PMC_CAP_USER|PMC_CAP_SYSTEM))
548 amd->pm_amd_config |= (AMD_PMC_USR | AMD_PMC_OS);
549 else if (pm->pm_caps & PMC_CAP_USER)
550 amd->pm_amd_config |= AMD_PMC_USR;
551 else if (pm->pm_caps & PMC_CAP_SYSTEM)
552 amd->pm_amd_config |= AMD_PMC_OS;
553 if (ped->ped_edge)
554 amd->pm_amd_config |= AMD_PMC_EDGE;
555 if (ped->ped_inv)
556 amd->pm_amd_config |= AMD_PMC_INVERT;
557 if (pm->pm_caps & PMC_CAP_INTERRUPT)
558 amd->pm_amd_config |= AMD_PMC_INT;
559 if (pm->pm_caps & PMC_CAP_PRECISE)
560 amd->pm_amd_config |= AMD_PMC_PRECISERETIRE;
561 } else if (strcmp("amd_l3", pe->pmu) == 0) {
562 amd->pm_amd_config |= AMD_PMC_L3_TO_EVENTMASK(ped->ped_event);
563 amd->pm_amd_config |= AMD_PMC_L3_TO_UNITMASK(ped->ped_umask);
564 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_L3_CACHE;
565 if (cpu_family <= 0x17) {
566 amd->pm_amd_config |=
567 AMD_PMC_L31_TO_SLICE(ped->ped_l3_slice);
568 amd->pm_amd_config |=
569 AMD_PMC_L31_TO_CORE(ped->ped_l3_thread);
570 } else {
571 amd->pm_amd_config |=
572 AMD_PMC_L32_TO_THREAD(ped->ped_l3_thread);
573 amd->pm_amd_config |=
574 AMD_PMC_L32_TO_SOURCEID(ped->ped_sourceid);
575 amd->pm_amd_config |=
576 AMD_PMC_L32_TO_COREID(ped->ped_coreid);
577 if (ped->ped_allcores)
578 amd->pm_amd_config |= AMD_PMC_L32_ALLCORES;
579 if (ped->ped_allsources)
580 amd->pm_amd_config |= AMD_PMC_L32_ALLSOURCES;
581 }
582 } else if (strcmp("amd_df", pe->pmu) == 0) {
583 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_DATA_FABRIC;
584 if (cpu_family <= 19) {
585 amd->pm_amd_config |=
586 AMD_PMC_DF1_TO_EVENTMASK(ped->ped_event);
587 amd->pm_amd_config |=
588 AMD_PMC_DF1_TO_UNITMASK(ped->ped_umask);
589 } else {
590 amd->pm_amd_config |=
591 AMD_PMC_DF2_TO_EVENTMASK(ped->ped_event);
592 amd->pm_amd_config |=
593 AMD_PMC_DF2_TO_UNITMASK(ped->ped_umask);
594 }
595 } else if (strcmp("amd_umc", pe->pmu) == 0) {
596 amd->pm_amd_sub_class = PMC_AMD_SUB_CLASS_UMC;
597 amd->pm_amd_config |= AMD_PMC_UMC_TO_EVENTMASK(ped->ped_event);
598 amd->pm_amd_config |= AMD_PMC_UMC_TO_RDWRMASK(ped->ped_rdwrmask);
599 } else {
600 printf("PMC pmu '%s' is not supported!\n", pe->pmu);
601 return (EOPNOTSUPP);
602 }
603
604 return (0);
605 }
606
607 static int
pmc_pmu_intel_pmcallocate(const char * event_name,struct pmc_op_pmcallocate * pm,struct pmu_event_desc * ped)608 pmc_pmu_intel_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm,
609 struct pmu_event_desc *ped)
610 {
611 struct pmc_md_iap_op_pmcallocate *iap;
612
613 iap = &pm->pm_md.pm_iap;
614 if (strcasestr(event_name, "UNC_") == event_name ||
615 strcasestr(event_name, "uncore") != NULL) {
616 pm->pm_class = PMC_CLASS_UCP;
617 pm->pm_caps |= PMC_CAP_QUALIFIER;
618 /* XXX Check and block unsupported uncore counters */
619 } else if (ped->ped_event == 0x0) {
620 pm->pm_class = PMC_CLASS_IAF;
621 } else {
622 pm->pm_class = PMC_CLASS_IAP;
623 pm->pm_caps |= PMC_CAP_QUALIFIER;
624 }
625 iap->pm_iap_config |= IAP_EVSEL(ped->ped_event);
626 if (ped->ped_umask > 0)
627 iap->pm_iap_config |= IAP_UMASK(ped->ped_umask);
628 iap->pm_iap_config |= IAP_CMASK(ped->ped_cmask);
629 iap->pm_iap_rsp = ped->ped_offcore_rsp;
630
631 if ((pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0 ||
632 (pm->pm_caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) ==
633 (PMC_CAP_USER|PMC_CAP_SYSTEM))
634 iap->pm_iap_config |= (IAP_USR | IAP_OS);
635 else if (pm->pm_caps & PMC_CAP_USER)
636 iap->pm_iap_config |= IAP_USR;
637 else if (pm->pm_caps & PMC_CAP_SYSTEM)
638 iap->pm_iap_config |= IAP_OS;
639 if (ped->ped_edge)
640 iap->pm_iap_config |= IAP_EDGE;
641 if (ped->ped_any)
642 iap->pm_iap_config |= IAP_ANY;
643 if (ped->ped_inv)
644 iap->pm_iap_config |= IAP_INV;
645 if (pm->pm_caps & PMC_CAP_INTERRUPT)
646 iap->pm_iap_config |= IAP_INT;
647
648 /* XXX Implement alone flag in the kernel module */
649
650 /*
651 * PEBS counters are not implemented on FreeBSD yet. Counters labeled
652 * with PEBS = 2 in the JSON definitions require PEBS. It seems that
653 * some of them return bogus counts if you attempt to use them.
654 */
655 if (ped->ped_pebs == 2) {
656 printf("PEBS only counters are not supported!\n");
657 return (EOPNOTSUPP);
658 }
659
660 return (0);
661 }
662
663 static int
pmc_pmu_pmcallocate_md(const char * event_name,struct pmc_op_pmcallocate * pm)664 pmc_pmu_pmcallocate_md(const char *event_name, struct pmc_op_pmcallocate *pm)
665 {
666 const struct pmu_event *pe;
667 struct pmu_event_desc ped;
668 pmu_mfr_t mfr;
669 int idx = -1;
670
671 if ((mfr = pmu_events_mfr()) == PMU_INVALID)
672 return (ENOENT);
673
674 bzero(&pm->pm_md, sizeof(pm->pm_md));
675 pm->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
676 event_name = pmu_alias_get(event_name);
677 if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL)
678 return (ENOENT);
679 assert(idx >= 0);
680 pm->pm_ev = idx;
681
682 if (pe->event == NULL)
683 return (ENOENT);
684 if (pmu_parse_event(&ped, pe->event))
685 return (ENOENT);
686
687 if (mfr == PMU_INTEL)
688 return (pmc_pmu_intel_pmcallocate(event_name, pm, &ped));
689 else
690 return (pmc_pmu_amd_pmcallocate(event_name, pm, &ped));
691 }
692
693 #elif defined(__powerpc64__)
694
695 static int
pmc_pmu_pmcallocate_md(const char * event_name,struct pmc_op_pmcallocate * pm)696 pmc_pmu_pmcallocate_md(const char *event_name, struct pmc_op_pmcallocate *pm)
697 {
698 const struct pmu_event *pe;
699 struct pmu_event_desc ped;
700 int idx = -1;
701
702 bzero(&pm->pm_md, sizeof(pm->pm_md));
703 pm->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
704 event_name = pmu_alias_get(event_name);
705
706 if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL)
707 return (ENOENT);
708 if (pe->event == NULL)
709 return (ENOENT);
710 if (pmu_parse_event(&ped, pe->event))
711 return (ENOENT);
712
713 pm->pm_ev = idx;
714 pm->pm_md.pm_event = ped.ped_event;
715 pm->pm_class = PMC_CLASS_POWER8;
716 return (0);
717 }
718
719 #elif defined(__aarch64__)
720
721 static int
pmc_pmu_pmcallocate_md(const char * event_name,struct pmc_op_pmcallocate * pm)722 pmc_pmu_pmcallocate_md(const char *event_name, struct pmc_op_pmcallocate *pm)
723 {
724 const struct pmu_event *pe;
725 struct pmu_event_desc ped;
726 int idx = -1;
727
728 event_name = pmu_alias_get(event_name);
729 if ((pe = pmu_event_get(NULL, event_name, &idx)) == NULL)
730 return (ENOENT);
731 if (pe->event == NULL)
732 return (ENOENT);
733 if (pmu_parse_event(&ped, pe->event))
734 return (ENOENT);
735
736 assert(idx >= 0);
737 pm->pm_ev = idx;
738 pm->pm_md.pm_md_config = ped.ped_event;
739 pm->pm_class = PMC_CLASS_ARMV8;
740 pm->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
741
742 return (0);
743 }
744
745 #else
746
747 static int
pmc_pmu_pmcallocate_md(const char * e __unused,struct pmc_op_pmcallocate * p __unused)748 pmc_pmu_pmcallocate_md(const char *e __unused, struct pmc_op_pmcallocate *p __unused)
749 {
750 return (EOPNOTSUPP);
751 }
752 #endif
753
754 int
pmc_pmu_pmcallocate(const char * event_name,struct pmc_op_pmcallocate * pm)755 pmc_pmu_pmcallocate(const char *event_name, struct pmc_op_pmcallocate *pm)
756 {
757 int error;
758
759 error = pmc_pmu_pmcallocate_md(event_name, pm);
760 if (error != 0) {
761 /* Reset any changes. */
762 pm->pm_ev = 0;
763 pm->pm_caps = 0;
764 pm->pm_class = 0;
765
766 return (error);
767 }
768
769 pm->pm_flags |= PMC_F_EV_PMU;
770 return (0);
771 }
772