1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com> 4 * 5 * Parts came from builtin-{top,stat,record}.c, see those files for further 6 * copyright notes. 7 */ 8 /* 9 * Powerpc needs __SANE_USERSPACE_TYPES__ before <linux/types.h> to select 10 * 'int-ll64.h' and avoid compile warnings when printing __u64 with %llu. 11 */ 12 #define __SANE_USERSPACE_TYPES__ 13 14 #include "evsel.h" 15 16 #include <errno.h> 17 #include <fcntl.h> 18 #include <inttypes.h> 19 #include <stdio.h> 20 #include <stdlib.h> 21 #include <string.h> 22 #include <unistd.h> 23 24 #include <dirent.h> 25 #include <linux/bitops.h> 26 #include <linux/bitmap.h> 27 #include <linux/compiler.h> 28 #include <linux/ctype.h> 29 #include <linux/err.h> 30 #include <linux/hw_breakpoint.h> 31 #include <linux/perf_event.h> 32 #include <linux/zalloc.h> 33 #include <sys/ioctl.h> 34 #include <sys/resource.h> 35 #include <sys/syscall.h> 36 #include <sys/types.h> 37 #include <sys/param.h> 38 39 #include <api/fs/fs.h> 40 #include <api/fs/tracing_path.h> 41 #include <byteswap.h> 42 #include <internal/lib.h> 43 #include <internal/threadmap.h> 44 #include <internal/xyarray.h> 45 #include <perf/cpumap.h> 46 #include <perf/evsel.h> 47 48 #include "../perf-sys.h" 49 #include "asm/bug.h" 50 #include "bpf-filter.h" 51 #include "bpf_counter.h" 52 #include "callchain.h" 53 #include "cgroup.h" 54 #include "counts.h" 55 #include "debug.h" 56 #include "drm_pmu.h" 57 #include "dwarf-regs.h" 58 #include "env.h" 59 #include "event.h" 60 #include "evlist.h" 61 #include "evsel_config.h" 62 #include "evsel_fprintf.h" 63 #include "hashmap.h" 64 #include "hist.h" 65 #include "hwmon_pmu.h" 66 #include "intel-tpebs.h" 67 #include "memswap.h" 68 #include "off_cpu.h" 69 #include "parse-branch-options.h" 70 #include "perf_regs.h" 71 #include "pmu.h" 72 #include "pmus.h" 73 #include "record.h" 74 #include "rlimit.h" 75 #include "session.h" 76 #include "stat.h" 77 #include "string2.h" 78 #include "target.h" 79 #include "thread_map.h" 80 #include "time-utils.h" 81 #include "tool_pmu.h" 82 #include "tp_pmu.h" 83 #include "trace-event.h" 84 #include "util.h" 85 86 #ifdef HAVE_LIBTRACEEVENT 87 #include <event-parse.h> 88 #endif 89 90 struct perf_missing_features perf_missing_features; 91 92 static clockid_t clockid; 93 94 static int evsel__no_extra_init(struct evsel *evsel __maybe_unused) 95 { 96 return 0; 97 } 98 99 static bool test_attr__enabled(void) 100 { 101 static bool test_attr__enabled; 102 static bool test_attr__enabled_tested; 103 104 if (!test_attr__enabled_tested) { 105 char *dir = getenv("PERF_TEST_ATTR"); 106 107 test_attr__enabled = (dir != NULL); 108 test_attr__enabled_tested = true; 109 } 110 return test_attr__enabled; 111 } 112 113 #define __WRITE_ASS(str, fmt, data) \ 114 do { \ 115 if (fprintf(file, #str "=%"fmt "\n", data) < 0) { \ 116 perror("test attr - failed to write event file"); \ 117 fclose(file); \ 118 return -1; \ 119 } \ 120 } while (0) 121 122 #define WRITE_ASS(field, fmt) __WRITE_ASS(field, fmt, attr->field) 123 124 static int store_event(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, 125 int fd, int group_fd, unsigned long flags) 126 { 127 FILE *file; 128 char path[PATH_MAX]; 129 char *dir = getenv("PERF_TEST_ATTR"); 130 131 snprintf(path, PATH_MAX, "%s/event-%d-%llu-%d", dir, 132 attr->type, attr->config, fd); 133 134 file = fopen(path, "w+"); 135 if (!file) { 136 perror("test attr - failed to open event file"); 137 return -1; 138 } 139 140 if (fprintf(file, "[event-%d-%llu-%d]\n", 141 attr->type, attr->config, fd) < 0) { 142 perror("test attr - failed to write event file"); 143 fclose(file); 144 return -1; 145 } 146 147 /* syscall arguments */ 148 __WRITE_ASS(fd, "d", fd); 149 __WRITE_ASS(group_fd, "d", group_fd); 150 __WRITE_ASS(cpu, "d", cpu.cpu); 151 __WRITE_ASS(pid, "d", pid); 152 __WRITE_ASS(flags, "lu", flags); 153 154 /* struct perf_event_attr */ 155 WRITE_ASS(type, PRIu32); 156 WRITE_ASS(size, PRIu32); 157 WRITE_ASS(config, "llu"); 158 WRITE_ASS(sample_period, "llu"); 159 WRITE_ASS(sample_type, "llu"); 160 WRITE_ASS(read_format, "llu"); 161 WRITE_ASS(disabled, "d"); 162 WRITE_ASS(inherit, "d"); 163 WRITE_ASS(pinned, "d"); 164 WRITE_ASS(exclusive, "d"); 165 WRITE_ASS(exclude_user, "d"); 166 WRITE_ASS(exclude_kernel, "d"); 167 WRITE_ASS(exclude_hv, "d"); 168 WRITE_ASS(exclude_idle, "d"); 169 WRITE_ASS(mmap, "d"); 170 WRITE_ASS(comm, "d"); 171 WRITE_ASS(freq, "d"); 172 WRITE_ASS(inherit_stat, "d"); 173 WRITE_ASS(enable_on_exec, "d"); 174 WRITE_ASS(task, "d"); 175 WRITE_ASS(watermark, "d"); 176 WRITE_ASS(precise_ip, "d"); 177 WRITE_ASS(mmap_data, "d"); 178 WRITE_ASS(sample_id_all, "d"); 179 WRITE_ASS(exclude_host, "d"); 180 WRITE_ASS(exclude_guest, "d"); 181 WRITE_ASS(exclude_callchain_kernel, "d"); 182 WRITE_ASS(exclude_callchain_user, "d"); 183 WRITE_ASS(mmap2, "d"); 184 WRITE_ASS(comm_exec, "d"); 185 WRITE_ASS(context_switch, "d"); 186 WRITE_ASS(write_backward, "d"); 187 WRITE_ASS(namespaces, "d"); 188 WRITE_ASS(use_clockid, "d"); 189 WRITE_ASS(wakeup_events, PRIu32); 190 WRITE_ASS(bp_type, PRIu32); 191 WRITE_ASS(config1, "llu"); 192 WRITE_ASS(config2, "llu"); 193 WRITE_ASS(branch_sample_type, "llu"); 194 WRITE_ASS(sample_regs_user, "llu"); 195 WRITE_ASS(sample_stack_user, PRIu32); 196 197 fclose(file); 198 return 0; 199 } 200 201 #undef __WRITE_ASS 202 #undef WRITE_ASS 203 204 static void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, 205 int fd, int group_fd, unsigned long flags) 206 { 207 int errno_saved = errno; 208 209 if ((fd != -1) && store_event(attr, pid, cpu, fd, group_fd, flags)) { 210 pr_err("test attr FAILED"); 211 exit(128); 212 } 213 214 errno = errno_saved; 215 } 216 217 static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused) 218 { 219 } 220 221 static struct { 222 size_t size; 223 int (*init)(struct evsel *evsel); 224 void (*fini)(struct evsel *evsel); 225 } perf_evsel__object = { 226 .size = sizeof(struct evsel), 227 .init = evsel__no_extra_init, 228 .fini = evsel__no_extra_fini, 229 }; 230 231 int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel), 232 void (*fini)(struct evsel *evsel)) 233 { 234 235 if (object_size == 0) 236 goto set_methods; 237 238 if (perf_evsel__object.size > object_size) 239 return -EINVAL; 240 241 perf_evsel__object.size = object_size; 242 243 set_methods: 244 if (init != NULL) 245 perf_evsel__object.init = init; 246 247 if (fini != NULL) 248 perf_evsel__object.fini = fini; 249 250 return 0; 251 } 252 253 const char *evsel__pmu_name(const struct evsel *evsel) 254 { 255 struct perf_pmu *pmu = evsel__find_pmu(evsel); 256 257 if (pmu) 258 return pmu->name; 259 260 return event_type(evsel->core.attr.type); 261 } 262 263 enum evsel_probe_type { 264 PROBE__UNKNOWN = 0, 265 PROBE__NOPE = 1, 266 PROBE__KPROBE = 2, 267 PROBE__UPROBE = 3, 268 /* 269 * Ftrace-based dynamic probes (kprobes/uprobes/fprobes created via 270 * tracefs) report PMU "tracepoint", not "kprobe"/"uprobe". Detect 271 * them by the __probe_ip field that the kernel adds to all dynamic 272 * probe formats. 273 */ 274 PROBE__FTRACE = 4, 275 }; 276 277 static void evsel__resolve_probe_type(struct evsel *evsel) 278 { 279 const char *name = evsel__pmu_name(evsel); 280 281 if (!strcmp(name, "kprobe")) 282 evsel->probe_type = PROBE__KPROBE; 283 else if (!strcmp(name, "uprobe")) 284 evsel->probe_type = PROBE__UPROBE; 285 else if (!strcmp(name, "tracepoint") && evsel__field(evsel, "__probe_ip")) 286 evsel->probe_type = PROBE__FTRACE; 287 else 288 evsel->probe_type = PROBE__NOPE; 289 } 290 291 bool evsel__is_probe(struct evsel *evsel) 292 { 293 if (evsel->probe_type == PROBE__UNKNOWN) 294 evsel__resolve_probe_type(evsel); 295 296 return evsel->probe_type > PROBE__NOPE; 297 } 298 299 bool evsel__is_kprobe(struct evsel *evsel) 300 { 301 if (evsel->probe_type == PROBE__UNKNOWN) 302 evsel__resolve_probe_type(evsel); 303 304 return evsel->probe_type == PROBE__KPROBE; 305 } 306 307 bool evsel__is_uprobe(struct evsel *evsel) 308 { 309 if (evsel->probe_type == PROBE__UNKNOWN) 310 evsel__resolve_probe_type(evsel); 311 312 return evsel->probe_type == PROBE__UPROBE; 313 } 314 315 #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) 316 317 int __evsel__sample_size(u64 sample_type) 318 { 319 u64 mask = sample_type & PERF_SAMPLE_MASK; 320 int size = 0; 321 int i; 322 323 for (i = 0; i < 64; i++) { 324 if (mask & (1ULL << i)) 325 size++; 326 } 327 328 size *= sizeof(u64); 329 330 return size; 331 } 332 333 /** 334 * __perf_evsel__calc_id_pos - calculate id_pos. 335 * @sample_type: sample type 336 * 337 * This function returns the position of the event id (PERF_SAMPLE_ID or 338 * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct 339 * perf_record_sample. 340 */ 341 static int __perf_evsel__calc_id_pos(u64 sample_type) 342 { 343 int idx = 0; 344 345 if (sample_type & PERF_SAMPLE_IDENTIFIER) 346 return 0; 347 348 if (!(sample_type & PERF_SAMPLE_ID)) 349 return -1; 350 351 if (sample_type & PERF_SAMPLE_IP) 352 idx += 1; 353 354 if (sample_type & PERF_SAMPLE_TID) 355 idx += 1; 356 357 if (sample_type & PERF_SAMPLE_TIME) 358 idx += 1; 359 360 if (sample_type & PERF_SAMPLE_ADDR) 361 idx += 1; 362 363 return idx; 364 } 365 366 /** 367 * __perf_evsel__calc_is_pos - calculate is_pos. 368 * @sample_type: sample type 369 * 370 * This function returns the position (counting backwards) of the event id 371 * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if 372 * sample_id_all is used there is an id sample appended to non-sample events. 373 */ 374 static int __perf_evsel__calc_is_pos(u64 sample_type) 375 { 376 int idx = 1; 377 378 if (sample_type & PERF_SAMPLE_IDENTIFIER) 379 return 1; 380 381 if (!(sample_type & PERF_SAMPLE_ID)) 382 return -1; 383 384 if (sample_type & PERF_SAMPLE_CPU) 385 idx += 1; 386 387 if (sample_type & PERF_SAMPLE_STREAM_ID) 388 idx += 1; 389 390 return idx; 391 } 392 393 void evsel__calc_id_pos(struct evsel *evsel) 394 { 395 evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type); 396 evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type); 397 } 398 399 void __evsel__set_sample_bit(struct evsel *evsel, 400 enum perf_event_sample_format bit) 401 { 402 if (!(evsel->core.attr.sample_type & bit)) { 403 evsel->core.attr.sample_type |= bit; 404 evsel->sample_size += sizeof(u64); 405 evsel__calc_id_pos(evsel); 406 } 407 } 408 409 void __evsel__reset_sample_bit(struct evsel *evsel, 410 enum perf_event_sample_format bit) 411 { 412 if (evsel->core.attr.sample_type & bit) { 413 evsel->core.attr.sample_type &= ~bit; 414 evsel->sample_size -= sizeof(u64); 415 evsel__calc_id_pos(evsel); 416 } 417 } 418 419 void evsel__set_sample_id(struct evsel *evsel, 420 bool can_sample_identifier) 421 { 422 if (can_sample_identifier) { 423 evsel__reset_sample_bit(evsel, ID); 424 evsel__set_sample_bit(evsel, IDENTIFIER); 425 } else { 426 evsel__set_sample_bit(evsel, ID); 427 } 428 evsel->core.attr.read_format |= PERF_FORMAT_ID; 429 } 430 431 bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel) 432 { 433 return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END; 434 } 435 436 /** 437 * evsel__is_function_event - Return whether given evsel is a function 438 * trace event 439 * 440 * @evsel - evsel selector to be tested 441 * 442 * Return %true if event is function trace event 443 */ 444 bool evsel__is_function_event(struct evsel *evsel) 445 { 446 #define FUNCTION_EVENT "ftrace:function" 447 448 return evsel->name && 449 !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT)); 450 451 #undef FUNCTION_EVENT 452 } 453 454 static void evsel__init(struct evsel *evsel, 455 struct perf_event_attr *attr, int idx) 456 { 457 perf_evsel__init(&evsel->core, attr, idx); 458 refcount_set(&evsel->refcnt, 1); 459 evsel->tracking = !idx; 460 evsel->unit = strdup(""); 461 evsel->scale = 1.0; 462 evsel->max_events = ULONG_MAX; 463 evsel->evlist = NULL; 464 INIT_LIST_HEAD(&evsel->config_terms); 465 INIT_LIST_HEAD(&evsel->bpf_counter_list); 466 INIT_LIST_HEAD(&evsel->bpf_filters); 467 perf_evsel__object.init(evsel); 468 evsel->sample_size = __evsel__sample_size(attr->sample_type); 469 evsel__calc_id_pos(evsel); 470 evsel->cmdline_group_boundary = false; 471 evsel->per_pkg_mask = NULL; 472 evsel->collect_stat = false; 473 evsel->group_pmu_name = NULL; 474 evsel->skippable = false; 475 evsel->supported = true; 476 evsel->alternate_hw_config = PERF_COUNT_HW_MAX; 477 evsel->script_output_type = -1; // FIXME: OUTPUT_TYPE_UNSET, see builtin-script.c 478 evsel->probe_type = PROBE__UNKNOWN; 479 } 480 481 struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx) 482 { 483 struct evsel *evsel = zalloc(perf_evsel__object.size); 484 485 if (!evsel) 486 return NULL; 487 evsel__init(evsel, attr, idx); 488 489 if (evsel__is_bpf_output(evsel) && !attr->sample_type) { 490 evsel->core.attr.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME | 491 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD), 492 evsel->core.attr.sample_period = 1; 493 } 494 495 if (evsel__is_clock(evsel)) { 496 free((char *)evsel->unit); 497 evsel->unit = strdup("msec"); 498 evsel->scale = 1e-6; 499 } 500 501 return evsel; 502 } 503 504 int copy_config_terms(struct list_head *dst, struct list_head *src) 505 { 506 struct evsel_config_term *pos, *tmp; 507 508 list_for_each_entry(pos, src, list) { 509 tmp = malloc(sizeof(*tmp)); 510 if (tmp == NULL) 511 return -ENOMEM; 512 513 *tmp = *pos; 514 if (tmp->free_str) { 515 tmp->val.str = strdup(pos->val.str); 516 if (tmp->val.str == NULL) { 517 free(tmp); 518 return -ENOMEM; 519 } 520 } 521 list_add_tail(&tmp->list, dst); 522 } 523 return 0; 524 } 525 526 static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src) 527 { 528 return copy_config_terms(&dst->config_terms, &src->config_terms); 529 } 530 531 /** 532 * evsel__clone - create a new evsel copied from @orig 533 * @orig: original evsel 534 * 535 * The assumption is that @orig is not configured nor opened yet. 536 * So we only care about the attributes that can be set while it's parsed. 537 */ 538 struct evsel *evsel__clone(struct evsel *orig) 539 { 540 struct evsel *evsel; 541 542 BUG_ON(orig->core.fd); 543 BUG_ON(orig->counts); 544 BUG_ON(orig->priv); 545 BUG_ON(orig->per_pkg_mask); 546 547 evsel = evsel__new(&orig->core.attr); 548 if (evsel == NULL) 549 return NULL; 550 551 evsel->core.cpus = perf_cpu_map__get(orig->core.cpus); 552 evsel->core.pmu_cpus = perf_cpu_map__get(orig->core.pmu_cpus); 553 evsel->core.threads = perf_thread_map__get(orig->core.threads); 554 evsel->core.nr_members = orig->core.nr_members; 555 evsel->core.system_wide = orig->core.system_wide; 556 evsel->core.requires_cpu = orig->core.requires_cpu; 557 evsel->core.is_pmu_core = orig->core.is_pmu_core; 558 evsel->core.is_pmu_uncore = orig->core.is_pmu_uncore; 559 560 if (orig->name) { 561 evsel->name = strdup(orig->name); 562 if (evsel->name == NULL) 563 goto out_err; 564 } 565 if (orig->group_name) { 566 evsel->group_name = strdup(orig->group_name); 567 if (evsel->group_name == NULL) 568 goto out_err; 569 } 570 if (orig->group_pmu_name) { 571 evsel->group_pmu_name = strdup(orig->group_pmu_name); 572 if (evsel->group_pmu_name == NULL) 573 goto out_err; 574 } 575 if (orig->filter) { 576 evsel->filter = strdup(orig->filter); 577 if (evsel->filter == NULL) 578 goto out_err; 579 } 580 if (orig->metric_id) { 581 evsel->metric_id = strdup(orig->metric_id); 582 if (evsel->metric_id == NULL) 583 goto out_err; 584 } 585 evsel->cgrp = cgroup__get(orig->cgrp); 586 #ifdef HAVE_LIBTRACEEVENT 587 if (orig->tp_sys) { 588 evsel->tp_sys = strdup(orig->tp_sys); 589 if (evsel->tp_sys == NULL) 590 goto out_err; 591 } 592 if (orig->tp_name) { 593 evsel->tp_name = strdup(orig->tp_name); 594 if (evsel->tp_name == NULL) 595 goto out_err; 596 } 597 evsel->tp_format = orig->tp_format; 598 #endif 599 evsel->handler = orig->handler; 600 evsel->core.leader = orig->core.leader; 601 evsel->metric_leader = orig->metric_leader; 602 603 evsel->max_events = orig->max_events; 604 zfree(&evsel->unit); 605 if (orig->unit) { 606 evsel->unit = strdup(orig->unit); 607 if (evsel->unit == NULL) 608 goto out_err; 609 } 610 evsel->scale = orig->scale; 611 evsel->snapshot = orig->snapshot; 612 evsel->per_pkg = orig->per_pkg; 613 evsel->percore = orig->percore; 614 evsel->precise_max = orig->precise_max; 615 evsel->is_libpfm_event = orig->is_libpfm_event; 616 617 evsel->exclude_GH = orig->exclude_GH; 618 evsel->sample_read = orig->sample_read; 619 evsel->collect_stat = orig->collect_stat; 620 evsel->weak_group = orig->weak_group; 621 evsel->bpf_counter = orig->bpf_counter; 622 evsel->use_config_name = orig->use_config_name; 623 evsel->skippable = orig->skippable; 624 evsel->dont_regroup = orig->dont_regroup; 625 evsel->default_metricgroup = orig->default_metricgroup; 626 evsel->default_show_events = orig->default_show_events; 627 628 evsel->pmu = orig->pmu; 629 evsel->first_wildcard_match = orig->first_wildcard_match; 630 631 if (evsel__copy_config_terms(evsel, orig) < 0) 632 goto out_err; 633 634 evsel->alternate_hw_config = orig->alternate_hw_config; 635 636 evsel->retire_lat = orig->retire_lat; 637 if (evsel->retire_lat) 638 evsel->retirement_latency = orig->retirement_latency; 639 640 return evsel; 641 642 out_err: 643 evsel__put(evsel); 644 return NULL; 645 } 646 647 /* 648 * Returns pointer with encoded error via <linux/err.h> interface. 649 */ 650 struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool format) 651 { 652 struct perf_event_attr attr = { 653 .type = PERF_TYPE_TRACEPOINT, 654 .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME | 655 PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD), 656 }; 657 struct evsel *evsel = zalloc(perf_evsel__object.size); 658 int err = -ENOMEM, id = -1; 659 660 if (evsel == NULL) 661 goto out_err; 662 663 664 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0) 665 goto out_free; 666 667 #ifdef HAVE_LIBTRACEEVENT 668 evsel->tp_sys = strdup(sys); 669 if (!evsel->tp_sys) 670 goto out_free; 671 672 evsel->tp_name = strdup(name); 673 if (!evsel->tp_name) 674 goto out_free; 675 #endif 676 677 event_attr_init(&attr); 678 679 if (format) { 680 id = tp_pmu__id(sys, name); 681 if (id < 0) { 682 err = id; 683 goto out_free; 684 } 685 } 686 attr.config = (__u64)id; 687 attr.sample_period = 1; 688 evsel__init(evsel, &attr, idx); 689 return evsel; 690 691 out_free: 692 zfree(&evsel->name); 693 #ifdef HAVE_LIBTRACEEVENT 694 zfree(&evsel->tp_sys); 695 zfree(&evsel->tp_name); 696 #endif 697 free(evsel); 698 out_err: 699 return ERR_PTR(err); 700 } 701 702 struct evsel *evsel__get(struct evsel *evsel) 703 { 704 refcount_inc(&evsel->refcnt); 705 return evsel; 706 } 707 708 #ifdef HAVE_LIBTRACEEVENT 709 struct tep_event *evsel__tp_format(struct evsel *evsel) 710 { 711 struct tep_event *tp_format = evsel->tp_format; 712 713 if (tp_format) 714 return tp_format; 715 716 if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) 717 return NULL; 718 719 if (!evsel->tp_sys) 720 tp_format = trace_event__tp_format_id(evsel->core.attr.config); 721 else 722 tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name); 723 724 if (!tp_format) { 725 pr_err("Error getting tracepoint format '%s': %m\n", 726 evsel__name(evsel)); 727 return NULL; 728 } 729 evsel->tp_format = tp_format; 730 return evsel->tp_format; 731 } 732 #endif 733 734 const char *const evsel__hw_names[PERF_COUNT_HW_MAX] = { 735 "cycles", 736 "instructions", 737 "cache-references", 738 "cache-misses", 739 "branches", 740 "branch-misses", 741 "bus-cycles", 742 "stalled-cycles-frontend", 743 "stalled-cycles-backend", 744 "ref-cycles", 745 }; 746 747 char *evsel__bpf_counter_events; 748 749 bool evsel__match_bpf_counter_events(const char *name) 750 { 751 int name_len; 752 bool match; 753 char *ptr; 754 755 if (!evsel__bpf_counter_events) 756 return false; 757 758 ptr = strstr(evsel__bpf_counter_events, name); 759 name_len = strlen(name); 760 761 /* check name matches a full token in evsel__bpf_counter_events */ 762 match = (ptr != NULL) && 763 ((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) && 764 ((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0')); 765 766 return match; 767 } 768 769 static const char *__evsel__hw_name(u64 config) 770 { 771 if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config]) 772 return evsel__hw_names[config]; 773 774 return "unknown-hardware"; 775 } 776 777 static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size) 778 { 779 int colon = 0, r = 0; 780 struct perf_event_attr *attr = &evsel->core.attr; 781 782 #define MOD_PRINT(context, mod) do { \ 783 if (!attr->exclude_##context) { \ 784 if (!colon) colon = ++r; \ 785 r += scnprintf(bf + r, size - r, "%c", mod); \ 786 } } while(0) 787 788 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) { 789 MOD_PRINT(kernel, 'k'); 790 MOD_PRINT(user, 'u'); 791 MOD_PRINT(hv, 'h'); 792 } 793 794 if (attr->precise_ip) { 795 if (!colon) 796 colon = ++r; 797 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp"); 798 } 799 800 if (attr->exclude_host || attr->exclude_guest) { 801 MOD_PRINT(host, 'H'); 802 MOD_PRINT(guest, 'G'); 803 } 804 #undef MOD_PRINT 805 if (colon) 806 bf[colon - 1] = ':'; 807 return r; 808 } 809 810 int __weak arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size) 811 { 812 return scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config)); 813 } 814 815 static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size) 816 { 817 int r = arch_evsel__hw_name(evsel, bf, size); 818 return r + evsel__add_modifiers(evsel, bf + r, size - r); 819 } 820 821 const char *const evsel__sw_names[PERF_COUNT_SW_MAX] = { 822 "cpu-clock", 823 "task-clock", 824 "page-faults", 825 "context-switches", 826 "cpu-migrations", 827 "minor-faults", 828 "major-faults", 829 "alignment-faults", 830 "emulation-faults", 831 "dummy", 832 }; 833 834 static const char *__evsel__sw_name(u64 config) 835 { 836 if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config]) 837 return evsel__sw_names[config]; 838 return "unknown-software"; 839 } 840 841 static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size) 842 { 843 int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config)); 844 return r + evsel__add_modifiers(evsel, bf + r, size - r); 845 } 846 847 static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type) 848 { 849 int r; 850 851 r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr); 852 853 if (type & HW_BREAKPOINT_R) 854 r += scnprintf(bf + r, size - r, "r"); 855 856 if (type & HW_BREAKPOINT_W) 857 r += scnprintf(bf + r, size - r, "w"); 858 859 if (type & HW_BREAKPOINT_X) 860 r += scnprintf(bf + r, size - r, "x"); 861 862 return r; 863 } 864 865 static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size) 866 { 867 struct perf_event_attr *attr = &evsel->core.attr; 868 int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type); 869 return r + evsel__add_modifiers(evsel, bf + r, size - r); 870 } 871 872 const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = { 873 { "L1-dcache", "l1-d", "l1d", "L1-data", }, 874 { "L1-icache", "l1-i", "l1i", "L1-instruction", }, 875 { "LLC", "L2", }, 876 { "dTLB", "d-tlb", "Data-TLB", }, 877 { "iTLB", "i-tlb", "Instruction-TLB", }, 878 { "branch", "branches", "bpu", "btb", "bpc", }, 879 { "node", }, 880 }; 881 882 const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = { 883 { "load", "loads", "read", }, 884 { "store", "stores", "write", }, 885 { "prefetch", "prefetches", "speculative-read", "speculative-load", }, 886 }; 887 888 const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = { 889 { "refs", "Reference", "ops", "access", }, 890 { "misses", "miss", }, 891 }; 892 893 #define C(x) PERF_COUNT_HW_CACHE_##x 894 #define CACHE_READ (1 << C(OP_READ)) 895 #define CACHE_WRITE (1 << C(OP_WRITE)) 896 #define CACHE_PREFETCH (1 << C(OP_PREFETCH)) 897 #define COP(x) (1 << x) 898 899 /* 900 * cache operation stat 901 * L1I : Read and prefetch only 902 * ITLB and BPU : Read-only 903 */ 904 static const unsigned long evsel__hw_cache_stat[C(MAX)] = { 905 [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), 906 [C(L1I)] = (CACHE_READ | CACHE_PREFETCH), 907 [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), 908 [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), 909 [C(ITLB)] = (CACHE_READ), 910 [C(BPU)] = (CACHE_READ), 911 [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), 912 }; 913 914 bool evsel__is_cache_op_valid(u8 type, u8 op) 915 { 916 if (evsel__hw_cache_stat[type] & COP(op)) 917 return true; /* valid */ 918 else 919 return false; /* invalid */ 920 } 921 922 int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size) 923 { 924 if (result) { 925 return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0], 926 evsel__hw_cache_op[op][0], 927 evsel__hw_cache_result[result][0]); 928 } 929 930 return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0], 931 evsel__hw_cache_op[op][1]); 932 } 933 934 static int __evsel__hw_cache_name(u64 config, char *bf, size_t size) 935 { 936 u8 op, result, type = (config >> 0) & 0xff; 937 const char *err = "unknown-ext-hardware-cache-type"; 938 939 if (type >= PERF_COUNT_HW_CACHE_MAX) 940 goto out_err; 941 942 op = (config >> 8) & 0xff; 943 err = "unknown-ext-hardware-cache-op"; 944 if (op >= PERF_COUNT_HW_CACHE_OP_MAX) 945 goto out_err; 946 947 result = (config >> 16) & 0xff; 948 err = "unknown-ext-hardware-cache-result"; 949 if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 950 goto out_err; 951 952 err = "invalid-cache"; 953 if (!evsel__is_cache_op_valid(type, op)) 954 goto out_err; 955 956 return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size); 957 out_err: 958 return scnprintf(bf, size, "%s", err); 959 } 960 961 static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size) 962 { 963 int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size); 964 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret); 965 } 966 967 static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size) 968 { 969 int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config); 970 return ret + evsel__add_modifiers(evsel, bf + ret, size - ret); 971 } 972 973 const char *evsel__name(struct evsel *evsel) 974 { 975 char bf[128]; 976 977 if (!evsel) 978 goto out_unknown; 979 980 if (evsel->name) 981 return evsel->name; 982 983 switch (evsel->core.attr.type) { 984 case PERF_TYPE_RAW: 985 evsel__raw_name(evsel, bf, sizeof(bf)); 986 break; 987 988 case PERF_TYPE_HARDWARE: 989 evsel__hw_name(evsel, bf, sizeof(bf)); 990 break; 991 992 case PERF_TYPE_HW_CACHE: 993 evsel__hw_cache_name(evsel, bf, sizeof(bf)); 994 break; 995 996 case PERF_TYPE_SOFTWARE: 997 evsel__sw_name(evsel, bf, sizeof(bf)); 998 break; 999 1000 case PERF_TYPE_TRACEPOINT: 1001 scnprintf(bf, sizeof(bf), "unknown tracepoint id=%#"PRIx64, 1002 evsel->core.attr.config); 1003 break; 1004 1005 case PERF_TYPE_BREAKPOINT: 1006 evsel__bp_name(evsel, bf, sizeof(bf)); 1007 break; 1008 1009 case PERF_PMU_TYPE_TOOL: 1010 scnprintf(bf, sizeof(bf), "%s", evsel__tool_pmu_event_name(evsel)); 1011 break; 1012 1013 default: 1014 scnprintf(bf, sizeof(bf), "unknown event PMU=%d config=%#"PRIx64, 1015 evsel->core.attr.type, evsel->core.attr.config); 1016 break; 1017 } 1018 1019 evsel->name = strdup(bf); 1020 1021 if (evsel->name) 1022 return evsel->name; 1023 out_unknown: 1024 return "unknown"; 1025 } 1026 1027 bool evsel__name_is(struct evsel *evsel, const char *name) 1028 { 1029 return !strcmp(evsel__name(evsel), name); 1030 } 1031 1032 const char *evsel__metric_id(const struct evsel *evsel) 1033 { 1034 if (evsel->metric_id) 1035 return evsel->metric_id; 1036 1037 if (evsel__is_tool(evsel)) 1038 return evsel__tool_pmu_event_name(evsel); 1039 1040 return "unknown"; 1041 } 1042 1043 const char *evsel__group_name(struct evsel *evsel) 1044 { 1045 return evsel->group_name ?: "anon group"; 1046 } 1047 1048 /* 1049 * Returns the group details for the specified leader, 1050 * with following rules. 1051 * 1052 * For record -e '{cycles,instructions}' 1053 * 'anon group { cycles:u, instructions:u }' 1054 * 1055 * For record -e 'cycles,instructions' and report --group 1056 * 'cycles:u, instructions:u' 1057 */ 1058 int evsel__group_desc(struct evsel *evsel, char *buf, size_t size) 1059 { 1060 int ret = 0; 1061 bool first = true; 1062 struct evsel *pos; 1063 const char *group_name = evsel__group_name(evsel); 1064 1065 if (!evsel->forced_leader) 1066 ret = scnprintf(buf, size, "%s { ", group_name); 1067 1068 for_each_group_evsel(pos, evsel) { 1069 if (symbol_conf.skip_empty && 1070 evsel__hists(pos)->stats.nr_samples == 0) 1071 continue; 1072 1073 ret += scnprintf(buf + ret, size - ret, "%s%s", 1074 first ? "" : ", ", evsel__name(pos)); 1075 first = false; 1076 } 1077 1078 if (!evsel->forced_leader) 1079 ret += scnprintf(buf + ret, size - ret, " }"); 1080 1081 return ret; 1082 } 1083 1084 uint16_t evsel__e_machine(struct evsel *evsel, uint32_t *e_flags) 1085 { 1086 struct perf_session *session = evsel__session(evsel); 1087 1088 return perf_session__e_machine(session, e_flags); 1089 } 1090 1091 static void __evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, 1092 const struct callchain_param *param) 1093 { 1094 bool function = evsel__is_function_event(evsel); 1095 struct perf_event_attr *attr = &evsel->core.attr; 1096 1097 if (EM_HOST == EM_S390 && evsel->pmu && !perf_pmu__is_software(evsel->pmu)) { 1098 pr_warning_once("Cannot use hardware PMU 'cpum_cf' / 'cpum_sf' event with callchain. " 1099 "Use a software 'cpu-clock' / 'task-clock' event.\n"); 1100 } 1101 1102 evsel__set_sample_bit(evsel, CALLCHAIN); 1103 1104 attr->sample_max_stack = param->max_stack; 1105 1106 if (opts->kernel_callchains) 1107 attr->exclude_callchain_user = 1; 1108 if (opts->user_callchains) 1109 attr->exclude_callchain_kernel = 1; 1110 if (param->record_mode == CALLCHAIN_LBR) { 1111 if (!opts->branch_stack) { 1112 if (attr->exclude_user) { 1113 pr_warning("LBR callstack option is only available " 1114 "to get user callchain information. " 1115 "Falling back to framepointers.\n"); 1116 } else { 1117 evsel__set_sample_bit(evsel, BRANCH_STACK); 1118 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER | 1119 PERF_SAMPLE_BRANCH_CALL_STACK | 1120 PERF_SAMPLE_BRANCH_NO_CYCLES | 1121 PERF_SAMPLE_BRANCH_NO_FLAGS | 1122 PERF_SAMPLE_BRANCH_HW_INDEX; 1123 } 1124 } else 1125 pr_warning("Cannot use LBR callstack with branch stack. " 1126 "Falling back to framepointers.\n"); 1127 } 1128 1129 if (param->record_mode == CALLCHAIN_DWARF) { 1130 int abi = -1; /* -1 indicates only basic GPRs are needed. */ 1131 1132 if (!function) { 1133 uint16_t e_machine = evsel__e_machine(evsel, /*e_flags=*/NULL); 1134 1135 evsel__set_sample_bit(evsel, REGS_USER); 1136 evsel__set_sample_bit(evsel, STACK_USER); 1137 if (opts->sample_user_regs && 1138 DWARF_MINIMAL_REGS(e_machine) != perf_user_reg_mask(EM_HOST, &abi)) { 1139 attr->sample_regs_user |= DWARF_MINIMAL_REGS(e_machine); 1140 pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, " 1141 "specifying a subset with --user-regs may render DWARF unwinding unreliable, " 1142 "so the minimal registers set (IP, SP) is explicitly forced.\n"); 1143 } else { 1144 abi = -1; 1145 attr->sample_regs_user |= perf_user_reg_mask(EM_HOST, &abi); 1146 } 1147 attr->sample_stack_user = param->dump_size; 1148 attr->exclude_callchain_user = 1; 1149 } else { 1150 pr_info("Cannot use DWARF unwind for function trace event," 1151 " falling back to framepointers.\n"); 1152 } 1153 } 1154 1155 if (function) { 1156 pr_info("Disabling user space callchains for function trace event.\n"); 1157 attr->exclude_callchain_user = 1; 1158 } 1159 1160 if (param->defer && !attr->exclude_callchain_user) 1161 attr->defer_callchain = 1; 1162 } 1163 1164 void evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, 1165 const struct callchain_param *param) 1166 { 1167 if (param->enabled) 1168 return __evsel__config_callchain(evsel, opts, param); 1169 } 1170 1171 static void evsel__reset_callgraph(struct evsel *evsel, const struct callchain_param *param) 1172 { 1173 struct perf_event_attr *attr = &evsel->core.attr; 1174 1175 evsel__reset_sample_bit(evsel, CALLCHAIN); 1176 if (param->record_mode == CALLCHAIN_LBR) { 1177 evsel__reset_sample_bit(evsel, BRANCH_STACK); 1178 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER | 1179 PERF_SAMPLE_BRANCH_CALL_STACK | 1180 PERF_SAMPLE_BRANCH_HW_INDEX); 1181 } 1182 if (param->record_mode == CALLCHAIN_DWARF) { 1183 evsel__reset_sample_bit(evsel, REGS_USER); 1184 evsel__reset_sample_bit(evsel, STACK_USER); 1185 } 1186 } 1187 1188 static void evsel__apply_ratio_to_prev(struct evsel *evsel, 1189 struct perf_event_attr *attr, 1190 const struct record_opts *opts, 1191 const char *buf) 1192 { 1193 struct perf_event_attr *prev_attr = NULL; 1194 struct evsel *evsel_prev = NULL; 1195 u64 type = evsel->core.attr.sample_type; 1196 u64 prev_type = 0; 1197 double rtp; 1198 1199 rtp = strtod(buf, NULL); 1200 if (rtp <= 0) { 1201 pr_err("Invalid ratio-to-prev value %lf\n", rtp); 1202 return; 1203 } 1204 if (evsel == evsel__leader(evsel)) { 1205 pr_err("Invalid use of ratio-to-prev term without preceding element in group\n"); 1206 return; 1207 } 1208 if (!evsel->pmu->is_core) { 1209 pr_err("Event using ratio-to-prev term must have a core PMU\n"); 1210 return; 1211 } 1212 1213 evsel_prev = evsel__prev(evsel); 1214 if (!evsel_prev) { 1215 pr_err("Previous event does not exist.\n"); 1216 return; 1217 } 1218 1219 if (evsel_prev->pmu->type != evsel->pmu->type) { 1220 pr_err("Compared events (\"%s\", \"%s\") must have same PMU\n", 1221 evsel->name, evsel_prev->name); 1222 return; 1223 } 1224 1225 prev_attr = &evsel_prev->core.attr; 1226 prev_type = evsel_prev->core.attr.sample_type; 1227 1228 if (!(prev_type & PERF_SAMPLE_PERIOD)) { 1229 attr->sample_period = prev_attr->sample_period * rtp; 1230 attr->freq = 0; 1231 evsel__reset_sample_bit(evsel, PERIOD); 1232 } else if (!(type & PERF_SAMPLE_PERIOD)) { 1233 prev_attr->sample_period = attr->sample_period / rtp; 1234 prev_attr->freq = 0; 1235 evsel__reset_sample_bit(evsel_prev, PERIOD); 1236 } else { 1237 if (opts->user_interval != ULLONG_MAX) { 1238 prev_attr->sample_period = opts->user_interval; 1239 attr->sample_period = prev_attr->sample_period * rtp; 1240 prev_attr->freq = 0; 1241 attr->freq = 0; 1242 evsel__reset_sample_bit(evsel_prev, PERIOD); 1243 evsel__reset_sample_bit(evsel, PERIOD); 1244 } else { 1245 pr_err("Event period term or count (-c) must be set when using ratio-to-prev term.\n"); 1246 return; 1247 } 1248 } 1249 1250 arch_evsel__apply_ratio_to_prev(evsel, attr); 1251 } 1252 1253 static void evsel__apply_config_terms(struct evsel *evsel, 1254 const struct record_opts *opts, bool track) 1255 { 1256 struct evsel_config_term *term; 1257 struct list_head *config_terms = &evsel->config_terms; 1258 struct perf_event_attr *attr = &evsel->core.attr; 1259 /* callgraph default */ 1260 struct callchain_param param = { 1261 .record_mode = callchain_param.record_mode, 1262 }; 1263 u32 dump_size = 0; 1264 int max_stack = 0; 1265 const char *callgraph_buf = NULL; 1266 const char *rtp_buf = NULL; 1267 1268 list_for_each_entry(term, config_terms, list) { 1269 switch (term->type) { 1270 case EVSEL__CONFIG_TERM_PERIOD: 1271 if (!(term->weak && opts->user_interval != ULLONG_MAX)) { 1272 attr->sample_period = term->val.period; 1273 attr->freq = 0; 1274 evsel__reset_sample_bit(evsel, PERIOD); 1275 } 1276 break; 1277 case EVSEL__CONFIG_TERM_FREQ: 1278 if (!(term->weak && opts->user_freq != UINT_MAX)) { 1279 attr->sample_freq = term->val.freq; 1280 attr->freq = 1; 1281 evsel__set_sample_bit(evsel, PERIOD); 1282 } 1283 break; 1284 case EVSEL__CONFIG_TERM_TIME: 1285 if (term->val.time) 1286 evsel__set_sample_bit(evsel, TIME); 1287 else 1288 evsel__reset_sample_bit(evsel, TIME); 1289 break; 1290 case EVSEL__CONFIG_TERM_CALLGRAPH: 1291 callgraph_buf = term->val.str; 1292 break; 1293 case EVSEL__CONFIG_TERM_BRANCH: 1294 if (term->val.str && strcmp(term->val.str, "no")) { 1295 evsel__set_sample_bit(evsel, BRANCH_STACK); 1296 parse_branch_str(term->val.str, 1297 &attr->branch_sample_type); 1298 } else 1299 evsel__reset_sample_bit(evsel, BRANCH_STACK); 1300 break; 1301 case EVSEL__CONFIG_TERM_STACK_USER: 1302 dump_size = term->val.stack_user; 1303 break; 1304 case EVSEL__CONFIG_TERM_MAX_STACK: 1305 max_stack = term->val.max_stack; 1306 break; 1307 case EVSEL__CONFIG_TERM_MAX_EVENTS: 1308 evsel->max_events = term->val.max_events; 1309 break; 1310 case EVSEL__CONFIG_TERM_INHERIT: 1311 /* 1312 * attr->inherit should has already been set by 1313 * evsel__config. If user explicitly set 1314 * inherit using config terms, override global 1315 * opt->no_inherit setting. 1316 */ 1317 attr->inherit = term->val.inherit ? 1 : 0; 1318 break; 1319 case EVSEL__CONFIG_TERM_OVERWRITE: 1320 attr->write_backward = term->val.overwrite ? 1 : 0; 1321 break; 1322 case EVSEL__CONFIG_TERM_DRV_CFG: 1323 break; 1324 case EVSEL__CONFIG_TERM_PERCORE: 1325 break; 1326 case EVSEL__CONFIG_TERM_AUX_OUTPUT: 1327 attr->aux_output = term->val.aux_output ? 1 : 0; 1328 break; 1329 case EVSEL__CONFIG_TERM_AUX_ACTION: 1330 /* Already applied by auxtrace */ 1331 break; 1332 case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE: 1333 /* Already applied by auxtrace */ 1334 break; 1335 case EVSEL__CONFIG_TERM_USR_CHG_CONFIG: 1336 case EVSEL__CONFIG_TERM_USR_CHG_CONFIG1: 1337 case EVSEL__CONFIG_TERM_USR_CHG_CONFIG2: 1338 case EVSEL__CONFIG_TERM_USR_CHG_CONFIG3: 1339 case EVSEL__CONFIG_TERM_USR_CHG_CONFIG4: 1340 break; 1341 case EVSEL__CONFIG_TERM_RATIO_TO_PREV: 1342 rtp_buf = term->val.str; 1343 break; 1344 default: 1345 break; 1346 } 1347 } 1348 1349 /* User explicitly set per-event callgraph, clear the old setting and reset. */ 1350 if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) { 1351 bool sample_address = false; 1352 1353 if (max_stack) { 1354 param.max_stack = max_stack; 1355 if (callgraph_buf == NULL) 1356 callgraph_buf = "fp"; 1357 } 1358 1359 /* parse callgraph parameters */ 1360 if (callgraph_buf != NULL) { 1361 if (!strcmp(callgraph_buf, "no")) { 1362 param.enabled = false; 1363 param.record_mode = CALLCHAIN_NONE; 1364 } else { 1365 param.enabled = true; 1366 if (parse_callchain_record(callgraph_buf, ¶m)) { 1367 pr_err("per-event callgraph setting for %s failed. " 1368 "Apply callgraph global setting for it\n", 1369 evsel->name); 1370 return; 1371 } 1372 if (param.record_mode == CALLCHAIN_DWARF) 1373 sample_address = true; 1374 } 1375 } 1376 if (dump_size > 0) { 1377 dump_size = round_up(dump_size, sizeof(u64)); 1378 param.dump_size = dump_size; 1379 } 1380 1381 /* If global callgraph set, clear it */ 1382 if (callchain_param.enabled) 1383 evsel__reset_callgraph(evsel, &callchain_param); 1384 1385 /* set perf-event callgraph */ 1386 if (param.enabled) { 1387 if (sample_address) { 1388 evsel__set_sample_bit(evsel, ADDR); 1389 evsel__set_sample_bit(evsel, DATA_SRC); 1390 evsel->core.attr.mmap_data = track; 1391 } 1392 evsel__config_callchain(evsel, opts, ¶m); 1393 } 1394 } 1395 if (rtp_buf) 1396 evsel__apply_ratio_to_prev(evsel, attr, opts, rtp_buf); 1397 } 1398 1399 struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type) 1400 { 1401 struct evsel_config_term *term, *found_term = NULL; 1402 1403 list_for_each_entry(term, &evsel->config_terms, list) { 1404 if (term->type == type) 1405 found_term = term; 1406 } 1407 1408 return found_term; 1409 } 1410 1411 /* 1412 * Set @config_name to @val as long as the user hasn't already set or cleared it 1413 * by passing a config term on the command line. 1414 * 1415 * @val is the value to put into the bits specified by @config_name rather than 1416 * the bit pattern. It is shifted into position by this function, so to set 1417 * something to true, pass 1 for val rather than a pre shifted value. 1418 */ 1419 void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, 1420 u64 val) 1421 { 1422 u64 user_bits = 0; 1423 struct evsel_config_term *term = evsel__get_config_term(evsel, 1424 USR_CHG_CONFIG); 1425 struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, 1426 config_name); 1427 int fbit; 1428 __u64 *vp; 1429 1430 if (!format) 1431 return; 1432 1433 switch (format->value) { 1434 case PERF_PMU_FORMAT_VALUE_CONFIG: 1435 term = evsel__get_config_term(evsel, USR_CHG_CONFIG); 1436 vp = &evsel->core.attr.config; 1437 break; 1438 case PERF_PMU_FORMAT_VALUE_CONFIG1: 1439 term = evsel__get_config_term(evsel, USR_CHG_CONFIG1); 1440 vp = &evsel->core.attr.config1; 1441 break; 1442 case PERF_PMU_FORMAT_VALUE_CONFIG2: 1443 term = evsel__get_config_term(evsel, USR_CHG_CONFIG2); 1444 vp = &evsel->core.attr.config2; 1445 break; 1446 case PERF_PMU_FORMAT_VALUE_CONFIG3: 1447 term = evsel__get_config_term(evsel, USR_CHG_CONFIG3); 1448 vp = &evsel->core.attr.config3; 1449 break; 1450 case PERF_PMU_FORMAT_VALUE_CONFIG4: 1451 term = evsel__get_config_term(evsel, USR_CHG_CONFIG4); 1452 vp = &evsel->core.attr.config4; 1453 break; 1454 default: 1455 pr_err("Unknown format value: %d\n", format->value); 1456 return; 1457 } 1458 1459 if (!format) 1460 return; 1461 1462 if (term) 1463 user_bits = term->val.cfg_chg; 1464 1465 /* Do nothing if the user changed the value */ 1466 for_each_set_bit(fbit, format->bits, PERF_PMU_FORMAT_BITS) 1467 if ((1ULL << fbit) & user_bits) 1468 return; 1469 1470 /* Otherwise replace it */ 1471 perf_pmu__format_pack(format->bits, val, vp, /*zero=*/true); 1472 } 1473 1474 bool evsel__config_exists(const struct evsel *evsel, const char *config_name) 1475 { 1476 struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); 1477 1478 return format && !bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS); 1479 } 1480 1481 int evsel__get_config_val(const struct evsel *evsel, const char *config_name, 1482 u64 *val) 1483 { 1484 struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); 1485 1486 if (!format || bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS)) { 1487 pr_err("Unknown/empty format name: %s\n", config_name); 1488 *val = 0; 1489 return -EINVAL; 1490 } 1491 1492 switch (format->value) { 1493 case PERF_PMU_FORMAT_VALUE_CONFIG: 1494 *val = perf_pmu__format_unpack(format->bits, 1495 evsel->core.attr.config); 1496 return 0; 1497 case PERF_PMU_FORMAT_VALUE_CONFIG1: 1498 *val = perf_pmu__format_unpack(format->bits, 1499 evsel->core.attr.config1); 1500 return 0; 1501 case PERF_PMU_FORMAT_VALUE_CONFIG2: 1502 *val = perf_pmu__format_unpack(format->bits, 1503 evsel->core.attr.config2); 1504 return 0; 1505 case PERF_PMU_FORMAT_VALUE_CONFIG3: 1506 *val = perf_pmu__format_unpack(format->bits, 1507 evsel->core.attr.config3); 1508 return 0; 1509 case PERF_PMU_FORMAT_VALUE_CONFIG4: 1510 *val = perf_pmu__format_unpack(format->bits, 1511 evsel->core.attr.config4); 1512 return 0; 1513 default: 1514 pr_err("Unknown format value: %d\n", format->value); 1515 *val = 0; 1516 return -EINVAL; 1517 } 1518 } 1519 1520 void __weak arch_evsel__set_sample_weight(struct evsel *evsel) 1521 { 1522 evsel__set_sample_bit(evsel, WEIGHT); 1523 } 1524 1525 void __weak arch__post_evsel_config(struct evsel *evsel __maybe_unused, 1526 struct perf_event_attr *attr __maybe_unused) 1527 { 1528 } 1529 1530 void __weak arch_evsel__apply_ratio_to_prev(struct evsel *evsel __maybe_unused, 1531 struct perf_event_attr *attr __maybe_unused) 1532 { 1533 } 1534 1535 static void evsel__set_default_freq_period(const struct record_opts *opts, 1536 struct perf_event_attr *attr) 1537 { 1538 if (opts->freq) { 1539 attr->freq = 1; 1540 attr->sample_freq = opts->freq; 1541 } else { 1542 attr->sample_period = opts->default_interval; 1543 } 1544 } 1545 1546 bool evsel__is_offcpu_event(struct evsel *evsel) 1547 { 1548 return evsel__is_bpf_output(evsel) && evsel__name_is(evsel, OFFCPU_EVENT) && 1549 evsel->core.attr.sample_type & PERF_SAMPLE_RAW; 1550 } 1551 1552 /* 1553 * The enable_on_exec/disabled value strategy: 1554 * 1555 * 1) For any type of traced program: 1556 * - all independent events and group leaders are disabled 1557 * - all group members are enabled 1558 * 1559 * Group members are ruled by group leaders. They need to 1560 * be enabled, because the group scheduling relies on that. 1561 * 1562 * 2) For traced programs executed by perf: 1563 * - all independent events and group leaders have 1564 * enable_on_exec set 1565 * - we don't specifically enable or disable any event during 1566 * the record command 1567 * 1568 * Independent events and group leaders are initially disabled 1569 * and get enabled by exec. Group members are ruled by group 1570 * leaders as stated in 1). 1571 * 1572 * 3) For traced programs attached by perf (pid/tid): 1573 * - we specifically enable or disable all events during 1574 * the record command 1575 * 1576 * When attaching events to already running traced we 1577 * enable/disable events specifically, as there's no 1578 * initial traced exec call. 1579 */ 1580 void evsel__config(struct evsel *evsel, const struct record_opts *opts, 1581 const struct callchain_param *callchain) 1582 { 1583 struct evsel *leader = evsel__leader(evsel); 1584 struct perf_event_attr *attr = &evsel->core.attr; 1585 int track = evsel->tracking; 1586 bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread; 1587 1588 attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1; 1589 attr->inherit = target__has_cpu(&opts->target) ? 0 : !opts->no_inherit; 1590 attr->write_backward = opts->overwrite ? 1 : 0; 1591 attr->read_format = PERF_FORMAT_LOST; 1592 1593 evsel__set_sample_bit(evsel, IP); 1594 evsel__set_sample_bit(evsel, TID); 1595 1596 if (evsel->sample_read) { 1597 evsel__set_sample_bit(evsel, READ); 1598 1599 /* 1600 * We need ID even in case of single event, because 1601 * PERF_SAMPLE_READ process ID specific data. 1602 */ 1603 evsel__set_sample_id(evsel, false); 1604 1605 /* 1606 * Apply group format only if we belong to group 1607 * with more than one members. 1608 */ 1609 if (leader->core.nr_members > 1) { 1610 attr->read_format |= PERF_FORMAT_GROUP; 1611 } 1612 1613 /* 1614 * Inherit + SAMPLE_READ requires SAMPLE_TID in the read_format 1615 */ 1616 if (attr->inherit) { 1617 evsel__set_sample_bit(evsel, TID); 1618 evsel->core.attr.read_format |= 1619 PERF_FORMAT_ID; 1620 } 1621 } 1622 1623 /* 1624 * We default some events to have a default interval. But keep 1625 * it a weak assumption overridable by the user. 1626 */ 1627 if ((evsel->is_libpfm_event && !attr->sample_period) || 1628 (!evsel->is_libpfm_event && (!attr->sample_period || 1629 opts->user_freq != UINT_MAX || 1630 opts->user_interval != ULLONG_MAX))) 1631 evsel__set_default_freq_period(opts, attr); 1632 1633 /* 1634 * If attr->freq was set (here or earlier), ask for period 1635 * to be sampled. 1636 */ 1637 if (attr->freq) 1638 evsel__set_sample_bit(evsel, PERIOD); 1639 1640 if (opts->no_samples) 1641 attr->sample_freq = 0; 1642 1643 if (opts->inherit_stat) { 1644 evsel->core.attr.read_format |= 1645 PERF_FORMAT_TOTAL_TIME_ENABLED | 1646 PERF_FORMAT_TOTAL_TIME_RUNNING | 1647 PERF_FORMAT_ID; 1648 attr->inherit_stat = 1; 1649 } 1650 1651 if (opts->sample_address) 1652 evsel__set_sample_bit(evsel, ADDR); 1653 1654 if (opts->record_data_mmap) 1655 attr->mmap_data = track; 1656 1657 /* 1658 * We don't allow user space callchains for function trace 1659 * event, due to issues with page faults while tracing page 1660 * fault handler and its overall trickiness nature. 1661 */ 1662 if (evsel__is_function_event(evsel)) 1663 evsel->core.attr.exclude_callchain_user = 1; 1664 1665 if (callchain && callchain->enabled && !evsel->no_aux_samples) 1666 evsel__config_callchain(evsel, opts, callchain); 1667 1668 if (opts->sample_intr_regs && !evsel->no_aux_samples && 1669 !evsel__is_dummy_event(evsel)) { 1670 attr->sample_regs_intr = opts->sample_intr_regs; 1671 attr->sample_simd_regs_enabled = 1672 evsel__is_non_software_event(evsel) ? 1673 !!opts->sample_simd_regs_enabled : 0; 1674 evsel__set_sample_bit(evsel, REGS_INTR); 1675 } 1676 1677 if ((opts->sample_intr_vec_regs || opts->sample_intr_pred_regs) && 1678 !evsel->no_aux_samples && !evsel__is_dummy_event(evsel)) { 1679 attr->sample_simd_regs_enabled = !!opts->sample_simd_regs_enabled; 1680 attr->sample_simd_vec_reg_intr = opts->sample_intr_vec_regs; 1681 attr->sample_simd_vec_reg_qwords = opts->sample_vec_reg_qwords; 1682 attr->sample_simd_pred_reg_intr = opts->sample_intr_pred_regs; 1683 attr->sample_simd_pred_reg_qwords = opts->sample_pred_reg_qwords; 1684 evsel__set_sample_bit(evsel, REGS_INTR); 1685 } 1686 1687 if (opts->sample_user_regs && !evsel->no_aux_samples && 1688 !evsel__is_dummy_event(evsel)) { 1689 attr->sample_regs_user |= opts->sample_user_regs; 1690 attr->sample_simd_regs_enabled = 1691 evsel__is_non_software_event(evsel) ? 1692 !!opts->sample_simd_regs_enabled : 0; 1693 evsel__set_sample_bit(evsel, REGS_USER); 1694 } 1695 1696 if ((opts->sample_user_vec_regs || opts->sample_user_pred_regs) && 1697 !evsel->no_aux_samples && !evsel__is_dummy_event(evsel)) { 1698 attr->sample_simd_regs_enabled = !!opts->sample_simd_regs_enabled; 1699 attr->sample_simd_vec_reg_user = opts->sample_user_vec_regs; 1700 attr->sample_simd_vec_reg_qwords = opts->sample_vec_reg_qwords; 1701 attr->sample_simd_pred_reg_user = opts->sample_user_pred_regs; 1702 attr->sample_simd_pred_reg_qwords = opts->sample_pred_reg_qwords; 1703 evsel__set_sample_bit(evsel, REGS_USER); 1704 } 1705 1706 if (target__has_cpu(&opts->target) || opts->sample_cpu) 1707 evsel__set_sample_bit(evsel, CPU); 1708 1709 /* 1710 * When the user explicitly disabled time don't force it here. 1711 */ 1712 if (opts->sample_time && 1713 (!perf_missing_features.sample_id_all && 1714 (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu || 1715 opts->sample_time_set))) 1716 evsel__set_sample_bit(evsel, TIME); 1717 1718 if (opts->raw_samples && !evsel->no_aux_samples) { 1719 evsel__set_sample_bit(evsel, TIME); 1720 evsel__set_sample_bit(evsel, RAW); 1721 evsel__set_sample_bit(evsel, CPU); 1722 } 1723 1724 if (opts->sample_data_src) 1725 evsel__set_sample_bit(evsel, DATA_SRC); 1726 1727 if (opts->sample_phys_addr) 1728 evsel__set_sample_bit(evsel, PHYS_ADDR); 1729 1730 if (opts->no_buffering) { 1731 attr->watermark = 0; 1732 attr->wakeup_events = 1; 1733 } 1734 if (opts->branch_stack && !evsel->no_aux_samples) { 1735 evsel__set_sample_bit(evsel, BRANCH_STACK); 1736 attr->branch_sample_type = opts->branch_stack; 1737 } 1738 1739 if (opts->sample_weight || evsel->retire_lat) { 1740 arch_evsel__set_sample_weight(evsel); 1741 evsel->retire_lat = false; 1742 } 1743 attr->task = track; 1744 attr->mmap = track; 1745 attr->mmap2 = track && !perf_missing_features.mmap2; 1746 attr->comm = track; 1747 attr->build_id = track && opts->build_id; 1748 attr->defer_output = track && callchain && callchain->defer; 1749 1750 /* 1751 * ksymbol is tracked separately with text poke because it needs to be 1752 * system wide and enabled immediately. 1753 */ 1754 if (!opts->text_poke) 1755 attr->ksymbol = track && !perf_missing_features.ksymbol; 1756 attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf; 1757 1758 if (opts->record_namespaces) 1759 attr->namespaces = track; 1760 1761 if (opts->record_cgroup) { 1762 attr->cgroup = track && !perf_missing_features.cgroup; 1763 evsel__set_sample_bit(evsel, CGROUP); 1764 } 1765 1766 if (opts->sample_data_page_size) 1767 evsel__set_sample_bit(evsel, DATA_PAGE_SIZE); 1768 1769 if (opts->sample_code_page_size) 1770 evsel__set_sample_bit(evsel, CODE_PAGE_SIZE); 1771 1772 if (opts->record_switch_events) 1773 attr->context_switch = track; 1774 1775 if (opts->sample_transaction) 1776 evsel__set_sample_bit(evsel, TRANSACTION); 1777 1778 if (opts->running_time) { 1779 evsel->core.attr.read_format |= 1780 PERF_FORMAT_TOTAL_TIME_ENABLED | 1781 PERF_FORMAT_TOTAL_TIME_RUNNING; 1782 } 1783 1784 /* 1785 * XXX see the function comment above 1786 * 1787 * Disabling only independent events or group leaders, 1788 * keeping group members enabled. 1789 */ 1790 if (evsel__is_group_leader(evsel)) 1791 attr->disabled = 1; 1792 1793 /* 1794 * Setting enable_on_exec for independent events and 1795 * group leaders for traced executed by perf. 1796 */ 1797 if (target__none(&opts->target) && evsel__is_group_leader(evsel) && 1798 !opts->target.initial_delay) 1799 attr->enable_on_exec = 1; 1800 1801 if (evsel->immediate) { 1802 attr->disabled = 0; 1803 attr->enable_on_exec = 0; 1804 } 1805 1806 clockid = opts->clockid; 1807 if (opts->use_clockid) { 1808 attr->use_clockid = 1; 1809 attr->clockid = opts->clockid; 1810 } 1811 1812 if (evsel->precise_max) 1813 attr->precise_ip = 3; 1814 1815 if (opts->all_user) { 1816 attr->exclude_kernel = 1; 1817 attr->exclude_user = 0; 1818 } 1819 1820 if (opts->all_kernel) { 1821 attr->exclude_kernel = 0; 1822 attr->exclude_user = 1; 1823 } 1824 1825 if (evsel->core.pmu_cpus || evsel->unit) 1826 evsel->core.attr.read_format |= PERF_FORMAT_ID; 1827 1828 /* 1829 * Apply event specific term settings, 1830 * it overloads any global configuration. 1831 */ 1832 evsel__apply_config_terms(evsel, opts, track); 1833 1834 evsel->ignore_missing_thread = opts->ignore_missing_thread; 1835 1836 /* The --period option takes the precedence. */ 1837 if (opts->period_set) { 1838 if (opts->period) 1839 evsel__set_sample_bit(evsel, PERIOD); 1840 else 1841 evsel__reset_sample_bit(evsel, PERIOD); 1842 } 1843 1844 /* 1845 * A dummy event never triggers any actual counter and therefore 1846 * cannot be used with branch_stack. 1847 * 1848 * For initial_delay, a dummy event is added implicitly. 1849 * The software event will trigger -EOPNOTSUPP error out, 1850 * if BRANCH_STACK bit is set. 1851 */ 1852 if (evsel__is_dummy_event(evsel)) 1853 evsel__reset_sample_bit(evsel, BRANCH_STACK); 1854 1855 if (evsel__is_offcpu_event(evsel)) { 1856 evsel->core.attr.sample_type &= OFFCPU_SAMPLE_TYPES; 1857 attr->inherit = 0; 1858 } 1859 1860 arch__post_evsel_config(evsel, attr); 1861 } 1862 1863 int evsel__set_filter(struct evsel *evsel, const char *filter) 1864 { 1865 char *new_filter = strdup(filter); 1866 1867 if (new_filter != NULL) { 1868 free(evsel->filter); 1869 evsel->filter = new_filter; 1870 return 0; 1871 } 1872 1873 return -ENOMEM; 1874 } 1875 1876 static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter) 1877 { 1878 char *new_filter; 1879 1880 if (evsel->filter == NULL) 1881 return evsel__set_filter(evsel, filter); 1882 1883 if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) { 1884 free(evsel->filter); 1885 evsel->filter = new_filter; 1886 return 0; 1887 } 1888 1889 return -ENOMEM; 1890 } 1891 1892 int evsel__append_tp_filter(struct evsel *evsel, const char *filter) 1893 { 1894 return evsel__append_filter(evsel, "(%s) && (%s)", filter); 1895 } 1896 1897 int evsel__append_addr_filter(struct evsel *evsel, const char *filter) 1898 { 1899 return evsel__append_filter(evsel, "%s,%s", filter); 1900 } 1901 1902 /* Caller has to clear disabled after going through all CPUs. */ 1903 int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx) 1904 { 1905 int err; 1906 1907 if (evsel__is_tool(evsel)) 1908 err = evsel__tool_pmu_enable_cpu(evsel, cpu_map_idx); 1909 else 1910 err = perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); 1911 1912 if (!err && evsel__is_group_leader(evsel)) { 1913 struct evsel *member; 1914 1915 for_each_group_member(member, evsel) { 1916 if (evsel__is_non_perf_event_open_pmu(evsel) || 1917 evsel__is_non_perf_event_open_pmu(member)) { 1918 /* 1919 * In a mixed PMU group, userspace PMUs are not 1920 * grouped in the kernel (opened with group_fd = -1) 1921 * and are skipped by the kernel when enabling the 1922 * group leader. We must manually enable them in 1923 * userspace. 1924 */ 1925 int mem_err = evsel__enable_cpu(member, cpu_map_idx); 1926 1927 if (mem_err) 1928 return mem_err; 1929 } 1930 } 1931 } 1932 return err; 1933 } 1934 1935 int evsel__enable(struct evsel *evsel) 1936 { 1937 int err; 1938 1939 if (evsel__is_tool(evsel)) 1940 err = evsel__tool_pmu_enable(evsel); 1941 else 1942 err = perf_evsel__enable(&evsel->core); 1943 1944 if (!err) 1945 evsel->disabled = false; 1946 1947 if (!err && evsel__is_group_leader(evsel)) { 1948 struct evsel *member; 1949 1950 for_each_group_member(member, evsel) { 1951 if (evsel__is_non_perf_event_open_pmu(evsel) || 1952 evsel__is_non_perf_event_open_pmu(member)) { 1953 /* 1954 * In a mixed PMU group, userspace PMUs are not 1955 * grouped in the kernel (opened with group_fd = -1) 1956 * and are skipped by the kernel when enabling the 1957 * group leader. We must manually enable them in 1958 * userspace. 1959 */ 1960 int mem_err = evsel__enable(member); 1961 1962 if (mem_err) 1963 return mem_err; 1964 } 1965 member->disabled = false; 1966 } 1967 } 1968 1969 return err; 1970 } 1971 1972 /* Caller has to set disabled after going through all CPUs. */ 1973 int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx) 1974 { 1975 int err; 1976 1977 if (evsel__is_tool(evsel)) 1978 err = evsel__tool_pmu_disable_cpu(evsel, cpu_map_idx); 1979 else 1980 err = perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); 1981 1982 if (!err && evsel__is_group_leader(evsel)) { 1983 struct evsel *member; 1984 1985 for_each_group_member(member, evsel) { 1986 if (evsel__is_non_perf_event_open_pmu(evsel) || 1987 evsel__is_non_perf_event_open_pmu(member)) { 1988 /* 1989 * In a mixed PMU group, userspace PMUs are not 1990 * grouped in the kernel and are skipped by the 1991 * kernel when disabling the group leader. We must 1992 * manually disable them in userspace. 1993 */ 1994 int mem_err = evsel__disable_cpu(member, cpu_map_idx); 1995 1996 if (mem_err) 1997 return mem_err; 1998 } 1999 } 2000 } 2001 return err; 2002 } 2003 2004 int evsel__disable(struct evsel *evsel) 2005 { 2006 int err; 2007 2008 if (evsel__is_tool(evsel)) 2009 err = evsel__tool_pmu_disable(evsel); 2010 else 2011 err = perf_evsel__disable(&evsel->core); 2012 2013 /* 2014 * We mark it disabled here so that tools that disable a event can 2015 * ignore events after they disable it. I.e. the ring buffer may have 2016 * already a few more events queued up before the kernel got the stop 2017 * request. 2018 */ 2019 if (!err) 2020 evsel->disabled = true; 2021 2022 if (!err && evsel__is_group_leader(evsel)) { 2023 struct evsel *member; 2024 2025 for_each_group_member(member, evsel) { 2026 if (evsel__is_non_perf_event_open_pmu(evsel) || 2027 evsel__is_non_perf_event_open_pmu(member)) { 2028 /* 2029 * In a mixed PMU group, userspace PMUs are not 2030 * grouped in the kernel and are skipped by the 2031 * kernel when disabling the group leader. We must 2032 * manually disable them in userspace. 2033 */ 2034 int mem_err = evsel__disable(member); 2035 2036 if (mem_err) 2037 return mem_err; 2038 } 2039 member->disabled = true; 2040 } 2041 } 2042 2043 return err; 2044 } 2045 2046 void free_config_terms(struct list_head *config_terms) 2047 { 2048 struct evsel_config_term *term, *h; 2049 2050 list_for_each_entry_safe(term, h, config_terms, list) { 2051 list_del_init(&term->list); 2052 if (term->free_str) 2053 zfree(&term->val.str); 2054 free(term); 2055 } 2056 } 2057 2058 static void evsel__free_config_terms(struct evsel *evsel) 2059 { 2060 free_config_terms(&evsel->config_terms); 2061 } 2062 2063 static void (*evsel__priv_destructor)(void *priv); 2064 2065 void evsel__set_priv_destructor(void (*destructor)(void *priv)) 2066 { 2067 assert(evsel__priv_destructor == NULL); 2068 2069 evsel__priv_destructor = destructor; 2070 } 2071 2072 static void evsel__exit(struct evsel *evsel) 2073 { 2074 assert(list_empty(&evsel->core.node)); 2075 assert(evsel->evlist == NULL); 2076 if (evsel__is_retire_lat(evsel)) 2077 evsel__tpebs_close(evsel); 2078 bpf_counter__destroy(evsel); 2079 perf_bpf_filter__destroy(evsel); 2080 evsel__free_counts(evsel); 2081 perf_evsel__free_fd(&evsel->core); 2082 perf_evsel__free_id(&evsel->core); 2083 evsel__free_config_terms(evsel); 2084 cgroup__put(evsel->cgrp); 2085 perf_evsel__exit(&evsel->core); 2086 zfree(&evsel->group_name); 2087 zfree(&evsel->name); 2088 #ifdef HAVE_LIBTRACEEVENT 2089 zfree(&evsel->tp_sys); 2090 zfree(&evsel->tp_name); 2091 #endif 2092 zfree(&evsel->filter); 2093 zfree(&evsel->group_pmu_name); 2094 zfree(&evsel->unit); 2095 zfree(&evsel->metric_id); 2096 evsel__zero_per_pkg(evsel); 2097 hashmap__free(evsel->per_pkg_mask); 2098 evsel->per_pkg_mask = NULL; 2099 if (evsel__priv_destructor) 2100 evsel__priv_destructor(evsel->priv); 2101 perf_evsel__object.fini(evsel); 2102 if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || 2103 evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) { 2104 xyarray__delete(evsel->process_time.start_times); 2105 xyarray__delete(evsel->process_time.accumulated_times); 2106 } 2107 } 2108 2109 void evsel__put(struct evsel *evsel) 2110 { 2111 if (!evsel) 2112 return; 2113 2114 if (!refcount_dec_and_test(&evsel->refcnt)) 2115 return; 2116 2117 evsel__exit(evsel); 2118 free(evsel); 2119 } 2120 2121 void evsel__compute_deltas(struct evsel *evsel, int cpu_map_idx, int thread, 2122 struct perf_counts_values *count) 2123 { 2124 struct perf_counts_values tmp; 2125 2126 if (!evsel->prev_raw_counts) 2127 return; 2128 2129 tmp = *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); 2130 *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread) = *count; 2131 2132 count->val = count->val - tmp.val; 2133 count->ena = count->ena - tmp.ena; 2134 count->run = count->run - tmp.run; 2135 } 2136 2137 static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread) 2138 { 2139 struct perf_counts_values *count = perf_counts(evsel->counts, cpu_map_idx, thread); 2140 2141 return perf_evsel__read(&evsel->core, cpu_map_idx, thread, count); 2142 } 2143 2144 static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread, 2145 u64 val, u64 ena, u64 run, u64 lost) 2146 { 2147 struct perf_counts_values *count; 2148 2149 count = perf_counts(counter->counts, cpu_map_idx, thread); 2150 2151 if (evsel__is_retire_lat(counter)) { 2152 evsel__tpebs_read(counter, cpu_map_idx, thread); 2153 perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true); 2154 return; 2155 } 2156 2157 count->val = val; 2158 count->ena = ena; 2159 count->run = run; 2160 count->lost = lost; 2161 2162 perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true); 2163 } 2164 2165 static bool evsel__group_has_tpebs(struct evsel *leader) 2166 { 2167 struct evsel *evsel; 2168 2169 for_each_group_evsel(evsel, leader) { 2170 if (evsel__is_retire_lat(evsel)) 2171 return true; 2172 } 2173 return false; 2174 } 2175 2176 static u64 evsel__group_read_nr_members(struct evsel *leader) 2177 { 2178 u64 nr = leader->core.nr_members; 2179 struct evsel *evsel; 2180 2181 for_each_group_evsel(evsel, leader) { 2182 if (evsel__is_retire_lat(evsel)) 2183 nr--; 2184 } 2185 return nr; 2186 } 2187 2188 static u64 evsel__group_read_size(struct evsel *leader) 2189 { 2190 u64 read_format = leader->core.attr.read_format; 2191 int entry = sizeof(u64); /* value */ 2192 int size = 0; 2193 int nr = 1; 2194 2195 if (!evsel__group_has_tpebs(leader)) 2196 return perf_evsel__read_size(&leader->core); 2197 2198 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 2199 size += sizeof(u64); 2200 2201 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 2202 size += sizeof(u64); 2203 2204 if (read_format & PERF_FORMAT_ID) 2205 entry += sizeof(u64); 2206 2207 if (read_format & PERF_FORMAT_LOST) 2208 entry += sizeof(u64); 2209 2210 if (read_format & PERF_FORMAT_GROUP) { 2211 nr = evsel__group_read_nr_members(leader); 2212 size += sizeof(u64); 2213 } 2214 2215 size += entry * nr; 2216 return size; 2217 } 2218 2219 static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int thread, u64 *data) 2220 { 2221 u64 read_format = leader->core.attr.read_format; 2222 struct sample_read_value *v; 2223 u64 nr, ena = 0, run = 0, lost = 0; 2224 2225 nr = *data++; 2226 2227 if (nr != evsel__group_read_nr_members(leader)) 2228 return -EINVAL; 2229 2230 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 2231 ena = *data++; 2232 2233 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 2234 run = *data++; 2235 2236 v = (void *)data; 2237 sample_read_group__for_each(v, nr, read_format) { 2238 struct evsel *counter; 2239 2240 counter = evlist__id2evsel(leader->evlist, v->id); 2241 if (!counter) 2242 return -EINVAL; 2243 2244 if (read_format & PERF_FORMAT_LOST) 2245 lost = v->lost; 2246 2247 evsel__set_count(counter, cpu_map_idx, thread, v->value, ena, run, lost); 2248 } 2249 2250 return 0; 2251 } 2252 2253 static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread) 2254 { 2255 struct perf_stat_evsel *ps = leader->stats; 2256 u64 read_format = leader->core.attr.read_format; 2257 int size = evsel__group_read_size(leader); 2258 u64 *data = ps->group_data; 2259 2260 if (!(read_format & PERF_FORMAT_ID)) 2261 return -EINVAL; 2262 2263 if (!evsel__is_group_leader(leader)) 2264 return -EINVAL; 2265 2266 if (!data) { 2267 data = zalloc(size); 2268 if (!data) 2269 return -ENOMEM; 2270 2271 ps->group_data = data; 2272 } 2273 2274 if (FD(leader, cpu_map_idx, thread) < 0) 2275 return -EINVAL; 2276 2277 if (readn(FD(leader, cpu_map_idx, thread), data, size) <= 0) 2278 return -errno; 2279 2280 return evsel__process_group_data(leader, cpu_map_idx, thread, data); 2281 } 2282 2283 bool __evsel__match(const struct evsel *evsel, u32 type, u64 config) 2284 { 2285 2286 u32 e_type = evsel->core.attr.type; 2287 u64 e_config = evsel->core.attr.config; 2288 2289 if (e_type == type && e_config == config) 2290 return true; 2291 if (type != PERF_TYPE_HARDWARE && type != PERF_TYPE_HW_CACHE) 2292 return false; 2293 if ((e_type == PERF_TYPE_HARDWARE || e_type == PERF_TYPE_HW_CACHE) && 2294 perf_pmus__supports_extended_type()) 2295 e_config &= PERF_HW_EVENT_MASK; 2296 if (e_type == type && e_config == config) 2297 return true; 2298 if (type == PERF_TYPE_HARDWARE && evsel->pmu && evsel->pmu->is_core && 2299 evsel->alternate_hw_config == config) 2300 return true; 2301 return false; 2302 } 2303 2304 int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread) 2305 { 2306 if (evsel__is_tool(evsel)) 2307 return evsel__tool_pmu_read(evsel, cpu_map_idx, thread); 2308 2309 if (evsel__is_hwmon(evsel)) 2310 return evsel__hwmon_pmu_read(evsel, cpu_map_idx, thread); 2311 2312 if (evsel__is_drm(evsel)) 2313 return evsel__drm_pmu_read(evsel, cpu_map_idx, thread); 2314 2315 if (evsel__is_retire_lat(evsel)) 2316 return evsel__tpebs_read(evsel, cpu_map_idx, thread); 2317 2318 if (evsel->core.attr.read_format & PERF_FORMAT_GROUP) 2319 return evsel__read_group(evsel, cpu_map_idx, thread); 2320 2321 return evsel__read_one(evsel, cpu_map_idx, thread); 2322 } 2323 2324 int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale) 2325 { 2326 struct perf_counts_values count; 2327 size_t nv = scale ? 3 : 1; 2328 2329 if (FD(evsel, cpu_map_idx, thread) < 0) 2330 return -EINVAL; 2331 2332 if (evsel->counts == NULL && evsel__alloc_counts(evsel) < 0) 2333 return -ENOMEM; 2334 2335 if (readn(FD(evsel, cpu_map_idx, thread), &count, nv * sizeof(u64)) <= 0) 2336 return -errno; 2337 2338 evsel__compute_deltas(evsel, cpu_map_idx, thread, &count); 2339 perf_counts_values__scale(&count, scale, NULL); 2340 *perf_counts(evsel->counts, cpu_map_idx, thread) = count; 2341 return 0; 2342 } 2343 2344 static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other, 2345 int cpu_map_idx) 2346 { 2347 struct perf_cpu cpu; 2348 2349 cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx); 2350 return perf_cpu_map__idx(other->core.cpus, cpu); 2351 } 2352 2353 static int evsel__hybrid_group_cpu_map_idx(struct evsel *evsel, int cpu_map_idx) 2354 { 2355 struct evsel *leader = evsel__leader(evsel); 2356 2357 if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) || 2358 (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) { 2359 return evsel__match_other_cpu(evsel, leader, cpu_map_idx); 2360 } 2361 2362 return cpu_map_idx; 2363 } 2364 2365 static int get_group_fd(struct evsel *evsel, int cpu_map_idx, int thread) 2366 { 2367 struct evsel *leader = evsel__leader(evsel); 2368 int fd; 2369 2370 if (!evsel->supported || evsel__is_group_leader(evsel)) 2371 return -1; 2372 2373 /* 2374 * Leader must be already processed/open, 2375 * if not it's a bug. 2376 */ 2377 BUG_ON(!leader->core.fd); 2378 2379 cpu_map_idx = evsel__hybrid_group_cpu_map_idx(evsel, cpu_map_idx); 2380 if (cpu_map_idx == -1) 2381 return -1; 2382 2383 fd = FD(leader, cpu_map_idx, thread); 2384 BUG_ON(fd == -1 && leader->supported); 2385 2386 /* 2387 * When the leader has been skipped, return -2 to distinguish from no 2388 * group leader case. 2389 */ 2390 return fd == -1 ? -2 : fd; 2391 } 2392 2393 static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx) 2394 { 2395 for (int cpu = 0; cpu < nr_cpus; cpu++) 2396 for (int thread = thread_idx; thread < nr_threads - 1; thread++) 2397 FD(pos, cpu, thread) = FD(pos, cpu, thread + 1); 2398 } 2399 2400 static int update_fds(struct evsel *evsel, 2401 int nr_cpus, int cpu_map_idx, 2402 int nr_threads, int thread_idx) 2403 { 2404 struct evsel *pos; 2405 2406 if (cpu_map_idx >= nr_cpus || thread_idx >= nr_threads) 2407 return -EINVAL; 2408 2409 evlist__for_each_entry(evsel->evlist, pos) { 2410 nr_cpus = pos != evsel ? nr_cpus : cpu_map_idx; 2411 2412 evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx); 2413 2414 /* 2415 * Since fds for next evsel has not been created, 2416 * there is no need to iterate whole event list. 2417 */ 2418 if (pos == evsel) 2419 break; 2420 } 2421 return 0; 2422 } 2423 2424 static bool evsel__ignore_missing_thread(struct evsel *evsel, 2425 int nr_cpus, int cpu_map_idx, 2426 struct perf_thread_map *threads, 2427 int thread, int err) 2428 { 2429 pid_t ignore_pid = perf_thread_map__pid(threads, thread); 2430 2431 if (!evsel->ignore_missing_thread) 2432 return false; 2433 2434 /* The system wide setup does not work with threads. */ 2435 if (evsel->core.system_wide) 2436 return false; 2437 2438 /* The -ESRCH is perf event syscall errno for pid's not found. */ 2439 if (err != -ESRCH) 2440 return false; 2441 2442 /* If there's only one thread, let it fail. */ 2443 if (threads->nr == 1) 2444 return false; 2445 2446 /* 2447 * We should remove fd for missing_thread first 2448 * because thread_map__remove() will decrease threads->nr. 2449 */ 2450 if (update_fds(evsel, nr_cpus, cpu_map_idx, threads->nr, thread)) 2451 return false; 2452 2453 if (thread_map__remove(threads, thread)) 2454 return false; 2455 2456 pr_warning("WARNING: Ignored open failure for pid %d\n", 2457 ignore_pid); 2458 return true; 2459 } 2460 2461 static int __open_attr__fprintf(FILE *fp, const char *name, const char *val, 2462 void *priv __maybe_unused) 2463 { 2464 return fprintf(fp, " %-32s %s\n", name, val); 2465 } 2466 2467 static void display_attr(struct perf_event_attr *attr) 2468 { 2469 if (verbose >= 2 || debug_peo_args) { 2470 fprintf(stderr, "%.60s\n", graph_dotted_line); 2471 fprintf(stderr, "perf_event_attr:\n"); 2472 perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL); 2473 fprintf(stderr, "%.60s\n", graph_dotted_line); 2474 } 2475 } 2476 2477 bool evsel__precise_ip_fallback(struct evsel *evsel) 2478 { 2479 /* Do not try less precise if not requested. */ 2480 if (!evsel->precise_max) 2481 return false; 2482 2483 /* 2484 * We tried all the precise_ip values, and it's 2485 * still failing, so leave it to standard fallback. 2486 */ 2487 if (!evsel->core.attr.precise_ip) { 2488 evsel->core.attr.precise_ip = evsel->precise_ip_original; 2489 return false; 2490 } 2491 2492 if (!evsel->precise_ip_original) 2493 evsel->precise_ip_original = evsel->core.attr.precise_ip; 2494 2495 evsel->core.attr.precise_ip--; 2496 pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip); 2497 display_attr(&evsel->core.attr); 2498 return true; 2499 } 2500 2501 static struct perf_cpu_map *empty_cpu_map; 2502 static struct perf_thread_map *empty_thread_map; 2503 2504 static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, 2505 struct perf_thread_map *threads) 2506 { 2507 int ret = 0; 2508 int nthreads = perf_thread_map__nr(threads); 2509 2510 if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) || 2511 (perf_missing_features.aux_output && evsel->core.attr.aux_output)) 2512 return -EINVAL; 2513 2514 if (cpus == NULL) { 2515 if (empty_cpu_map == NULL) { 2516 empty_cpu_map = perf_cpu_map__new_any_cpu(); 2517 if (empty_cpu_map == NULL) 2518 return -ENOMEM; 2519 } 2520 2521 cpus = empty_cpu_map; 2522 } 2523 2524 if (threads == NULL) { 2525 if (empty_thread_map == NULL) { 2526 empty_thread_map = thread_map__new_by_tid(-1); 2527 if (empty_thread_map == NULL) 2528 return -ENOMEM; 2529 } 2530 2531 threads = empty_thread_map; 2532 } 2533 2534 if (evsel->core.fd == NULL && 2535 perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0) 2536 return -ENOMEM; 2537 2538 if (evsel__is_tool(evsel)) 2539 ret = evsel__tool_pmu_prepare_open(evsel, cpus, nthreads); 2540 2541 evsel->open_flags = PERF_FLAG_FD_CLOEXEC; 2542 if (evsel->cgrp) 2543 evsel->open_flags |= PERF_FLAG_PID_CGROUP; 2544 2545 return ret; 2546 } 2547 2548 static void evsel__disable_missing_features(struct evsel *evsel) 2549 { 2550 if (perf_missing_features.defer_callchain && evsel->core.attr.defer_callchain) 2551 evsel->core.attr.defer_callchain = 0; 2552 if (perf_missing_features.defer_callchain && evsel->core.attr.defer_output) 2553 evsel->core.attr.defer_output = 0; 2554 if (perf_missing_features.inherit_sample_read && evsel->core.attr.inherit && 2555 (evsel->core.attr.sample_type & PERF_SAMPLE_READ)) 2556 evsel->core.attr.inherit = 0; 2557 if (perf_missing_features.branch_counters) 2558 evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_COUNTERS; 2559 if (perf_missing_features.read_lost) 2560 evsel->core.attr.read_format &= ~PERF_FORMAT_LOST; 2561 if (perf_missing_features.weight_struct) { 2562 evsel__set_sample_bit(evsel, WEIGHT); 2563 evsel__reset_sample_bit(evsel, WEIGHT_STRUCT); 2564 } 2565 if (perf_missing_features.clockid_wrong) 2566 evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */ 2567 if (perf_missing_features.clockid) { 2568 evsel->core.attr.use_clockid = 0; 2569 evsel->core.attr.clockid = 0; 2570 } 2571 if (perf_missing_features.cloexec) 2572 evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC; 2573 if (perf_missing_features.mmap2) 2574 evsel->core.attr.mmap2 = 0; 2575 if (evsel->pmu && evsel->pmu->missing_features.exclude_guest) 2576 evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0; 2577 if (perf_missing_features.lbr_flags) 2578 evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS | 2579 PERF_SAMPLE_BRANCH_NO_CYCLES); 2580 if (perf_missing_features.group_read && evsel->core.attr.inherit) 2581 evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID); 2582 if (perf_missing_features.ksymbol) 2583 evsel->core.attr.ksymbol = 0; 2584 if (perf_missing_features.bpf) 2585 evsel->core.attr.bpf_event = 0; 2586 if (perf_missing_features.branch_hw_idx) 2587 evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX; 2588 if (perf_missing_features.sample_id_all) 2589 evsel->core.attr.sample_id_all = 0; 2590 } 2591 2592 int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, 2593 struct perf_thread_map *threads) 2594 { 2595 int err; 2596 2597 err = __evsel__prepare_open(evsel, cpus, threads); 2598 if (err) 2599 return err; 2600 2601 evsel__disable_missing_features(evsel); 2602 2603 return err; 2604 } 2605 2606 static bool __has_attr_feature(struct perf_event_attr *attr, 2607 struct perf_cpu cpu, unsigned long flags) 2608 { 2609 int fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, 2610 /*group_fd=*/-1, flags); 2611 close(fd); 2612 2613 if (fd < 0) { 2614 attr->exclude_kernel = 1; 2615 2616 fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, 2617 /*group_fd=*/-1, flags); 2618 close(fd); 2619 } 2620 2621 if (fd < 0) { 2622 attr->exclude_hv = 1; 2623 2624 fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, 2625 /*group_fd=*/-1, flags); 2626 close(fd); 2627 } 2628 2629 if (fd < 0) { 2630 attr->exclude_guest = 1; 2631 2632 fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, 2633 /*group_fd=*/-1, flags); 2634 close(fd); 2635 } 2636 2637 attr->exclude_kernel = 0; 2638 attr->exclude_guest = 0; 2639 attr->exclude_hv = 0; 2640 2641 return fd >= 0; 2642 } 2643 2644 static bool has_attr_feature(struct perf_event_attr *attr, unsigned long flags) 2645 { 2646 struct perf_cpu cpu = {.cpu = -1}; 2647 2648 return __has_attr_feature(attr, cpu, flags); 2649 } 2650 2651 static void evsel__detect_missing_pmu_features(struct evsel *evsel) 2652 { 2653 struct perf_event_attr attr = { 2654 .type = evsel->core.attr.type, 2655 .config = evsel->core.attr.config, 2656 .disabled = 1, 2657 }; 2658 struct perf_pmu *pmu = evsel->pmu; 2659 int old_errno; 2660 2661 old_errno = errno; 2662 2663 if (pmu == NULL) 2664 pmu = evsel->pmu = evsel__find_pmu(evsel); 2665 2666 if (pmu == NULL || pmu->missing_features.checked) 2667 goto out; 2668 2669 /* 2670 * Must probe features in the order they were added to the 2671 * perf_event_attr interface. These are kernel core limitation but 2672 * specific to PMUs with branch stack. So we can detect with the given 2673 * hardware event and stop on the first one succeeded. 2674 */ 2675 2676 /* Please add new feature detection here. */ 2677 2678 attr.exclude_guest = 1; 2679 if (has_attr_feature(&attr, /*flags=*/0)) 2680 goto found; 2681 pmu->missing_features.exclude_guest = true; 2682 pr_debug2("switching off exclude_guest for PMU %s\n", pmu->name); 2683 2684 found: 2685 pmu->missing_features.checked = true; 2686 out: 2687 errno = old_errno; 2688 } 2689 2690 static void evsel__detect_missing_brstack_features(struct evsel *evsel) 2691 { 2692 static bool detection_done = false; 2693 struct perf_event_attr attr = { 2694 .type = evsel->core.attr.type, 2695 .config = evsel->core.attr.config, 2696 .disabled = 1, 2697 .sample_type = PERF_SAMPLE_BRANCH_STACK, 2698 .sample_period = 1000, 2699 }; 2700 int old_errno; 2701 2702 if (detection_done) 2703 return; 2704 2705 old_errno = errno; 2706 2707 /* 2708 * Must probe features in the order they were added to the 2709 * perf_event_attr interface. These are PMU specific limitation 2710 * so we can detect with the given hardware event and stop on the 2711 * first one succeeded. 2712 */ 2713 2714 /* Please add new feature detection here. */ 2715 2716 attr.branch_sample_type = PERF_SAMPLE_BRANCH_COUNTERS; 2717 if (has_attr_feature(&attr, /*flags=*/0)) 2718 goto found; 2719 perf_missing_features.branch_counters = true; 2720 pr_debug2("switching off branch counters support\n"); 2721 2722 attr.branch_sample_type = PERF_SAMPLE_BRANCH_HW_INDEX; 2723 if (has_attr_feature(&attr, /*flags=*/0)) 2724 goto found; 2725 perf_missing_features.branch_hw_idx = true; 2726 pr_debug2("switching off branch HW index support\n"); 2727 2728 attr.branch_sample_type = PERF_SAMPLE_BRANCH_NO_CYCLES | PERF_SAMPLE_BRANCH_NO_FLAGS; 2729 if (has_attr_feature(&attr, /*flags=*/0)) 2730 goto found; 2731 perf_missing_features.lbr_flags = true; 2732 pr_debug2_peo("switching off branch sample type no (cycles/flags)\n"); 2733 2734 found: 2735 detection_done = true; 2736 errno = old_errno; 2737 } 2738 2739 static bool evsel__probe_aux_action(struct evsel *evsel, struct perf_cpu cpu) 2740 { 2741 struct perf_event_attr attr = evsel->core.attr; 2742 int old_errno = errno; 2743 2744 attr.disabled = 1; 2745 attr.aux_start_paused = 1; 2746 2747 if (__has_attr_feature(&attr, cpu, /*flags=*/0)) { 2748 errno = old_errno; 2749 return true; 2750 } 2751 2752 /* 2753 * EOPNOTSUPP means the kernel supports the feature but the PMU does 2754 * not, so keep that distinction if possible. 2755 */ 2756 if (errno != EOPNOTSUPP) 2757 errno = old_errno; 2758 2759 return false; 2760 } 2761 2762 static void evsel__detect_missing_aux_action_feature(struct evsel *evsel, struct perf_cpu cpu) 2763 { 2764 static bool detection_done; 2765 struct evsel *leader; 2766 2767 /* 2768 * Don't bother probing aux_action if it is not being used or has been 2769 * probed before. 2770 */ 2771 if (!evsel->core.attr.aux_action || detection_done) 2772 return; 2773 2774 detection_done = true; 2775 2776 /* 2777 * The leader is an AUX area event. If it has failed, assume the feature 2778 * is not supported. 2779 */ 2780 leader = evsel__leader(evsel); 2781 if (evsel == leader) { 2782 perf_missing_features.aux_action = true; 2783 return; 2784 } 2785 2786 /* 2787 * AUX area event with aux_action must have been opened successfully 2788 * already, so feature is supported. 2789 */ 2790 if (leader->core.attr.aux_action) 2791 return; 2792 2793 if (!evsel__probe_aux_action(leader, cpu)) 2794 perf_missing_features.aux_action = true; 2795 } 2796 2797 static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu cpu) 2798 { 2799 static bool detection_done = false; 2800 struct perf_event_attr attr = { 2801 .type = PERF_TYPE_SOFTWARE, 2802 .config = PERF_COUNT_SW_TASK_CLOCK, 2803 .disabled = 1, 2804 }; 2805 int old_errno; 2806 2807 evsel__detect_missing_aux_action_feature(evsel, cpu); 2808 2809 evsel__detect_missing_pmu_features(evsel); 2810 2811 if (evsel__has_br_stack(evsel)) 2812 evsel__detect_missing_brstack_features(evsel); 2813 2814 if (detection_done) 2815 goto check; 2816 2817 old_errno = errno; 2818 2819 /* 2820 * Must probe features in the order they were added to the 2821 * perf_event_attr interface. These are kernel core limitation 2822 * not PMU-specific so we can detect with a software event and 2823 * stop on the first one succeeded. 2824 */ 2825 2826 /* Please add new feature detection here. */ 2827 2828 attr.defer_callchain = true; 2829 if (has_attr_feature(&attr, /*flags=*/0)) 2830 goto found; 2831 perf_missing_features.defer_callchain = true; 2832 pr_debug2("switching off deferred callchain support\n"); 2833 attr.defer_callchain = false; 2834 2835 attr.inherit = true; 2836 attr.sample_type = PERF_SAMPLE_READ | PERF_SAMPLE_TID; 2837 if (has_attr_feature(&attr, /*flags=*/0)) 2838 goto found; 2839 perf_missing_features.inherit_sample_read = true; 2840 pr_debug2("Using PERF_SAMPLE_READ / :S modifier is not compatible with inherit, falling back to no-inherit.\n"); 2841 attr.inherit = false; 2842 attr.sample_type = 0; 2843 2844 attr.read_format = PERF_FORMAT_LOST; 2845 if (has_attr_feature(&attr, /*flags=*/0)) 2846 goto found; 2847 perf_missing_features.read_lost = true; 2848 pr_debug2("switching off PERF_FORMAT_LOST support\n"); 2849 attr.read_format = 0; 2850 2851 attr.sample_type = PERF_SAMPLE_WEIGHT_STRUCT; 2852 if (has_attr_feature(&attr, /*flags=*/0)) 2853 goto found; 2854 perf_missing_features.weight_struct = true; 2855 pr_debug2("switching off weight struct support\n"); 2856 attr.sample_type = 0; 2857 2858 attr.sample_type = PERF_SAMPLE_CODE_PAGE_SIZE; 2859 if (has_attr_feature(&attr, /*flags=*/0)) 2860 goto found; 2861 perf_missing_features.code_page_size = true; 2862 pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support\n"); 2863 attr.sample_type = 0; 2864 2865 attr.sample_type = PERF_SAMPLE_DATA_PAGE_SIZE; 2866 if (has_attr_feature(&attr, /*flags=*/0)) 2867 goto found; 2868 perf_missing_features.data_page_size = true; 2869 pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support\n"); 2870 attr.sample_type = 0; 2871 2872 attr.cgroup = 1; 2873 if (has_attr_feature(&attr, /*flags=*/0)) 2874 goto found; 2875 perf_missing_features.cgroup = true; 2876 pr_debug2_peo("Kernel has no cgroup sampling support\n"); 2877 attr.cgroup = 0; 2878 2879 attr.aux_output = 1; 2880 if (has_attr_feature(&attr, /*flags=*/0)) 2881 goto found; 2882 perf_missing_features.aux_output = true; 2883 pr_debug2_peo("Kernel has no attr.aux_output support\n"); 2884 attr.aux_output = 0; 2885 2886 attr.bpf_event = 1; 2887 if (has_attr_feature(&attr, /*flags=*/0)) 2888 goto found; 2889 perf_missing_features.bpf = true; 2890 pr_debug2_peo("switching off bpf_event\n"); 2891 attr.bpf_event = 0; 2892 2893 attr.ksymbol = 1; 2894 if (has_attr_feature(&attr, /*flags=*/0)) 2895 goto found; 2896 perf_missing_features.ksymbol = true; 2897 pr_debug2_peo("switching off ksymbol\n"); 2898 attr.ksymbol = 0; 2899 2900 attr.write_backward = 1; 2901 if (has_attr_feature(&attr, /*flags=*/0)) 2902 goto found; 2903 perf_missing_features.write_backward = true; 2904 pr_debug2_peo("switching off write_backward\n"); 2905 attr.write_backward = 0; 2906 2907 attr.use_clockid = 1; 2908 attr.clockid = CLOCK_MONOTONIC; 2909 if (has_attr_feature(&attr, /*flags=*/0)) 2910 goto found; 2911 perf_missing_features.clockid = true; 2912 pr_debug2_peo("switching off clockid\n"); 2913 attr.use_clockid = 0; 2914 attr.clockid = 0; 2915 2916 if (has_attr_feature(&attr, /*flags=*/PERF_FLAG_FD_CLOEXEC)) 2917 goto found; 2918 perf_missing_features.cloexec = true; 2919 pr_debug2_peo("switching off cloexec flag\n"); 2920 2921 attr.mmap2 = 1; 2922 if (has_attr_feature(&attr, /*flags=*/0)) 2923 goto found; 2924 perf_missing_features.mmap2 = true; 2925 pr_debug2_peo("switching off mmap2\n"); 2926 attr.mmap2 = 0; 2927 2928 /* set this unconditionally? */ 2929 perf_missing_features.sample_id_all = true; 2930 pr_debug2_peo("switching off sample_id_all\n"); 2931 2932 attr.inherit = 1; 2933 attr.read_format = PERF_FORMAT_GROUP; 2934 if (has_attr_feature(&attr, /*flags=*/0)) 2935 goto found; 2936 perf_missing_features.group_read = true; 2937 pr_debug2_peo("switching off group read\n"); 2938 attr.inherit = 0; 2939 attr.read_format = 0; 2940 2941 found: 2942 detection_done = true; 2943 errno = old_errno; 2944 2945 check: 2946 if ((evsel->core.attr.defer_callchain || evsel->core.attr.defer_output) && 2947 perf_missing_features.defer_callchain) 2948 return true; 2949 2950 if (evsel->core.attr.inherit && 2951 (evsel->core.attr.sample_type & PERF_SAMPLE_READ) && 2952 perf_missing_features.inherit_sample_read) 2953 return true; 2954 2955 if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && 2956 perf_missing_features.branch_counters) 2957 return true; 2958 2959 if ((evsel->core.attr.read_format & PERF_FORMAT_LOST) && 2960 perf_missing_features.read_lost) 2961 return true; 2962 2963 if ((evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT) && 2964 perf_missing_features.weight_struct) 2965 return true; 2966 2967 if (evsel->core.attr.use_clockid && evsel->core.attr.clockid != CLOCK_MONOTONIC && 2968 !perf_missing_features.clockid) { 2969 perf_missing_features.clockid_wrong = true; 2970 return true; 2971 } 2972 2973 if (evsel->core.attr.use_clockid && perf_missing_features.clockid) 2974 return true; 2975 2976 if ((evsel->open_flags & PERF_FLAG_FD_CLOEXEC) && 2977 perf_missing_features.cloexec) 2978 return true; 2979 2980 if (evsel->core.attr.mmap2 && perf_missing_features.mmap2) 2981 return true; 2982 2983 if ((evsel->core.attr.branch_sample_type & (PERF_SAMPLE_BRANCH_NO_FLAGS | 2984 PERF_SAMPLE_BRANCH_NO_CYCLES)) && 2985 perf_missing_features.lbr_flags) 2986 return true; 2987 2988 if (evsel->core.attr.inherit && (evsel->core.attr.read_format & PERF_FORMAT_GROUP) && 2989 perf_missing_features.group_read) 2990 return true; 2991 2992 if (evsel->core.attr.ksymbol && perf_missing_features.ksymbol) 2993 return true; 2994 2995 if (evsel->core.attr.bpf_event && perf_missing_features.bpf) 2996 return true; 2997 2998 if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) && 2999 perf_missing_features.branch_hw_idx) 3000 return true; 3001 3002 if (evsel->core.attr.sample_id_all && perf_missing_features.sample_id_all) 3003 return true; 3004 3005 return false; 3006 } 3007 3008 static void evsel__close_fds(struct evsel *evsel, int start_cpu_map_idx, 3009 int cpu_map_idx, int thread, int nthreads) 3010 { 3011 do { 3012 while (--thread >= 0) { 3013 if (FD(evsel, cpu_map_idx, thread) >= 0) 3014 close(FD(evsel, cpu_map_idx, thread)); 3015 FD(evsel, cpu_map_idx, thread) = -1; 3016 } 3017 thread = nthreads; 3018 } while (--cpu_map_idx >= start_cpu_map_idx); 3019 } 3020 3021 static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, 3022 struct perf_thread_map *threads, 3023 int start_cpu_map_idx, int end_cpu_map_idx) 3024 { 3025 int idx, thread, nthreads; 3026 int pid = -1, err, old_errno; 3027 enum rlimit_action set_rlimit = NO_CHANGE; 3028 struct perf_cpu cpu; 3029 3030 /* 3031 * A failed attempt being retried after evsel__fallback() would 3032 * otherwise keep group members out of their group in get_group_fd(). 3033 */ 3034 evsel->supported = true; 3035 3036 if (evsel__is_retire_lat(evsel)) { 3037 err = evsel__tpebs_open(evsel); 3038 goto out; 3039 } 3040 3041 err = __evsel__prepare_open(evsel, cpus, threads); 3042 if (err) 3043 goto out; 3044 3045 if (cpus == NULL) 3046 cpus = empty_cpu_map; 3047 3048 if (threads == NULL) 3049 threads = empty_thread_map; 3050 3051 nthreads = perf_thread_map__nr(threads); 3052 3053 if (evsel->cgrp) 3054 pid = evsel->cgrp->fd; 3055 3056 fallback_missing_features: 3057 evsel__disable_missing_features(evsel); 3058 3059 pr_debug3("Opening: %s\n", evsel__name(evsel)); 3060 display_attr(&evsel->core.attr); 3061 3062 if (evsel__is_tool(evsel)) { 3063 err = evsel__tool_pmu_open(evsel, threads, 3064 start_cpu_map_idx, 3065 end_cpu_map_idx); 3066 goto out; 3067 } 3068 if (evsel__is_hwmon(evsel)) { 3069 err = evsel__hwmon_pmu_open(evsel, threads, 3070 start_cpu_map_idx, 3071 end_cpu_map_idx); 3072 goto out; 3073 } 3074 if (evsel__is_drm(evsel)) { 3075 err = evsel__drm_pmu_open(evsel, threads, 3076 start_cpu_map_idx, 3077 end_cpu_map_idx); 3078 goto out; 3079 } 3080 3081 for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) { 3082 cpu = perf_cpu_map__cpu(cpus, idx); 3083 3084 for (thread = 0; thread < nthreads; thread++) { 3085 int fd, group_fd; 3086 retry_open: 3087 if (thread >= nthreads) 3088 break; 3089 3090 if (!evsel->cgrp && !evsel->core.system_wide) 3091 pid = perf_thread_map__pid(threads, thread); 3092 3093 group_fd = get_group_fd(evsel, idx, thread); 3094 3095 if (group_fd == -2) { 3096 pr_debug("broken group leader for %s\n", evsel->name); 3097 err = -EINVAL; 3098 goto out_close; 3099 } 3100 3101 /* Debug message used by test scripts */ 3102 pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx", 3103 pid, cpu.cpu, group_fd, evsel->open_flags); 3104 3105 fd = sys_perf_event_open(&evsel->core.attr, pid, cpu.cpu, 3106 group_fd, evsel->open_flags); 3107 3108 FD(evsel, idx, thread) = fd; 3109 3110 if (fd < 0) { 3111 err = -errno; 3112 3113 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n", 3114 err); 3115 goto try_fallback; 3116 } 3117 3118 bpf_counter__install_pe(evsel, idx, fd); 3119 3120 /* 3121 * Support probes aren't what was asked for, leave them 3122 * out of the PERF_TEST_ATTR dump. 3123 */ 3124 if (unlikely(test_attr__enabled() && !evsel->support_probe)) { 3125 test_attr__open(&evsel->core.attr, pid, cpu, 3126 fd, group_fd, evsel->open_flags); 3127 } 3128 3129 /* Debug message used by test scripts */ 3130 pr_debug2_peo(" = %d\n", fd); 3131 3132 set_rlimit = NO_CHANGE; 3133 3134 /* 3135 * If we succeeded but had to kill clockid, fail and 3136 * have evsel__open_strerror() print us a nice error. 3137 */ 3138 if (perf_missing_features.clockid || 3139 perf_missing_features.clockid_wrong) { 3140 err = -EINVAL; 3141 goto out_close; 3142 } 3143 } 3144 } 3145 3146 err = 0; 3147 goto out; 3148 3149 try_fallback: 3150 if (evsel__ignore_missing_thread(evsel, perf_cpu_map__nr(cpus), 3151 idx, threads, thread, err)) { 3152 /* We just removed 1 thread, so lower the upper nthreads limit. */ 3153 nthreads--; 3154 3155 /* ... and pretend like nothing have happened. */ 3156 err = 0; 3157 goto retry_open; 3158 } 3159 /* 3160 * perf stat needs between 5 and 22 fds per CPU. When we run out 3161 * of them try to increase the limits. 3162 */ 3163 if (err == -EMFILE && rlimit__increase_nofile(&set_rlimit)) 3164 goto retry_open; 3165 3166 if (err == -EINVAL && evsel__detect_missing_features(evsel, cpu)) { 3167 /* 3168 * The restart opens the events again, so close what this 3169 * attempt opened before its entries are overwritten. 3170 */ 3171 evsel__close_fds(evsel, start_cpu_map_idx, idx, thread, nthreads); 3172 goto fallback_missing_features; 3173 } 3174 3175 if (evsel__precise_ip_fallback(evsel)) 3176 goto retry_open; 3177 3178 out_close: 3179 if (err) 3180 threads->err_thread = thread; 3181 3182 old_errno = errno; 3183 evsel__close_fds(evsel, start_cpu_map_idx, idx, thread, nthreads); 3184 errno = old_errno; 3185 out: 3186 if (err) 3187 evsel->supported = false; 3188 return err; 3189 } 3190 3191 int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus, 3192 struct perf_thread_map *threads) 3193 { 3194 return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus)); 3195 } 3196 3197 void evsel__close(struct evsel *evsel) 3198 { 3199 if (evsel__is_retire_lat(evsel)) 3200 evsel__tpebs_close(evsel); 3201 perf_evsel__close(&evsel->core); 3202 perf_evsel__free_id(&evsel->core); 3203 } 3204 3205 int evsel__open_per_cpu_and_thread(struct evsel *evsel, 3206 struct perf_cpu_map *cpus, int cpu_map_idx, 3207 struct perf_thread_map *threads) 3208 { 3209 if (cpu_map_idx == -1) 3210 return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus)); 3211 3212 return evsel__open_cpu(evsel, cpus, threads, cpu_map_idx, cpu_map_idx + 1); 3213 } 3214 3215 int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx) 3216 { 3217 struct perf_thread_map *threads = thread_map__new_by_tid(-1); 3218 int ret = evsel__open_per_cpu_and_thread(evsel, cpus, cpu_map_idx, threads); 3219 3220 perf_thread_map__put(threads); 3221 return ret; 3222 } 3223 3224 int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads) 3225 { 3226 struct perf_cpu_map *cpus = perf_cpu_map__new_any_cpu(); 3227 int ret = evsel__open_per_cpu_and_thread(evsel, cpus, -1, threads); 3228 3229 perf_cpu_map__put(cpus); 3230 return ret; 3231 } 3232 3233 static int perf_evsel__parse_id_sample(const union perf_event *event, 3234 struct perf_sample *sample) 3235 { 3236 const struct evsel *evsel = sample->evsel; 3237 u64 type = evsel->core.attr.sample_type; 3238 const __u64 *array = event->sample.array; 3239 bool swapped = evsel->needs_swap; 3240 union u64_swap u; 3241 int i = ((evsel__event_size(evsel, event) - sizeof(event->header)) / sizeof(u64)) - 1; 3242 3243 if (type & PERF_SAMPLE_IDENTIFIER) { 3244 if (i < 0) 3245 return -EFAULT; 3246 3247 sample->id = array[i--]; 3248 } 3249 3250 if (type & PERF_SAMPLE_CPU) { 3251 if (i < 0) 3252 return -EFAULT; 3253 3254 u.val64 = array[i--]; 3255 if (swapped) { 3256 /* undo swap of u64, then swap on individual u32s */ 3257 u.val64 = bswap_64(u.val64); 3258 u.val32[0] = bswap_32(u.val32[0]); 3259 } 3260 sample->cpu = u.val32[0]; 3261 } 3262 3263 if (type & PERF_SAMPLE_STREAM_ID) { 3264 if (i < 0) 3265 return -EFAULT; 3266 3267 sample->stream_id = array[i--]; 3268 } 3269 3270 if (type & PERF_SAMPLE_ID) { 3271 if (i < 0) 3272 return -EFAULT; 3273 3274 sample->id = array[i--]; 3275 } 3276 3277 if (type & PERF_SAMPLE_TIME) { 3278 if (i < 0) 3279 return -EFAULT; 3280 3281 sample->time = array[i--]; 3282 } 3283 3284 if (type & PERF_SAMPLE_TID) { 3285 if (i < 0) 3286 return -EFAULT; 3287 3288 u.val64 = array[i--]; 3289 if (swapped) { 3290 /* undo swap of u64, then swap on individual u32s */ 3291 u.val64 = bswap_64(u.val64); 3292 u.val32[0] = bswap_32(u.val32[0]); 3293 u.val32[1] = bswap_32(u.val32[1]); 3294 } 3295 3296 sample->pid = u.val32[0]; 3297 sample->tid = u.val32[1]; 3298 } 3299 3300 return 0; 3301 } 3302 3303 static inline bool overflow(const void *endp, u16 max_size, const void *offset, 3304 u64 size) 3305 { 3306 return size > max_size || offset + size > endp; 3307 } 3308 3309 #define OVERFLOW_CHECK(offset, size, max_size) \ 3310 do { \ 3311 if (overflow(endp, (max_size), (offset), (size))) \ 3312 goto out_efault; \ 3313 } while (0) 3314 3315 #define OVERFLOW_CHECK_u64(offset) \ 3316 OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64)) 3317 3318 static int 3319 perf_event__check_size(union perf_event *event, unsigned int sample_size) 3320 { 3321 /* 3322 * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes 3323 * up to PERF_SAMPLE_PERIOD. After that overflow() must be used to 3324 * check the format does not go past the end of the event. 3325 */ 3326 if (sample_size + sizeof(event->header) > event->header.size) 3327 return -EFAULT; 3328 3329 return 0; 3330 } 3331 3332 static void perf_parse_sample_weight(struct perf_sample *data, const __u64 *array, u64 type) 3333 { 3334 union perf_sample_weight weight; 3335 3336 weight.full = *array; 3337 if (type & PERF_SAMPLE_WEIGHT_STRUCT) { 3338 data->weight = weight.var1_dw; 3339 data->ins_lat = weight.var2_w; 3340 data->weight3 = weight.var3_w; 3341 } else { 3342 data->weight = weight.full; 3343 } 3344 } 3345 3346 u64 evsel__bitfield_swap_branch_flags(u64 value) 3347 { 3348 u64 new_val = 0; 3349 3350 /* 3351 * branch_flags 3352 * union { 3353 * u64 values; 3354 * struct { 3355 * mispred:1 //target mispredicted 3356 * predicted:1 //target predicted 3357 * in_tx:1 //in transaction 3358 * abort:1 //transaction abort 3359 * cycles:16 //cycle count to last branch 3360 * type:4 //branch type 3361 * spec:2 //branch speculation info 3362 * new_type:4 //additional branch type 3363 * priv:3 //privilege level 3364 * reserved:31 3365 * } 3366 * } 3367 * 3368 * Avoid bswap64() the entire branch_flag.value, 3369 * as it has variable bit-field sizes. Instead the 3370 * macro takes the bit-field position/size, 3371 * swaps it based on the host endianness. 3372 */ 3373 if (host_is_bigendian()) { 3374 new_val = bitfield_swap(value, 0, 1); 3375 new_val |= bitfield_swap(value, 1, 1); 3376 new_val |= bitfield_swap(value, 2, 1); 3377 new_val |= bitfield_swap(value, 3, 1); 3378 new_val |= bitfield_swap(value, 4, 16); 3379 new_val |= bitfield_swap(value, 20, 4); 3380 new_val |= bitfield_swap(value, 24, 2); 3381 new_val |= bitfield_swap(value, 26, 4); 3382 new_val |= bitfield_swap(value, 30, 3); 3383 new_val |= bitfield_swap(value, 33, 31); 3384 } else { 3385 new_val = bitfield_swap(value, 63, 1); 3386 new_val |= bitfield_swap(value, 62, 1); 3387 new_val |= bitfield_swap(value, 61, 1); 3388 new_val |= bitfield_swap(value, 60, 1); 3389 new_val |= bitfield_swap(value, 44, 16); 3390 new_val |= bitfield_swap(value, 40, 4); 3391 new_val |= bitfield_swap(value, 38, 2); 3392 new_val |= bitfield_swap(value, 34, 4); 3393 new_val |= bitfield_swap(value, 31, 3); 3394 new_val |= bitfield_swap(value, 0, 31); 3395 } 3396 3397 return new_val; 3398 } 3399 3400 static inline bool evsel__has_branch_counters(const struct evsel *evsel) 3401 { 3402 struct evsel *leader = evsel__leader(evsel); 3403 3404 /* The branch counters feature only supports group */ 3405 if (!leader || !evsel->evlist) 3406 return false; 3407 3408 if (evlist__nr_br_cntr(evsel->evlist) < 0) 3409 evlist__update_br_cntr(evsel->evlist); 3410 3411 if (leader->br_cntr_nr > 0) 3412 return true; 3413 3414 return false; 3415 } 3416 3417 static int __set_offcpu_sample(struct perf_sample *data) 3418 { 3419 u64 *array = data->raw_data; 3420 u32 max_size = data->raw_size, *p32; 3421 const void *endp = (void *)array + max_size; 3422 3423 if (array == NULL) 3424 return -EFAULT; 3425 3426 OVERFLOW_CHECK_u64(array); 3427 p32 = (void *)array++; 3428 data->pid = p32[0]; 3429 data->tid = p32[1]; 3430 3431 OVERFLOW_CHECK_u64(array); 3432 data->period = *array++; 3433 3434 OVERFLOW_CHECK_u64(array); 3435 data->callchain = (struct ip_callchain *)array++; 3436 OVERFLOW_CHECK(array, data->callchain->nr * sizeof(u64), max_size); 3437 data->ip = data->callchain->ips[1]; 3438 array += data->callchain->nr; 3439 3440 OVERFLOW_CHECK_u64(array); 3441 data->cgroup = *array; 3442 3443 return 0; 3444 out_efault: 3445 return -EFAULT; 3446 } 3447 3448 int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, 3449 struct perf_sample *data, bool needs_swap) 3450 { 3451 u64 type = evsel->core.attr.sample_type; 3452 bool swapped = needs_swap; 3453 const __u64 *array; 3454 u16 max_size = event->header.size; 3455 const void *endp = (void *)event + max_size; 3456 u64 sz; 3457 3458 /* 3459 * used for cross-endian analysis. See git commit 65014ab3 3460 * for why this goofiness is needed. 3461 */ 3462 union u64_swap u; 3463 3464 perf_sample__init(data, /*all=*/true); 3465 data->evsel = evsel__get(evsel); 3466 data->cpu = data->pid = data->tid = -1; 3467 data->stream_id = data->id = data->time = -1ULL; 3468 data->period = evsel->core.attr.sample_period; 3469 data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; 3470 data->misc = event->header.misc; 3471 data->data_src = PERF_MEM_DATA_SRC_NONE; 3472 data->vcpu = -1; 3473 3474 if (event->header.type == PERF_RECORD_CALLCHAIN_DEFERRED) { 3475 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64); 3476 3477 data->callchain = (struct ip_callchain *)&event->callchain_deferred.nr; 3478 if (data->callchain->nr > max_callchain_nr) 3479 goto out_efault; 3480 3481 data->deferred_cookie = event->callchain_deferred.cookie; 3482 3483 if (evsel->core.attr.sample_id_all) { 3484 if (perf_evsel__parse_id_sample(event, data)) 3485 goto out_efault; 3486 } 3487 return 0; 3488 } 3489 3490 if (event->header.type != PERF_RECORD_SAMPLE) { 3491 if (evsel->core.attr.sample_id_all) { 3492 if (perf_evsel__parse_id_sample(event, data)) 3493 goto out_efault; 3494 } 3495 return 0; 3496 } 3497 3498 array = event->sample.array; 3499 3500 if (perf_event__check_size(event, evsel->sample_size)) 3501 goto out_efault; 3502 3503 if (type & PERF_SAMPLE_IDENTIFIER) { 3504 data->id = *array; 3505 array++; 3506 } 3507 3508 if (type & PERF_SAMPLE_IP) { 3509 data->ip = *array; 3510 array++; 3511 } 3512 3513 if (type & PERF_SAMPLE_TID) { 3514 u.val64 = *array; 3515 if (swapped) { 3516 /* undo swap of u64, then swap on individual u32s */ 3517 u.val64 = bswap_64(u.val64); 3518 u.val32[0] = bswap_32(u.val32[0]); 3519 u.val32[1] = bswap_32(u.val32[1]); 3520 } 3521 3522 data->pid = u.val32[0]; 3523 data->tid = u.val32[1]; 3524 array++; 3525 } 3526 3527 if (type & PERF_SAMPLE_TIME) { 3528 data->time = *array; 3529 array++; 3530 } 3531 3532 if (type & PERF_SAMPLE_ADDR) { 3533 data->addr = *array; 3534 array++; 3535 } 3536 3537 if (type & PERF_SAMPLE_ID) { 3538 data->id = *array; 3539 array++; 3540 } 3541 3542 if (type & PERF_SAMPLE_STREAM_ID) { 3543 data->stream_id = *array; 3544 array++; 3545 } 3546 3547 if (type & PERF_SAMPLE_CPU) { 3548 3549 u.val64 = *array; 3550 if (swapped) { 3551 /* undo swap of u64, then swap on individual u32s */ 3552 u.val64 = bswap_64(u.val64); 3553 u.val32[0] = bswap_32(u.val32[0]); 3554 } 3555 3556 data->cpu = u.val32[0]; 3557 array++; 3558 } 3559 3560 if (type & PERF_SAMPLE_PERIOD) { 3561 data->period = *array; 3562 array++; 3563 } 3564 3565 if (type & PERF_SAMPLE_READ) { 3566 u64 read_format = evsel->core.attr.read_format; 3567 3568 OVERFLOW_CHECK_u64(array); 3569 if (read_format & PERF_FORMAT_GROUP) 3570 data->read.group.nr = *array; 3571 else 3572 data->read.one.value = *array; 3573 3574 array++; 3575 3576 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 3577 OVERFLOW_CHECK_u64(array); 3578 data->read.time_enabled = *array; 3579 array++; 3580 } 3581 3582 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 3583 OVERFLOW_CHECK_u64(array); 3584 data->read.time_running = *array; 3585 array++; 3586 } 3587 3588 /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */ 3589 if (read_format & PERF_FORMAT_GROUP) { 3590 const u64 max_group_nr = UINT64_MAX / 3591 sizeof(struct sample_read_value); 3592 3593 if (data->read.group.nr > max_group_nr) 3594 goto out_efault; 3595 3596 sz = data->read.group.nr * sample_read_value_size(read_format); 3597 OVERFLOW_CHECK(array, sz, max_size); 3598 data->read.group.values = 3599 (struct sample_read_value *)array; 3600 array = (void *)array + sz; 3601 } else { 3602 OVERFLOW_CHECK_u64(array); 3603 data->read.one.id = *array; 3604 array++; 3605 3606 if (read_format & PERF_FORMAT_LOST) { 3607 OVERFLOW_CHECK_u64(array); 3608 data->read.one.lost = *array; 3609 array++; 3610 } 3611 } 3612 } 3613 3614 if (type & PERF_SAMPLE_CALLCHAIN) { 3615 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64); 3616 u64 callchain_nr; 3617 3618 OVERFLOW_CHECK_u64(array); 3619 data->callchain = (struct ip_callchain *)array++; 3620 callchain_nr = data->callchain->nr; 3621 if (callchain_nr > max_callchain_nr) 3622 goto out_efault; 3623 sz = callchain_nr * sizeof(u64); 3624 /* 3625 * Save the cookie for the deferred user callchain. The last 2 3626 * entries in the callchain should be the context marker and the 3627 * cookie. The cookie will be used to match PERF_RECORD_ 3628 * CALLCHAIN_DEFERRED later. 3629 */ 3630 if (evsel->core.attr.defer_callchain && callchain_nr >= 2 && 3631 data->callchain->ips[callchain_nr - 2] == PERF_CONTEXT_USER_DEFERRED) { 3632 data->deferred_cookie = data->callchain->ips[callchain_nr - 1]; 3633 data->deferred_callchain = true; 3634 } 3635 OVERFLOW_CHECK(array, sz, max_size); 3636 array = (void *)array + sz; 3637 } 3638 3639 if (type & PERF_SAMPLE_RAW) { 3640 const __u64 *raw; 3641 3642 OVERFLOW_CHECK_u64(array); 3643 raw = array; 3644 u.val64 = *array; 3645 3646 /* 3647 * Undo swap of u64, then swap on individual u32s, 3648 * get the size of the raw area and undo all of the 3649 * swap. The pevent interface handles endianness by 3650 * itself. 3651 */ 3652 if (swapped) { 3653 u.val64 = bswap_64(u.val64); 3654 u.val32[0] = bswap_32(u.val32[0]); 3655 u.val32[1] = bswap_32(u.val32[1]); 3656 } 3657 data->raw_size = u.val32[0]; 3658 3659 array = (void *)array + sizeof(u32); 3660 OVERFLOW_CHECK(array, data->raw_size, max_size); 3661 if (swapped) { 3662 /* mem_bswap_64() accesses complete 64-bit words. */ 3663 sz = roundup((u64)data->raw_size, sizeof(u64)); 3664 OVERFLOW_CHECK(raw, sz, max_size); 3665 mem_bswap_64((void *)raw, data->raw_size); 3666 } 3667 data->raw_data = (void *)array; 3668 array = (void *)array + data->raw_size; 3669 } 3670 3671 if (type & PERF_SAMPLE_BRANCH_STACK) { 3672 const u64 max_branch_nr = UINT64_MAX / 3673 sizeof(struct branch_entry); 3674 struct branch_entry *e; 3675 unsigned int i; 3676 3677 OVERFLOW_CHECK_u64(array); 3678 data->branch_stack = (struct branch_stack *)array++; 3679 3680 if (data->branch_stack->nr > max_branch_nr) 3681 goto out_efault; 3682 3683 sz = data->branch_stack->nr * sizeof(struct branch_entry); 3684 if (evsel__has_branch_hw_idx(evsel)) { 3685 sz += sizeof(u64); 3686 e = &data->branch_stack->entries[0]; 3687 } else { 3688 data->no_hw_idx = true; 3689 /* 3690 * if the PERF_SAMPLE_BRANCH_HW_INDEX is not applied, 3691 * only nr and entries[] will be output by kernel. 3692 */ 3693 e = (struct branch_entry *)&data->branch_stack->hw_idx; 3694 } 3695 3696 OVERFLOW_CHECK(array, sz, max_size); 3697 3698 if (swapped) { 3699 /* 3700 * struct branch_flag does not have endian 3701 * specific bit field definition. And bswap 3702 * will not resolve the issue, since these 3703 * are bit fields. 3704 * 3705 * evsel__bitfield_swap_branch_flags() uses a 3706 * bitfield_swap macro to swap the bit position 3707 * based on the host endians. 3708 */ 3709 for (i = 0; i < data->branch_stack->nr; i++, e++) 3710 e->flags.value = evsel__bitfield_swap_branch_flags(e->flags.value); 3711 } 3712 3713 array = (void *)array + sz; 3714 3715 if (evsel__has_branch_counters(evsel)) { 3716 data->branch_stack_cntr = (u64 *)array; 3717 sz = data->branch_stack->nr * sizeof(u64); 3718 3719 OVERFLOW_CHECK(array, sz, max_size); 3720 array = (void *)array + sz; 3721 } 3722 } 3723 3724 if (type & PERF_SAMPLE_REGS_USER) { 3725 struct regs_dump *regs = perf_sample__user_regs(data); 3726 3727 OVERFLOW_CHECK_u64(array); 3728 regs->abi = *array; 3729 array++; 3730 3731 if (regs->abi) { 3732 u64 mask = evsel->core.attr.sample_regs_user; 3733 3734 sz = hweight64(mask) * sizeof(u64); 3735 OVERFLOW_CHECK(array, sz, max_size); 3736 regs->mask = mask; 3737 regs->regs = (u64 *)array; 3738 array = (void *)array + sz; 3739 3740 if (regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) { 3741 u64 attr_nr_vectors = 3742 hweight64(evsel->core.attr.sample_simd_vec_reg_user); 3743 u64 attr_vec_qwords = 3744 evsel->core.attr.sample_simd_vec_reg_qwords; 3745 u64 attr_nr_pred = 3746 hweight32(evsel->core.attr.sample_simd_pred_reg_user); 3747 u64 attr_pred_qwords = 3748 evsel->core.attr.sample_simd_pred_reg_qwords; 3749 3750 OVERFLOW_CHECK_u64(array); 3751 regs->nr_vectors = *(u64 *)array; 3752 array = (void *)array + sizeof(u64); 3753 OVERFLOW_CHECK_u64(array); 3754 regs->vector_qwords = *(u64 *)array; 3755 array = (void *)array + sizeof(u64); 3756 OVERFLOW_CHECK_u64(array); 3757 regs->nr_pred = *(u64 *)array; 3758 array = (void *)array + sizeof(u64); 3759 OVERFLOW_CHECK_u64(array); 3760 regs->pred_qwords = *(u64 *)array; 3761 array = (void *)array + sizeof(u64); 3762 3763 if (regs->nr_vectors > attr_nr_vectors || 3764 regs->vector_qwords > attr_vec_qwords || 3765 regs->nr_pred > attr_nr_pred || 3766 regs->pred_qwords > attr_pred_qwords) 3767 goto out_efault; 3768 3769 sz = (regs->nr_vectors * regs->vector_qwords + 3770 regs->nr_pred * regs->pred_qwords) * sizeof(u64); 3771 OVERFLOW_CHECK(array, sz, max_size); 3772 3773 regs->simd_data = (u64 *)array; 3774 array = (void *)array + sz; 3775 } 3776 } 3777 } 3778 3779 if (type & PERF_SAMPLE_STACK_USER) { 3780 OVERFLOW_CHECK_u64(array); 3781 sz = *array++; 3782 3783 data->user_stack.offset = ((char *)(array - 1) 3784 - (char *) event); 3785 3786 if (!sz) { 3787 data->user_stack.size = 0; 3788 } else { 3789 OVERFLOW_CHECK(array, sz, max_size); 3790 data->user_stack.data = (char *)array; 3791 array = (void *)array + sz; 3792 OVERFLOW_CHECK_u64(array); 3793 data->user_stack.size = *array++; 3794 if (WARN_ONCE(data->user_stack.size > sz, 3795 "user stack dump failure\n")) 3796 goto out_efault; 3797 } 3798 } 3799 3800 if (type & PERF_SAMPLE_WEIGHT_TYPE) { 3801 OVERFLOW_CHECK_u64(array); 3802 perf_parse_sample_weight(data, array, type); 3803 array++; 3804 } 3805 3806 if (type & PERF_SAMPLE_DATA_SRC) { 3807 OVERFLOW_CHECK_u64(array); 3808 data->data_src = *array; 3809 array++; 3810 } 3811 3812 if (type & PERF_SAMPLE_TRANSACTION) { 3813 OVERFLOW_CHECK_u64(array); 3814 data->transaction = *array; 3815 array++; 3816 } 3817 3818 if (type & PERF_SAMPLE_REGS_INTR) { 3819 struct regs_dump *regs = perf_sample__intr_regs(data); 3820 3821 OVERFLOW_CHECK_u64(array); 3822 regs->abi = *array; 3823 array++; 3824 3825 if (regs->abi != PERF_SAMPLE_REGS_ABI_NONE) { 3826 u64 mask = evsel->core.attr.sample_regs_intr; 3827 3828 sz = hweight64(mask) * sizeof(u64); 3829 OVERFLOW_CHECK(array, sz, max_size); 3830 regs->mask = mask; 3831 regs->regs = (u64 *)array; 3832 array = (void *)array + sz; 3833 3834 if (regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) { 3835 u64 attr_nr_vectors = 3836 hweight64(evsel->core.attr.sample_simd_vec_reg_intr); 3837 u64 attr_vec_qwords = 3838 evsel->core.attr.sample_simd_vec_reg_qwords; 3839 u64 attr_nr_pred = 3840 hweight32(evsel->core.attr.sample_simd_pred_reg_intr); 3841 u64 attr_pred_qwords = 3842 evsel->core.attr.sample_simd_pred_reg_qwords; 3843 3844 OVERFLOW_CHECK_u64(array); 3845 regs->nr_vectors = *(u64 *)array; 3846 array = (void *)array + sizeof(u64); 3847 OVERFLOW_CHECK_u64(array); 3848 regs->vector_qwords = *(u64 *)array; 3849 array = (void *)array + sizeof(u64); 3850 OVERFLOW_CHECK_u64(array); 3851 regs->nr_pred = *(u64 *)array; 3852 array = (void *)array + sizeof(u64); 3853 OVERFLOW_CHECK_u64(array); 3854 regs->pred_qwords = *(u64 *)array; 3855 array = (void *)array + sizeof(u64); 3856 3857 if (regs->nr_vectors > attr_nr_vectors || 3858 regs->vector_qwords > attr_vec_qwords || 3859 regs->nr_pred > attr_nr_pred || 3860 regs->pred_qwords > attr_pred_qwords) 3861 goto out_efault; 3862 3863 sz = (regs->nr_vectors * regs->vector_qwords + 3864 regs->nr_pred * regs->pred_qwords) * sizeof(u64); 3865 OVERFLOW_CHECK(array, sz, max_size); 3866 3867 regs->simd_data = (u64 *)array; 3868 array = (void *)array + sz; 3869 } 3870 } 3871 } 3872 3873 data->phys_addr = 0; 3874 if (type & PERF_SAMPLE_PHYS_ADDR) { 3875 data->phys_addr = *array; 3876 array++; 3877 } 3878 3879 data->cgroup = 0; 3880 if (type & PERF_SAMPLE_CGROUP) { 3881 data->cgroup = *array; 3882 array++; 3883 } 3884 3885 data->data_page_size = 0; 3886 if (type & PERF_SAMPLE_DATA_PAGE_SIZE) { 3887 data->data_page_size = *array; 3888 array++; 3889 } 3890 3891 data->code_page_size = 0; 3892 if (type & PERF_SAMPLE_CODE_PAGE_SIZE) { 3893 data->code_page_size = *array; 3894 array++; 3895 } 3896 3897 if (type & PERF_SAMPLE_AUX) { 3898 OVERFLOW_CHECK_u64(array); 3899 sz = *array++; 3900 3901 OVERFLOW_CHECK(array, sz, max_size); 3902 /* Undo swap of data */ 3903 if (swapped) 3904 mem_bswap_64((char *)array, sz); 3905 data->aux_sample.size = sz; 3906 data->aux_sample.data = (char *)array; 3907 array = (void *)array + sz; 3908 } 3909 3910 if (evsel__is_offcpu_event(evsel)) { 3911 if (__set_offcpu_sample(data)) 3912 goto out_efault; 3913 } 3914 3915 return 0; 3916 out_efault: 3917 perf_sample__exit(data); 3918 return -EFAULT; 3919 } 3920 3921 int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, 3922 u64 *timestamp) 3923 { 3924 u64 type = evsel->core.attr.sample_type; 3925 const __u64 *array; 3926 3927 if (!(type & PERF_SAMPLE_TIME)) 3928 return -1; 3929 3930 if (event->header.type != PERF_RECORD_SAMPLE) { 3931 struct perf_sample data = { 3932 .evsel = evsel, 3933 .time = -1ULL, 3934 }; 3935 3936 if (!evsel->core.attr.sample_id_all) 3937 return -1; 3938 if (perf_evsel__parse_id_sample(event, &data)) 3939 return -1; 3940 3941 *timestamp = data.time; 3942 return 0; 3943 } 3944 3945 array = event->sample.array; 3946 3947 if (perf_event__check_size(event, evsel->sample_size)) 3948 return -EFAULT; 3949 3950 if (type & PERF_SAMPLE_IDENTIFIER) 3951 array++; 3952 3953 if (type & PERF_SAMPLE_IP) 3954 array++; 3955 3956 if (type & PERF_SAMPLE_TID) 3957 array++; 3958 3959 if (type & PERF_SAMPLE_TIME) 3960 *timestamp = *array; 3961 3962 return 0; 3963 } 3964 3965 u16 evsel__id_hdr_size(const struct evsel *evsel) 3966 { 3967 u64 sample_type = evsel->core.attr.sample_type; 3968 u16 size = 0; 3969 3970 if (sample_type & PERF_SAMPLE_TID) 3971 size += sizeof(u64); 3972 3973 if (sample_type & PERF_SAMPLE_TIME) 3974 size += sizeof(u64); 3975 3976 if (sample_type & PERF_SAMPLE_ID) 3977 size += sizeof(u64); 3978 3979 if (sample_type & PERF_SAMPLE_STREAM_ID) 3980 size += sizeof(u64); 3981 3982 if (sample_type & PERF_SAMPLE_CPU) 3983 size += sizeof(u64); 3984 3985 if (sample_type & PERF_SAMPLE_IDENTIFIER) 3986 size += sizeof(u64); 3987 3988 return size; 3989 } 3990 3991 /* 3992 * Prior to kernel fix, perf_event_ksymbol_output(), perf_event_bpf_output(), 3993 * and perf_event_text_poke_output() in kernel/events/core.c did not save and 3994 * restore event_id.header.size across perf_iterate_sb() iterations. When 3995 * multiple perf_events had attr.ksymbol, attr.bpf_event, or attr.text_poke 3996 * enabled, header.size was incremented by id_header_size for each matching 3997 * event while only a single id_sample was written immediately after the event 3998 * payload. Clamp the effective size used to locate the trailing id_sample to 3999 * payload + id_hdr_size so events recorded on unpatched kernels can be parsed 4000 * without reading uninitialized ring-buffer bytes. 4001 */ 4002 u16 evsel__event_size(const struct evsel *evsel, const union perf_event *event) 4003 { 4004 u16 size = event->header.size; 4005 u16 id_hdr_size; 4006 size_t payload; 4007 4008 if (!evsel->core.attr.sample_id_all) 4009 return size; 4010 4011 switch (event->header.type) { 4012 case PERF_RECORD_KSYMBOL: { 4013 const char *name = event->ksymbol.name; 4014 size_t fixed = offsetof(struct perf_record_ksymbol, name); 4015 size_t max_len, len; 4016 4017 if (size <= fixed) 4018 return size; 4019 max_len = size - fixed; 4020 len = strnlen(name, max_len); 4021 if (len == max_len) 4022 return size; 4023 payload = fixed + PERF_ALIGN(len + 1, sizeof(u64)); 4024 break; 4025 } 4026 case PERF_RECORD_BPF_EVENT: 4027 payload = sizeof(struct perf_record_bpf_event); 4028 break; 4029 case PERF_RECORD_TEXT_POKE: { 4030 size_t fixed = offsetof(struct perf_record_text_poke_event, bytes); 4031 4032 if (size < fixed) 4033 return size; 4034 payload = PERF_ALIGN(fixed + (size_t)event->text_poke.old_len + 4035 event->text_poke.new_len, sizeof(u64)); 4036 break; 4037 } 4038 default: 4039 return size; 4040 } 4041 4042 id_hdr_size = evsel__id_hdr_size(evsel); 4043 if (payload + id_hdr_size < size) 4044 return payload + id_hdr_size; 4045 return size; 4046 } 4047 4048 #ifdef HAVE_LIBTRACEEVENT 4049 struct tep_format_field *evsel__field(struct evsel *evsel, const char *name) 4050 { 4051 struct tep_event *tp_format = evsel__tp_format(evsel); 4052 4053 return tp_format ? tep_find_field(tp_format, name) : NULL; 4054 } 4055 4056 struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name) 4057 { 4058 struct tep_event *tp_format = evsel__tp_format(evsel); 4059 4060 return tp_format ? tep_find_common_field(tp_format, name) : NULL; 4061 } 4062 4063 static bool out_of_bounds(const struct tep_format_field *field, int offset, int size, u32 raw_size) 4064 { 4065 if (offset < 0) { 4066 pr_warning("Negative trace point field offset %d in %s\n", 4067 offset, field->name); 4068 return true; 4069 } 4070 if (size < 0) { 4071 pr_warning("Negative trace point field size %d in %s\n", 4072 size, field->name); 4073 return true; 4074 } 4075 if ((u32)offset + (u32)size > raw_size) { 4076 pr_warning("Out of bound tracepoint field (%s) offset %d size %d in %u\n", 4077 field->name, offset, size, raw_size); 4078 return true; 4079 } 4080 return false; 4081 } 4082 4083 void *perf_sample__rawptr(struct perf_sample *sample, const char *name) 4084 { 4085 struct tep_format_field *field = evsel__field(sample->evsel, name); 4086 int offset, size; 4087 4088 if (!field) 4089 return NULL; 4090 4091 offset = field->offset; 4092 size = field->size; 4093 if (field->flags & TEP_FIELD_IS_DYNAMIC) { 4094 int dynamic_data; 4095 4096 if (out_of_bounds(field, offset, 4, sample->raw_size)) 4097 return NULL; 4098 4099 dynamic_data = *(int *)(sample->raw_data + field->offset); 4100 4101 if (sample->evsel->needs_swap) 4102 dynamic_data = bswap_32(dynamic_data); 4103 4104 offset = dynamic_data & 0xffff; 4105 size = (dynamic_data >> 16) & 0xffff; 4106 4107 if (tep_field_is_relative(field->flags)) { 4108 /* 4109 * Newer kernel feature: Relative offsets (__rel_loc). 4110 * If the relative flag is set, the parsed offset is not 4111 * absolute from the start of the record. Instead, it is 4112 * relative to the *end* of the dynamic field descriptor 4113 * itself. 4114 */ 4115 offset += field->offset + field->size; 4116 } 4117 } 4118 if (out_of_bounds(field, offset, size, sample->raw_size)) 4119 return NULL; 4120 4121 return sample->raw_data + offset; 4122 } 4123 4124 void *format_field__get_raw_data(struct tep_format_field *field, struct 4125 perf_sample *sample, bool needs_swap, 4126 u16 *len_out) 4127 { 4128 int offset = field->offset; 4129 int size = field->size; 4130 4131 if (field->flags & TEP_FIELD_IS_DYNAMIC) { 4132 unsigned int dynamic_data; 4133 4134 if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) 4135 return NULL; 4136 4137 dynamic_data = format_field__intval(field, sample, needs_swap); 4138 4139 offset = dynamic_data & 0xffff; 4140 size = (dynamic_data >> 16) & 0xffff; 4141 4142 if (tep_field_is_relative(field->flags)) 4143 offset += field->offset + field->size; 4144 } 4145 4146 if (out_of_bounds(field, offset, size, sample->raw_size)) 4147 return NULL; 4148 4149 *len_out = size; 4150 return sample->raw_data + offset; 4151 } 4152 4153 unsigned long *format_field__get_cpumask(struct tep_format_field *field, 4154 struct perf_sample *sample, 4155 bool needs_swap, u16 *len_out) 4156 { 4157 u16 len; 4158 void *ptr = format_field__get_raw_data(field, sample, needs_swap, &len); 4159 unsigned long *mask; 4160 struct perf_env *env; 4161 bool target_is_64; 4162 int target_word_size; 4163 int nr_words; 4164 int bit_idx; 4165 int nbits; 4166 4167 if (!ptr) 4168 return NULL; 4169 4170 nbits = len * 8; 4171 mask = bitmap_zalloc(nbits ?: 1); 4172 if (!mask) 4173 return NULL; 4174 4175 env = evsel__env(sample->evsel); 4176 target_is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); 4177 target_word_size = target_is_64 ? 8 : 4; 4178 nr_words = len / target_word_size; 4179 4180 for (bit_idx = 0; bit_idx < nbits; bit_idx++) { 4181 int w_idx = bit_idx / (target_word_size * 8); 4182 int bit_in_word = bit_idx % (target_word_size * 8); 4183 bool set = false; 4184 4185 if (w_idx >= nr_words) 4186 break; 4187 4188 if (target_is_64) { 4189 u64 word; 4190 memcpy(&word, (unsigned char *)ptr + w_idx * 8, 8); 4191 if (needs_swap) 4192 word = bswap_64(word); 4193 set = (word & (1ULL << bit_in_word)) != 0; 4194 } else { 4195 u32 word32; 4196 memcpy(&word32, (unsigned char *)ptr + w_idx * 4, 4); 4197 if (needs_swap) 4198 word32 = bswap_32(word32); 4199 set = (word32 & (1U << bit_in_word)) != 0; 4200 } 4201 4202 if (set) { 4203 int host_w_idx = bit_idx / BITS_PER_LONG; 4204 int host_bit_in_word = bit_idx % BITS_PER_LONG; 4205 mask[host_w_idx] |= (1UL << host_bit_in_word); 4206 } 4207 } 4208 4209 *len_out = len; 4210 return mask; 4211 } 4212 4213 u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, 4214 bool needs_swap) 4215 { 4216 u64 value; 4217 void *ptr = sample->raw_data + field->offset; 4218 4219 if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) 4220 return 0; 4221 4222 switch (field->size) { 4223 case 1: 4224 return *(u8 *)ptr; 4225 case 2: 4226 value = *(u16 *)ptr; 4227 break; 4228 case 4: 4229 value = *(u32 *)ptr; 4230 break; 4231 case 8: 4232 memcpy(&value, ptr, sizeof(u64)); 4233 break; 4234 default: 4235 return 0; 4236 } 4237 4238 if (!needs_swap) 4239 return value; 4240 4241 switch (field->size) { 4242 case 2: 4243 return bswap_16(value); 4244 case 4: 4245 return bswap_32(value); 4246 case 8: 4247 return bswap_64(value); 4248 default: 4249 return 0; 4250 } 4251 4252 return 0; 4253 } 4254 4255 u64 perf_sample__intval(struct perf_sample *sample, const char *name) 4256 { 4257 struct tep_format_field *field = evsel__field(sample->evsel, name); 4258 4259 return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; 4260 } 4261 4262 u64 perf_sample__intval_common(struct perf_sample *sample, const char *name) 4263 { 4264 struct tep_format_field *field = evsel__common_field(sample->evsel, name); 4265 4266 return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; 4267 } 4268 4269 char perf_sample__taskstate(struct perf_sample *sample, const char *name) 4270 { 4271 static struct tep_format_field *prev_state_field; 4272 static const char *states; 4273 struct tep_format_field *field; 4274 unsigned long long val; 4275 unsigned int bit; 4276 char state = '?'; /* '?' denotes unknown task state */ 4277 4278 field = evsel__field(sample->evsel, name); 4279 4280 if (!field) 4281 return state; 4282 4283 if (!states || field != prev_state_field) { 4284 states = parse_task_states(field); 4285 if (!states) 4286 return state; 4287 prev_state_field = field; 4288 } 4289 4290 /* 4291 * Note since the kernel exposes TASK_REPORT_MAX to userspace 4292 * to denote the 'preempted' state, we might as welll report 4293 * 'R' for this case, which make senses to users as well. 4294 * 4295 * We can change this if we have a good reason in the future. 4296 */ 4297 val = perf_sample__intval(sample, name); 4298 bit = val ? ffs(val) : 0; 4299 state = (!bit || bit > strlen(states)) ? 'R' : states[bit-1]; 4300 return state; 4301 } 4302 #endif 4303 4304 bool evsel__fallback(struct evsel *evsel, struct target *target, int err, 4305 char *msg, size_t msgsize) 4306 { 4307 int paranoid; 4308 4309 if ((err == ENODEV || err == ENOENT || err == ENXIO) && 4310 evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) { 4311 /* 4312 * If it's the legacy hardware cycles event fails then fall back 4313 * to hrtimer based cpu-clock sw counter, which is always 4314 * available even if no PMU support. PPC returned ENXIO rather 4315 * than ENODEV or ENOENT until 2.6.37. 4316 */ 4317 evsel->pmu = perf_pmus__find_by_type(PERF_TYPE_SOFTWARE); 4318 assert(evsel->pmu); /* software is a "well-known" and can't fail PMU type. */ 4319 4320 /* Configure the event. */ 4321 evsel->core.attr.type = PERF_TYPE_SOFTWARE; 4322 evsel->core.attr.config = target__has_cpu(target) 4323 ? PERF_COUNT_SW_CPU_CLOCK 4324 : PERF_COUNT_SW_TASK_CLOCK; 4325 evsel->core.is_pmu_core = false; 4326 4327 /* Remove excludes for new event. */ 4328 if (evsel->fallenback_eacces) { 4329 evsel->core.attr.exclude_kernel = 0; 4330 evsel->core.attr.exclude_hv = 0; 4331 evsel->fallenback_eacces = false; 4332 } 4333 if (evsel->fallenback_eopnotsupp) { 4334 evsel->core.attr.exclude_guest = 0; 4335 evsel->fallenback_eopnotsupp = false; 4336 } 4337 4338 /* Name is recomputed by evsel__name. */ 4339 zfree(&evsel->name); 4340 4341 /* Log message. */ 4342 scnprintf(msg, msgsize, 4343 "The cycles event is not supported, trying to fall back to %s", 4344 evsel__name(evsel)); 4345 return true; 4346 } else if (err == EACCES && !evsel->core.attr.exclude_kernel && 4347 (paranoid = perf_event_paranoid()) > 1) { 4348 const char *name = evsel__name(evsel); 4349 char *new_name; 4350 const char *sep = ":"; 4351 4352 /* If event has exclude user then don't exclude kernel. */ 4353 if (evsel->core.attr.exclude_user) 4354 goto no_fallback; 4355 4356 /* Is there already the separator in the name. */ 4357 if (strchr(name, '/') || 4358 (strchr(name, ':') && !evsel->is_libpfm_event)) 4359 sep = ""; 4360 4361 if (asprintf(&new_name, "%s%su", name, sep) < 0) 4362 goto no_fallback; 4363 4364 free(evsel->name); 4365 evsel->name = new_name; 4366 scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying " 4367 "to fall back to excluding kernel and hypervisor " 4368 " samples", paranoid); 4369 evsel->core.attr.exclude_kernel = 1; 4370 evsel->core.attr.exclude_hv = 1; 4371 evsel->fallenback_eacces = true; 4372 return true; 4373 } else if (err == EOPNOTSUPP && !evsel->core.attr.exclude_guest && 4374 !evsel->exclude_GH) { 4375 const char *name = evsel__name(evsel); 4376 char *new_name; 4377 const char *sep = ":"; 4378 4379 /* Is there already the separator in the name. */ 4380 if (strchr(name, '/') || 4381 (strchr(name, ':') && !evsel->is_libpfm_event)) 4382 sep = ""; 4383 4384 if (asprintf(&new_name, "%s%sH", name, sep) < 0) 4385 goto no_fallback; 4386 4387 free(evsel->name); 4388 evsel->name = new_name; 4389 /* Apple M1 requires exclude_guest */ 4390 scnprintf(msg, msgsize, "Trying to fall back to excluding guest samples"); 4391 evsel->core.attr.exclude_guest = 1; 4392 evsel->fallenback_eopnotsupp = true; 4393 return true; 4394 } 4395 no_fallback: 4396 scnprintf(msg, msgsize, "No fallback found for '%s' for error %d", 4397 evsel__name(evsel), err); 4398 return false; 4399 } 4400 4401 static bool find_process(const char *name) 4402 { 4403 size_t len = strlen(name); 4404 DIR *dir; 4405 struct dirent *d; 4406 int ret = -1; 4407 4408 dir = opendir(procfs__mountpoint()); 4409 if (!dir) 4410 return false; 4411 4412 /* Walk through the directory. */ 4413 while (ret && (d = readdir(dir)) != NULL) { 4414 char path[PATH_MAX]; 4415 char *data; 4416 size_t size; 4417 4418 if ((d->d_type != DT_DIR) || 4419 !strcmp(".", d->d_name) || 4420 !strcmp("..", d->d_name)) 4421 continue; 4422 4423 scnprintf(path, sizeof(path), "%s/%s/comm", 4424 procfs__mountpoint(), d->d_name); 4425 4426 if (filename__read_str(path, &data, &size)) 4427 continue; 4428 4429 ret = strncmp(name, data, len); 4430 free(data); 4431 } 4432 4433 closedir(dir); 4434 return ret ? false : true; 4435 } 4436 4437 static int dump_perf_event_processes(const struct evsel *evsel, 4438 char *msg, size_t size) 4439 { 4440 const struct perf_event_attr *failed_attr = &evsel->core.attr; 4441 u32 target_pmu_type = evsel->pmu ? evsel->pmu->type : UINT32_MAX; 4442 DIR *proc_dir; 4443 struct dirent *proc_entry; 4444 int printed = 0; 4445 4446 proc_dir = opendir(procfs__mountpoint()); 4447 if (!proc_dir) 4448 return 0; 4449 4450 /* Walk through the /proc directory. */ 4451 while ((proc_entry = readdir(proc_dir)) != NULL) { 4452 char buf[256]; 4453 DIR *fd_dir; 4454 struct dirent *fd_entry; 4455 int fd_dir_fd; 4456 4457 if (proc_entry->d_type != DT_DIR || 4458 !isdigit(proc_entry->d_name[0]) || 4459 strlen(proc_entry->d_name) > sizeof(buf) - 4) 4460 continue; 4461 4462 scnprintf(buf, sizeof(buf), "%s/fd", proc_entry->d_name); 4463 fd_dir_fd = openat(dirfd(proc_dir), buf, O_DIRECTORY); 4464 if (fd_dir_fd == -1) 4465 continue; 4466 fd_dir = fdopendir(fd_dir_fd); 4467 if (!fd_dir) { 4468 close(fd_dir_fd); 4469 continue; 4470 } 4471 while ((fd_entry = readdir(fd_dir)) != NULL) { 4472 const char *target_lnk = "anon_inode:[perf_event]"; 4473 size_t target_lnk_len = sizeof("anon_inode:[perf_event]") - 1; 4474 char fdinfo_buf[1024]; 4475 int fdinfo_fd; 4476 ssize_t fdinfo_size; 4477 char *line; 4478 char *saveptr; 4479 u32 perf_event_type = UINT32_MAX; 4480 u32 pmu_type = UINT32_MAX; 4481 ssize_t link_size; 4482 4483 if (fd_entry->d_type != DT_LNK) 4484 continue; 4485 link_size = readlinkat(fd_dir_fd, fd_entry->d_name, buf, sizeof(buf)); 4486 if (link_size < 0) 4487 continue; 4488 /* Take care as readlink doesn't null terminate the string. */ 4489 if (link_size != (ssize_t)target_lnk_len || 4490 strncmp(buf, target_lnk, target_lnk_len)) 4491 continue; 4492 4493 /* Let's check the PMU type reserved by this process */ 4494 scnprintf(buf, sizeof(buf), "%s/fdinfo/%s", 4495 proc_entry->d_name, fd_entry->d_name); 4496 fdinfo_fd = openat(dirfd(proc_dir), buf, O_RDONLY | O_NONBLOCK); 4497 if (fdinfo_fd == -1) 4498 continue; 4499 fdinfo_size = read(fdinfo_fd, fdinfo_buf, sizeof(fdinfo_buf) - 1); 4500 close(fdinfo_fd); 4501 if (fdinfo_size < 0) 4502 continue; 4503 fdinfo_buf[fdinfo_size] = '\0'; 4504 4505 line = strtok_r(fdinfo_buf, "\n", &saveptr); 4506 while (line) { 4507 if (sscanf(line, 4508 "perf_event_attr.type:\t%u", 4509 &perf_event_type) == 1) { 4510 /* continue parsing */ 4511 } else if (sscanf(line, 4512 "pmu_type:\t%u", 4513 &pmu_type) == 1) { 4514 /* continue parsing */ 4515 } 4516 line = strtok_r(NULL, "\n", &saveptr); 4517 } 4518 4519 /* Report the process which reserves the conflicted PMU. */ 4520 /* If fdinfo does not contain PMU type, report it too. */ 4521 if (perf_event_type == failed_attr->type || 4522 pmu_type == failed_attr->type || 4523 (target_pmu_type != UINT32_MAX && 4524 pmu_type == target_pmu_type) || 4525 (perf_event_type == UINT32_MAX && 4526 pmu_type == UINT32_MAX)) { 4527 int cmdline_fd; 4528 ssize_t cmdline_size; 4529 4530 scnprintf(buf, sizeof(buf), 4531 "%s/cmdline", 4532 proc_entry->d_name); 4533 cmdline_fd = openat(dirfd(proc_dir), buf, O_RDONLY | O_NONBLOCK); 4534 if (cmdline_fd == -1) 4535 continue; 4536 cmdline_size = read(cmdline_fd, buf, sizeof(buf) - 1); 4537 close(cmdline_fd); 4538 if (cmdline_size < 0) 4539 continue; 4540 buf[cmdline_size] = '\0'; 4541 for (ssize_t i = 0; i < cmdline_size; i++) { 4542 if (buf[i] == '\0') 4543 buf[i] = ' '; 4544 else if (!isprint((unsigned char)buf[i])) 4545 buf[i] = '.'; 4546 } 4547 if (printed == 0) 4548 printed += scnprintf(msg, size, 4549 "Possible processes:\n"); 4550 4551 printed += scnprintf(msg + printed, size - printed, 4552 "%s %s\n", proc_entry->d_name, buf); 4553 break; 4554 } 4555 } 4556 closedir(fd_dir); 4557 } 4558 closedir(proc_dir); 4559 return printed; 4560 } 4561 4562 int __weak arch_evsel__open_strerror(struct evsel *evsel __maybe_unused, 4563 int err __maybe_unused, 4564 char *msg __maybe_unused, 4565 size_t size __maybe_unused) 4566 { 4567 return 0; 4568 } 4569 4570 int evsel__open_strerror(struct evsel *evsel, struct target *target, 4571 int err, char *msg, size_t size) 4572 { 4573 struct perf_pmu *pmu; 4574 int printed = 0, enforced = 0; 4575 int ret; 4576 4577 switch (err) { 4578 case EPERM: 4579 case EACCES: 4580 printed += scnprintf(msg + printed, size - printed, 4581 "Access to performance monitoring and observability operations is limited.\n"); 4582 4583 if (!sysfs__read_int("fs/selinux/enforce", &enforced)) { 4584 if (enforced) { 4585 printed += scnprintf(msg + printed, size - printed, 4586 "Enforced MAC policy settings (SELinux) can limit access to performance\n" 4587 "monitoring and observability operations. Inspect system audit records for\n" 4588 "more perf_event access control information and adjusting the policy.\n"); 4589 } 4590 } 4591 4592 if (err == EPERM) 4593 printed += scnprintf(msg, size, 4594 "No permission to enable %s event.\n\n", evsel__name(evsel)); 4595 4596 return printed + scnprintf(msg + printed, size - printed, 4597 "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n" 4598 "access to performance monitoring and observability operations for processes\n" 4599 "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n" 4600 "More information can be found at 'Perf events and tool security' document:\n" 4601 "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n" 4602 "perf_event_paranoid setting is %d:\n" 4603 " -1: Allow use of (almost) all events by all users\n" 4604 " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n" 4605 ">= 0: Disallow raw and ftrace function tracepoint access\n" 4606 ">= 1: Disallow CPU event access\n" 4607 ">= 2: Disallow kernel profiling\n" 4608 "To make the adjusted perf_event_paranoid setting permanent preserve it\n" 4609 "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)", 4610 perf_event_paranoid()); 4611 case ENOENT: 4612 return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel)); 4613 case EMFILE: 4614 return scnprintf(msg, size, "%s", 4615 "Too many events are opened.\n" 4616 "Probably the maximum number of open file descriptors has been reached.\n" 4617 "Hint: Try again after reducing the number of events.\n" 4618 "Hint: Try increasing the limit with 'ulimit -n <limit>'"); 4619 case ENOMEM: 4620 if (evsel__has_callchain(evsel) && 4621 access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0) 4622 return scnprintf(msg, size, 4623 "Not enough memory to setup event with callchain.\n" 4624 "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n" 4625 "Hint: Current value: %d", sysctl__max_stack()); 4626 break; 4627 case ENODEV: 4628 if (target->cpu_list) 4629 return scnprintf(msg, size, "%s", 4630 "No such device - did you specify an out-of-range profile CPU?"); 4631 break; 4632 case EOPNOTSUPP: 4633 if (evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK) 4634 return scnprintf(msg, size, 4635 "%s: PMU Hardware or event type doesn't support branch stack sampling.", 4636 evsel__name(evsel)); 4637 if (evsel->core.attr.aux_output) 4638 return scnprintf(msg, size, 4639 "%s: PMU Hardware doesn't support 'aux_output' feature", 4640 evsel__name(evsel)); 4641 if (evsel->core.attr.aux_action) 4642 return scnprintf(msg, size, 4643 "%s: PMU Hardware doesn't support 'aux_action' feature", 4644 evsel__name(evsel)); 4645 if (evsel->core.attr.sample_period != 0) 4646 return scnprintf(msg, size, 4647 "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'", 4648 evsel__name(evsel)); 4649 if (evsel->core.attr.precise_ip) 4650 return scnprintf(msg, size, "%s", 4651 "\'precise\' request may not be supported. Try removing 'p' modifier."); 4652 #if defined(__i386__) || defined(__x86_64__) 4653 if (evsel->core.attr.type == PERF_TYPE_HARDWARE) 4654 return scnprintf(msg, size, "%s", 4655 "No hardware sampling interrupt available.\n"); 4656 #endif 4657 if (!target__has_cpu(target)) 4658 return scnprintf(msg, size, 4659 "Unsupported event (%s) in per-thread mode, enable system wide with '-a'.", 4660 evsel__name(evsel)); 4661 break; 4662 case EBUSY: 4663 if (find_process("oprofiled")) 4664 return scnprintf(msg, size, 4665 "The PMU counters are busy/taken by another profiler.\n" 4666 "We found oprofile daemon running, please stop it and try again."); 4667 printed += scnprintf( 4668 msg, size, 4669 "The PMU %s counters are busy and in use by another process.\n", 4670 evsel->pmu ? evsel->pmu->name : ""); 4671 return printed + dump_perf_event_processes(evsel, 4672 msg + printed, 4673 size - printed); 4674 break; 4675 case EINVAL: 4676 if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size) 4677 return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel."); 4678 if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size) 4679 return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel."); 4680 if (evsel->core.attr.write_backward && perf_missing_features.write_backward) 4681 return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel."); 4682 if (perf_missing_features.clockid) 4683 return scnprintf(msg, size, "clockid feature not supported."); 4684 if (perf_missing_features.clockid_wrong) 4685 return scnprintf(msg, size, "wrong clockid (%d).", clockid); 4686 if (perf_missing_features.aux_action) 4687 return scnprintf(msg, size, "The 'aux_action' feature is not supported, update the kernel."); 4688 if (perf_missing_features.aux_output) 4689 return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel."); 4690 pmu = evsel__find_pmu(evsel); 4691 if (!pmu->is_core && !target__has_cpu(target)) 4692 return scnprintf(msg, size, 4693 "Invalid event (%s) in per-thread mode, enable system wide with '-a'.", 4694 evsel__name(evsel)); 4695 4696 break; 4697 case ENODATA: 4698 return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. " 4699 "Please add an auxiliary event in front of the load latency event."); 4700 default: 4701 break; 4702 } 4703 4704 ret = arch_evsel__open_strerror(evsel, err, msg, size); 4705 if (ret) 4706 return ret; 4707 4708 errno = err; 4709 return scnprintf(msg, size, 4710 "The sys_perf_event_open() syscall failed for event (%s): %m\n" 4711 "\"dmesg | grep -i perf\" may provide additional information.\n", 4712 evsel__name(evsel)); 4713 } 4714 4715 struct perf_session *evsel__session(struct evsel *evsel) 4716 { 4717 return evsel && evsel->evlist ? evlist__session(evsel->evlist) : NULL; 4718 } 4719 4720 struct perf_env *evsel__env(struct evsel *evsel) 4721 { 4722 struct perf_session *session = evsel__session(evsel); 4723 4724 return session ? perf_session__env(session) : NULL; 4725 } 4726 4727 static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist) 4728 { 4729 int cpu_map_idx, thread; 4730 4731 if (evsel__is_retire_lat(evsel)) 4732 return 0; 4733 4734 if (perf_pmu__kind(evsel->pmu) != PERF_PMU_KIND_PE) 4735 return 0; 4736 4737 for (cpu_map_idx = 0; cpu_map_idx < xyarray__max_x(evsel->core.fd); cpu_map_idx++) { 4738 for (thread = 0; thread < xyarray__max_y(evsel->core.fd); 4739 thread++) { 4740 int fd = FD(evsel, cpu_map_idx, thread); 4741 4742 if (perf_evlist__id_add_fd(evlist__core(evlist), &evsel->core, 4743 cpu_map_idx, thread, fd) < 0) 4744 return -1; 4745 } 4746 } 4747 4748 return 0; 4749 } 4750 4751 int evsel__store_ids(struct evsel *evsel, struct evlist *evlist) 4752 { 4753 struct perf_cpu_map *cpus = evsel->core.cpus; 4754 struct perf_thread_map *threads = evsel->core.threads; 4755 4756 if (perf_evsel__alloc_id(&evsel->core, perf_cpu_map__nr(cpus), threads->nr)) 4757 return -ENOMEM; 4758 4759 return store_evsel_ids(evsel, evlist); 4760 } 4761 4762 void evsel__zero_per_pkg(struct evsel *evsel) 4763 { 4764 struct hashmap_entry *cur; 4765 size_t bkt; 4766 4767 if (evsel->per_pkg_mask) { 4768 hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt) 4769 zfree(&cur->pkey); 4770 4771 hashmap__clear(evsel->per_pkg_mask); 4772 } 4773 } 4774 4775 /** 4776 * evsel__is_hybrid - does the evsel have a known PMU that is hybrid. Note, this 4777 * will be false on hybrid systems for hardware and legacy 4778 * cache events. 4779 */ 4780 bool evsel__is_hybrid(const struct evsel *evsel) 4781 { 4782 if (!evsel->core.is_pmu_core) 4783 return false; 4784 4785 return perf_pmus__num_core_pmus() > 1; 4786 } 4787 4788 struct evsel *evsel__leader(const struct evsel *evsel) 4789 { 4790 if (evsel->core.leader == NULL) 4791 return NULL; 4792 return container_of(evsel->core.leader, struct evsel, core); 4793 } 4794 4795 bool evsel__has_leader(struct evsel *evsel, struct evsel *leader) 4796 { 4797 return evsel->core.leader == &leader->core; 4798 } 4799 4800 bool evsel__is_leader(struct evsel *evsel) 4801 { 4802 return evsel__has_leader(evsel, evsel); 4803 } 4804 4805 void evsel__set_leader(struct evsel *evsel, struct evsel *leader) 4806 { 4807 evsel->core.leader = &leader->core; 4808 } 4809 4810 bool evsel__is_aux_event(const struct evsel *evsel) 4811 { 4812 struct perf_pmu *pmu; 4813 4814 if (evsel->needs_auxtrace_mmap) 4815 return true; 4816 4817 pmu = evsel__find_pmu(evsel); 4818 return pmu && pmu->auxtrace; 4819 } 4820 4821 int evsel__source_count(const struct evsel *evsel) 4822 { 4823 struct evsel *pos; 4824 int count = 0; 4825 4826 evlist__for_each_entry(evsel->evlist, pos) { 4827 if (pos->metric_leader == evsel) 4828 count++; 4829 } 4830 return count; 4831 } 4832 4833 bool __weak arch_evsel__must_be_in_group(const struct evsel *evsel __maybe_unused) 4834 { 4835 return false; 4836 } 4837 4838 /* 4839 * Remove an event from a given group (leader). 4840 * Some events, e.g., perf metrics Topdown events, 4841 * must always be grouped. Ignore the events. 4842 */ 4843 void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader) 4844 { 4845 if (!arch_evsel__must_be_in_group(evsel) && evsel != leader) { 4846 evsel__set_leader(evsel, evsel); 4847 evsel->core.nr_members = 0; 4848 leader->core.nr_members--; 4849 } 4850 } 4851 4852 bool evsel__set_needs_uniquify(struct evsel *counter, const struct perf_stat_config *config) 4853 { 4854 struct evsel *evsel; 4855 4856 if (counter->needs_uniquify) { 4857 /* Already set. */ 4858 return true; 4859 } 4860 4861 if (counter->use_config_name || counter->is_libpfm_event) { 4862 /* Original name will be used. */ 4863 return false; 4864 } 4865 4866 if (!config->hybrid_merge && evsel__is_hybrid(counter)) { 4867 /* Unique hybrid counters necessary. */ 4868 counter->needs_uniquify = true; 4869 return true; 4870 } 4871 4872 if (counter->core.attr.type < PERF_TYPE_MAX && counter->core.attr.type != PERF_TYPE_RAW) { 4873 /* Legacy event, don't uniquify. */ 4874 return false; 4875 } 4876 4877 if (counter->pmu && counter->pmu->is_core && 4878 counter->alternate_hw_config != PERF_COUNT_HW_MAX) { 4879 /* A sysfs or json event replacing a legacy event, don't uniquify. */ 4880 return false; 4881 } 4882 4883 if (config->aggr_mode == AGGR_NONE) { 4884 /* Always unique with no aggregation. */ 4885 counter->needs_uniquify = true; 4886 return true; 4887 } 4888 4889 if (counter->first_wildcard_match != NULL) { 4890 /* 4891 * If stats are merged then only the first_wildcard_match is 4892 * displayed, there is no need to uniquify this evsel as the 4893 * name won't be shown. 4894 */ 4895 return false; 4896 } 4897 4898 /* 4899 * Do other non-merged events in the evlist have the same name? If so 4900 * uniquify is necessary. 4901 */ 4902 evlist__for_each_entry(counter->evlist, evsel) { 4903 if (evsel == counter || evsel->first_wildcard_match || evsel->pmu == counter->pmu) 4904 continue; 4905 4906 if (evsel__name_is(counter, evsel__name(evsel))) { 4907 counter->needs_uniquify = true; 4908 return true; 4909 } 4910 } 4911 return false; 4912 } 4913 4914 void evsel__uniquify_counter(struct evsel *counter) 4915 { 4916 const char *name, *pmu_name, *config; 4917 char *new_name; 4918 int len, ret; 4919 4920 /* No uniquification necessary. */ 4921 if (!counter->needs_uniquify) 4922 return; 4923 4924 /* The evsel was already uniquified. */ 4925 if (counter->uniquified_name) 4926 return; 4927 4928 /* Avoid checking to uniquify twice. */ 4929 counter->uniquified_name = true; 4930 4931 name = evsel__name(counter); 4932 config = strchr(name, '/'); 4933 pmu_name = counter->pmu->name; 4934 4935 /* Already prefixed by the PMU name? */ 4936 len = pmu_name_len_no_suffix(pmu_name); 4937 4938 if (!strncmp(name, pmu_name, len)) { 4939 /* 4940 * If the PMU name is there, then there is no sense in not 4941 * having a slash. Do this for robustness. 4942 */ 4943 if (config == NULL) 4944 config = name - 1; 4945 4946 ret = asprintf(&new_name, "%s/%s", pmu_name, config + 1); 4947 } else if (config) { 4948 len = config - name; 4949 if (config[1] == '/') { 4950 /* case: event// */ 4951 ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 2); 4952 } else { 4953 /* case: event/.../ */ 4954 ret = asprintf(&new_name, "%s/%.*s,%s", pmu_name, len, name, config + 1); 4955 } 4956 } else { 4957 config = strchr(name, ':'); 4958 if (config) { 4959 /* case: event:.. */ 4960 len = config - name; 4961 4962 ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 1); 4963 } else { 4964 /* case: event */ 4965 ret = asprintf(&new_name, "%s/%s/", pmu_name, name); 4966 } 4967 } 4968 if (ret > 0) { 4969 free(counter->name); 4970 counter->name = new_name; 4971 } else { 4972 /* ENOMEM from asprintf. */ 4973 counter->uniquified_name = false; 4974 } 4975 } 4976 4977 void evsel__warn_user_requested_cpus(struct evsel *evsel, struct perf_cpu_map *user_requested_cpus) 4978 { 4979 struct perf_cpu_map *intersect, *online = NULL; 4980 const struct perf_pmu *pmu = evsel__find_pmu(evsel); 4981 4982 if (pmu && pmu->is_core) { 4983 intersect = perf_cpu_map__intersect(pmu->cpus, user_requested_cpus); 4984 } else { 4985 online = cpu_map__online(); 4986 intersect = perf_cpu_map__intersect(online, user_requested_cpus); 4987 } 4988 if (!perf_cpu_map__equal(intersect, user_requested_cpus)) { 4989 char buf1[128]; 4990 char buf2[128]; 4991 4992 cpu_map__snprint(user_requested_cpus, buf1, sizeof(buf1)); 4993 cpu_map__snprint(online ?: pmu->cpus, buf2, sizeof(buf2)); 4994 pr_warning("WARNING: A requested CPU in '%s' is not supported by PMU '%s' (CPUs %s) for event '%s'\n", 4995 buf1, pmu ? pmu->name : "cpu", buf2, evsel__name(evsel)); 4996 } 4997 perf_cpu_map__put(intersect); 4998 perf_cpu_map__put(online); 4999 } 5000