1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <unistd.h> 4 #include <sys/syscall.h> 5 #include <perf/evsel.h> 6 #include <perf/cpumap.h> 7 #include <perf/threadmap.h> 8 #include <linux/hash.h> 9 #include <linux/list.h> 10 #include <internal/evsel.h> 11 #include <linux/zalloc.h> 12 #include <stdlib.h> 13 #include <internal/xyarray.h> 14 #include <internal/cpumap.h> 15 #include <internal/mmap.h> 16 #include <internal/threadmap.h> 17 #include <internal/lib.h> 18 #include <linux/string.h> 19 #include <sys/ioctl.h> 20 #include <sys/mman.h> 21 #include <asm/bug.h> 22 23 void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, 24 int idx) 25 { 26 INIT_LIST_HEAD(&evsel->node); 27 INIT_LIST_HEAD(&evsel->per_stream_periods); 28 evsel->attr = *attr; 29 evsel->idx = idx; 30 evsel->leader = evsel; 31 } 32 33 struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr) 34 { 35 struct perf_evsel *evsel = zalloc(sizeof(*evsel)); 36 37 if (evsel != NULL) 38 perf_evsel__init(evsel, attr, 0); 39 40 return evsel; 41 } 42 43 void perf_evsel__exit(struct perf_evsel *evsel) 44 { 45 assert(evsel->fd == NULL); /* If not fds were not closed. */ 46 assert(evsel->mmap == NULL); /* If not munmap wasn't called. */ 47 assert(evsel->sample_id == NULL); /* If not free_id wasn't called. */ 48 perf_cpu_map__put(evsel->cpus); 49 perf_cpu_map__put(evsel->pmu_cpus); 50 perf_thread_map__put(evsel->threads); 51 } 52 53 void perf_evsel__delete(struct perf_evsel *evsel) 54 { 55 perf_evsel__exit(evsel); 56 free(evsel); 57 } 58 59 #define FD(_evsel, _cpu_map_idx, _thread) \ 60 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread)) 61 #define MMAP(_evsel, _cpu_map_idx, _thread) \ 62 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \ 63 : NULL) 64 65 int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads) 66 { 67 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int)); 68 69 if (evsel->fd) { 70 int idx, thread; 71 72 for (idx = 0; idx < ncpus; idx++) { 73 for (thread = 0; thread < nthreads; thread++) { 74 int *fd = FD(evsel, idx, thread); 75 76 if (fd) 77 *fd = -1; 78 } 79 } 80 } 81 82 return evsel->fd != NULL ? 0 : -ENOMEM; 83 } 84 85 static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads) 86 { 87 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap)); 88 89 return evsel->mmap != NULL ? 0 : -ENOMEM; 90 } 91 92 static int 93 sys_perf_event_open(struct perf_event_attr *attr, 94 pid_t pid, struct perf_cpu cpu, int group_fd, 95 unsigned long flags) 96 { 97 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags); 98 } 99 100 static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd) 101 { 102 struct perf_evsel *leader = evsel->leader; 103 int *fd; 104 105 if (evsel == leader) { 106 *group_fd = -1; 107 return 0; 108 } 109 110 /* 111 * Leader must be already processed/open, 112 * if not it's a bug. 113 */ 114 if (!leader->fd) 115 return -ENOTCONN; 116 117 fd = FD(leader, cpu_map_idx, thread); 118 if (fd == NULL || *fd == -1) 119 return -EBADF; 120 121 *group_fd = *fd; 122 123 return 0; 124 } 125 126 int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, 127 struct perf_thread_map *threads) 128 { 129 struct perf_cpu cpu; 130 unsigned int idx; 131 int thread, err = 0; 132 133 if (cpus == NULL) { 134 static struct perf_cpu_map *empty_cpu_map; 135 136 if (empty_cpu_map == NULL) { 137 empty_cpu_map = perf_cpu_map__new_any_cpu(); 138 if (empty_cpu_map == NULL) 139 return -ENOMEM; 140 } 141 142 cpus = empty_cpu_map; 143 } 144 145 if (threads == NULL) { 146 static struct perf_thread_map *empty_thread_map; 147 148 if (empty_thread_map == NULL) { 149 empty_thread_map = perf_thread_map__new_dummy(); 150 if (empty_thread_map == NULL) 151 return -ENOMEM; 152 } 153 154 threads = empty_thread_map; 155 } 156 157 if (evsel->fd == NULL && 158 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 159 return -ENOMEM; 160 161 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 162 for (thread = 0; thread < threads->nr; thread++) { 163 int fd, group_fd, *evsel_fd; 164 165 evsel_fd = FD(evsel, idx, thread); 166 if (evsel_fd == NULL) { 167 err = -EINVAL; 168 goto out; 169 } 170 171 err = get_group_fd(evsel, idx, thread, &group_fd); 172 if (err < 0) 173 goto out; 174 175 fd = sys_perf_event_open(&evsel->attr, 176 threads->map[thread].pid, 177 cpu, group_fd, 0); 178 179 if (fd < 0) { 180 err = -errno; 181 goto out; 182 } 183 184 *evsel_fd = fd; 185 } 186 } 187 out: 188 if (err) 189 perf_evsel__close(evsel); 190 191 return err; 192 } 193 194 static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx) 195 { 196 int thread; 197 198 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) { 199 int *fd = FD(evsel, cpu_map_idx, thread); 200 201 if (fd && *fd >= 0) { 202 close(*fd); 203 *fd = -1; 204 } 205 } 206 } 207 208 void perf_evsel__close_fd(struct perf_evsel *evsel) 209 { 210 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++) 211 perf_evsel__close_fd_cpu(evsel, idx); 212 } 213 214 void perf_evsel__free_fd(struct perf_evsel *evsel) 215 { 216 xyarray__delete(evsel->fd); 217 evsel->fd = NULL; 218 } 219 220 void perf_evsel__close(struct perf_evsel *evsel) 221 { 222 if (evsel->fd == NULL) 223 return; 224 225 perf_evsel__close_fd(evsel); 226 perf_evsel__free_fd(evsel); 227 } 228 229 void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx) 230 { 231 if (evsel->fd == NULL) 232 return; 233 234 perf_evsel__close_fd_cpu(evsel, cpu_map_idx); 235 } 236 237 void perf_evsel__munmap(struct perf_evsel *evsel) 238 { 239 int idx, thread; 240 241 if (evsel->fd == NULL || evsel->mmap == NULL) 242 return; 243 244 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 245 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 246 int *fd = FD(evsel, idx, thread); 247 248 if (fd == NULL || *fd < 0) 249 continue; 250 251 perf_mmap__munmap(MMAP(evsel, idx, thread)); 252 } 253 } 254 255 xyarray__delete(evsel->mmap); 256 evsel->mmap = NULL; 257 } 258 259 int perf_evsel__mmap(struct perf_evsel *evsel, int pages) 260 { 261 int ret, idx, thread; 262 struct perf_mmap_param mp = { 263 .prot = PROT_READ | PROT_WRITE, 264 .mask = (pages * page_size) - 1, 265 }; 266 267 if (evsel->fd == NULL || evsel->mmap) 268 return -EINVAL; 269 270 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0) 271 return -ENOMEM; 272 273 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 274 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 275 int *fd = FD(evsel, idx, thread); 276 struct perf_mmap *map; 277 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx); 278 279 if (fd == NULL || *fd < 0) 280 continue; 281 282 map = MMAP(evsel, idx, thread); 283 perf_mmap__init(map, NULL, false, NULL); 284 285 ret = perf_mmap__mmap(map, &mp, *fd, cpu); 286 if (ret) { 287 perf_evsel__munmap(evsel); 288 return ret; 289 } 290 } 291 } 292 293 return 0; 294 } 295 296 void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread) 297 { 298 int *fd = FD(evsel, cpu_map_idx, thread); 299 300 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL) 301 return NULL; 302 303 return MMAP(evsel, cpu_map_idx, thread)->base; 304 } 305 306 int perf_evsel__read_size(struct perf_evsel *evsel) 307 { 308 u64 read_format = evsel->attr.read_format; 309 int entry = sizeof(u64); /* value */ 310 int size = 0; 311 int nr = 1; 312 313 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 314 size += sizeof(u64); 315 316 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 317 size += sizeof(u64); 318 319 if (read_format & PERF_FORMAT_ID) 320 entry += sizeof(u64); 321 322 if (read_format & PERF_FORMAT_LOST) 323 entry += sizeof(u64); 324 325 if (read_format & PERF_FORMAT_GROUP) { 326 nr = evsel->nr_members; 327 size += sizeof(u64); 328 } 329 330 size += entry * nr; 331 return size; 332 } 333 334 /* This only reads values for the leader */ 335 static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, 336 int thread, struct perf_counts_values *count) 337 { 338 size_t size = perf_evsel__read_size(evsel); 339 int *fd = FD(evsel, cpu_map_idx, thread); 340 u64 read_format = evsel->attr.read_format; 341 u64 *data; 342 int idx = 1; 343 344 if (fd == NULL || *fd < 0) 345 return -EINVAL; 346 347 data = calloc(1, size); 348 if (data == NULL) 349 return -ENOMEM; 350 351 if (readn(*fd, data, size) <= 0) { 352 free(data); 353 return -errno; 354 } 355 356 /* 357 * This reads only the leader event intentionally since we don't have 358 * perf counts values for sibling events. 359 */ 360 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 361 count->ena = data[idx++]; 362 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 363 count->run = data[idx++]; 364 365 /* value is always available */ 366 count->val = data[idx++]; 367 if (read_format & PERF_FORMAT_ID) 368 count->id = data[idx++]; 369 if (read_format & PERF_FORMAT_LOST) 370 count->lost = data[idx++]; 371 372 free(data); 373 return 0; 374 } 375 376 /* 377 * The perf read format is very flexible. It needs to set the proper 378 * values according to the read format. 379 */ 380 static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, 381 struct perf_counts_values *count) 382 { 383 u64 read_format = evsel->attr.read_format; 384 int n = 0; 385 386 count->val = buf[n++]; 387 388 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 389 count->ena = buf[n++]; 390 391 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 392 count->run = buf[n++]; 393 394 if (read_format & PERF_FORMAT_ID) 395 count->id = buf[n++]; 396 397 if (read_format & PERF_FORMAT_LOST) 398 count->lost = buf[n++]; 399 } 400 401 int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, 402 struct perf_counts_values *count) 403 { 404 size_t size = perf_evsel__read_size(evsel); 405 int *fd = FD(evsel, cpu_map_idx, thread); 406 u64 read_format = evsel->attr.read_format; 407 struct perf_counts_values buf; 408 409 memset(count, 0, sizeof(*count)); 410 411 if (fd == NULL || *fd < 0) 412 return -EINVAL; 413 414 if (read_format & PERF_FORMAT_GROUP) 415 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); 416 417 if (MMAP(evsel, cpu_map_idx, thread) && 418 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && 419 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) 420 return 0; 421 422 if (readn(*fd, buf.values, size) <= 0) 423 return -errno; 424 425 perf_evsel__adjust_values(evsel, buf.values, count); 426 return 0; 427 } 428 429 static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, 430 int cpu_map_idx, int thread) 431 { 432 int *fd = FD(evsel, cpu_map_idx, thread); 433 434 if (fd == NULL || *fd < 0) 435 return -1; 436 437 return ioctl(*fd, ioc, arg); 438 } 439 440 static int perf_evsel__run_ioctl(struct perf_evsel *evsel, 441 int ioc, void *arg, 442 int cpu_map_idx) 443 { 444 int thread; 445 446 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 447 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); 448 449 if (err) 450 return err; 451 } 452 453 return 0; 454 } 455 456 int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 457 { 458 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); 459 } 460 461 int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) 462 { 463 struct perf_cpu cpu __maybe_unused; 464 unsigned int idx; 465 int err; 466 467 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { 468 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread); 469 if (err) 470 return err; 471 } 472 473 return 0; 474 } 475 476 int perf_evsel__enable(struct perf_evsel *evsel) 477 { 478 int i; 479 int err = 0; 480 481 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 482 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i); 483 return err; 484 } 485 486 int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 487 { 488 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx); 489 } 490 491 int perf_evsel__disable(struct perf_evsel *evsel) 492 { 493 int i; 494 int err = 0; 495 496 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 497 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i); 498 return err; 499 } 500 501 int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) 502 { 503 int err = 0; 504 505 for (unsigned int i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) { 506 err = perf_evsel__run_ioctl(evsel, 507 PERF_EVENT_IOC_SET_FILTER, 508 (void *)filter, i); 509 } 510 return err; 511 } 512 513 struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel) 514 { 515 return evsel->cpus; 516 } 517 518 struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel) 519 { 520 return evsel->threads; 521 } 522 523 struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel) 524 { 525 return &evsel->attr; 526 } 527 528 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) 529 { 530 if (ncpus == 0 || nthreads == 0) 531 return 0; 532 533 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); 534 if (evsel->sample_id == NULL) 535 return -ENOMEM; 536 537 evsel->id = zalloc(ncpus * nthreads * sizeof(u64)); 538 if (evsel->id == NULL) { 539 xyarray__delete(evsel->sample_id); 540 evsel->sample_id = NULL; 541 return -ENOMEM; 542 } 543 544 return 0; 545 } 546 547 void perf_evsel__free_id(struct perf_evsel *evsel) 548 { 549 struct perf_sample_id_period *pos, *n; 550 551 xyarray__delete(evsel->sample_id); 552 evsel->sample_id = NULL; 553 zfree(&evsel->id); 554 evsel->ids = 0; 555 556 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) { 557 list_del_init(&pos->node); 558 free(pos); 559 } 560 } 561 562 bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel) 563 { 564 return (evsel->attr.sample_type & PERF_SAMPLE_READ) && 565 (evsel->attr.sample_type & PERF_SAMPLE_TID) && 566 evsel->attr.inherit; 567 } 568 569 u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread) 570 { 571 struct hlist_head *head; 572 struct perf_sample_id_period *res; 573 int hash; 574 575 if (!per_thread) 576 return &sid->period; 577 578 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS); 579 head = &sid->periods[hash]; 580 581 hlist_for_each_entry(res, head, hnode) 582 if (res->tid == tid) 583 return &res->period; 584 585 if (sid->evsel == NULL) 586 return NULL; 587 588 res = zalloc(sizeof(struct perf_sample_id_period)); 589 if (res == NULL) 590 return NULL; 591 592 INIT_LIST_HEAD(&res->node); 593 res->tid = tid; 594 595 list_add_tail(&res->node, &sid->evsel->per_stream_periods); 596 hlist_add_head(&res->hnode, &sid->periods[hash]); 597 598 return &res->period; 599 } 600 601 void perf_counts_values__scale(struct perf_counts_values *count, 602 bool scale, __s8 *pscaled) 603 { 604 s8 scaled = 0; 605 606 if (scale) { 607 if (count->run == 0) { 608 scaled = -1; 609 count->val = 0; 610 } else if (count->run < count->ena) { 611 scaled = 1; 612 count->val = (u64)((double)count->val * count->ena / count->run); 613 } 614 } 615 616 if (pscaled) 617 *pscaled = scaled; 618 } 619