1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <unistd.h> 4 #include <sys/syscall.h> 5 #include <perf/evsel.h> 6 #include <perf/cpumap.h> 7 #include <perf/threadmap.h> 8 #include <linux/hash.h> 9 #include <linux/list.h> 10 #include <internal/evsel.h> 11 #include <linux/zalloc.h> 12 #include <stdlib.h> 13 #include <internal/xyarray.h> 14 #include <internal/cpumap.h> 15 #include <internal/mmap.h> 16 #include <internal/threadmap.h> 17 #include <internal/lib.h> 18 #include <linux/string.h> 19 #include <sys/ioctl.h> 20 #include <sys/mman.h> 21 #include <asm/bug.h> 22 23 void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, 24 int idx) 25 { 26 INIT_LIST_HEAD(&evsel->node); 27 INIT_LIST_HEAD(&evsel->per_stream_periods); 28 evsel->attr = *attr; 29 evsel->idx = idx; 30 evsel->leader = evsel; 31 } 32 33 struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr) 34 { 35 struct perf_evsel *evsel = zalloc(sizeof(*evsel)); 36 37 if (evsel != NULL) 38 perf_evsel__init(evsel, attr, 0); 39 40 return evsel; 41 } 42 43 void perf_evsel__exit(struct perf_evsel *evsel) 44 { 45 assert(evsel->fd == NULL); /* If not fds were not closed. */ 46 assert(evsel->mmap == NULL); /* If not munmap wasn't called. */ 47 assert(evsel->sample_id == NULL); /* If not free_id wasn't called. */ 48 perf_cpu_map__put(evsel->cpus); 49 perf_cpu_map__put(evsel->pmu_cpus); 50 perf_thread_map__put(evsel->threads); 51 } 52 53 void perf_evsel__delete(struct perf_evsel *evsel) 54 { 55 perf_evsel__exit(evsel); 56 free(evsel); 57 } 58 59 #define FD(_evsel, _cpu_map_idx, _thread) \ 60 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread)) 61 #define MMAP(_evsel, _cpu_map_idx, _thread) \ 62 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \ 63 : NULL) 64 65 int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads) 66 { 67 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int)); 68 69 if (evsel->fd) { 70 int idx, thread; 71 72 for (idx = 0; idx < ncpus; idx++) { 73 for (thread = 0; thread < nthreads; thread++) { 74 int *fd = FD(evsel, idx, thread); 75 76 if (fd) 77 *fd = -1; 78 } 79 } 80 } 81 82 return evsel->fd != NULL ? 0 : -ENOMEM; 83 } 84 85 static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads) 86 { 87 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap)); 88 89 return evsel->mmap != NULL ? 0 : -ENOMEM; 90 } 91 92 static int 93 sys_perf_event_open(struct perf_event_attr *attr, 94 pid_t pid, struct perf_cpu cpu, int group_fd, 95 unsigned long flags) 96 { 97 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags); 98 } 99 100 static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd) 101 { 102 struct perf_evsel *leader = evsel->leader; 103 int *fd; 104 105 if (evsel == leader) { 106 *group_fd = -1; 107 return 0; 108 } 109 110 /* 111 * Leader must be already processed/open, 112 * if not it's a bug. 113 */ 114 if (!leader->fd) 115 return -ENOTCONN; 116 117 fd = FD(leader, cpu_map_idx, thread); 118 if (fd == NULL || *fd == -1) 119 return -EBADF; 120 121 *group_fd = *fd; 122 123 return 0; 124 } 125 126 int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, 127 struct perf_thread_map *threads) 128 { 129 struct perf_cpu cpu; 130 int idx, thread, err = 0; 131 132 if (cpus == NULL) { 133 static struct perf_cpu_map *empty_cpu_map; 134 135 if (empty_cpu_map == NULL) { 136 empty_cpu_map = perf_cpu_map__new_any_cpu(); 137 if (empty_cpu_map == NULL) 138 return -ENOMEM; 139 } 140 141 cpus = empty_cpu_map; 142 } 143 144 if (threads == NULL) { 145 static struct perf_thread_map *empty_thread_map; 146 147 if (empty_thread_map == NULL) { 148 empty_thread_map = perf_thread_map__new_dummy(); 149 if (empty_thread_map == NULL) 150 return -ENOMEM; 151 } 152 153 threads = empty_thread_map; 154 } 155 156 if (evsel->fd == NULL && 157 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 158 return -ENOMEM; 159 160 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 161 for (thread = 0; thread < threads->nr; thread++) { 162 int fd, group_fd, *evsel_fd; 163 164 evsel_fd = FD(evsel, idx, thread); 165 if (evsel_fd == NULL) { 166 err = -EINVAL; 167 goto out; 168 } 169 170 err = get_group_fd(evsel, idx, thread, &group_fd); 171 if (err < 0) 172 goto out; 173 174 fd = sys_perf_event_open(&evsel->attr, 175 threads->map[thread].pid, 176 cpu, group_fd, 0); 177 178 if (fd < 0) { 179 err = -errno; 180 goto out; 181 } 182 183 *evsel_fd = fd; 184 } 185 } 186 out: 187 if (err) 188 perf_evsel__close(evsel); 189 190 return err; 191 } 192 193 static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx) 194 { 195 int thread; 196 197 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) { 198 int *fd = FD(evsel, cpu_map_idx, thread); 199 200 if (fd && *fd >= 0) { 201 close(*fd); 202 *fd = -1; 203 } 204 } 205 } 206 207 void perf_evsel__close_fd(struct perf_evsel *evsel) 208 { 209 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++) 210 perf_evsel__close_fd_cpu(evsel, idx); 211 } 212 213 void perf_evsel__free_fd(struct perf_evsel *evsel) 214 { 215 xyarray__delete(evsel->fd); 216 evsel->fd = NULL; 217 } 218 219 void perf_evsel__close(struct perf_evsel *evsel) 220 { 221 if (evsel->fd == NULL) 222 return; 223 224 perf_evsel__close_fd(evsel); 225 perf_evsel__free_fd(evsel); 226 } 227 228 void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx) 229 { 230 if (evsel->fd == NULL) 231 return; 232 233 perf_evsel__close_fd_cpu(evsel, cpu_map_idx); 234 } 235 236 void perf_evsel__munmap(struct perf_evsel *evsel) 237 { 238 int idx, thread; 239 240 if (evsel->fd == NULL || evsel->mmap == NULL) 241 return; 242 243 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 244 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 245 int *fd = FD(evsel, idx, thread); 246 247 if (fd == NULL || *fd < 0) 248 continue; 249 250 perf_mmap__munmap(MMAP(evsel, idx, thread)); 251 } 252 } 253 254 xyarray__delete(evsel->mmap); 255 evsel->mmap = NULL; 256 } 257 258 int perf_evsel__mmap(struct perf_evsel *evsel, int pages) 259 { 260 int ret, idx, thread; 261 struct perf_mmap_param mp = { 262 .prot = PROT_READ | PROT_WRITE, 263 .mask = (pages * page_size) - 1, 264 }; 265 266 if (evsel->fd == NULL || evsel->mmap) 267 return -EINVAL; 268 269 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0) 270 return -ENOMEM; 271 272 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 273 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 274 int *fd = FD(evsel, idx, thread); 275 struct perf_mmap *map; 276 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx); 277 278 if (fd == NULL || *fd < 0) 279 continue; 280 281 map = MMAP(evsel, idx, thread); 282 perf_mmap__init(map, NULL, false, NULL); 283 284 ret = perf_mmap__mmap(map, &mp, *fd, cpu); 285 if (ret) { 286 perf_evsel__munmap(evsel); 287 return ret; 288 } 289 } 290 } 291 292 return 0; 293 } 294 295 void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread) 296 { 297 int *fd = FD(evsel, cpu_map_idx, thread); 298 299 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL) 300 return NULL; 301 302 return MMAP(evsel, cpu_map_idx, thread)->base; 303 } 304 305 int perf_evsel__read_size(struct perf_evsel *evsel) 306 { 307 u64 read_format = evsel->attr.read_format; 308 int entry = sizeof(u64); /* value */ 309 int size = 0; 310 int nr = 1; 311 312 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 313 size += sizeof(u64); 314 315 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 316 size += sizeof(u64); 317 318 if (read_format & PERF_FORMAT_ID) 319 entry += sizeof(u64); 320 321 if (read_format & PERF_FORMAT_LOST) 322 entry += sizeof(u64); 323 324 if (read_format & PERF_FORMAT_GROUP) { 325 nr = evsel->nr_members; 326 size += sizeof(u64); 327 } 328 329 size += entry * nr; 330 return size; 331 } 332 333 /* This only reads values for the leader */ 334 static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, 335 int thread, struct perf_counts_values *count) 336 { 337 size_t size = perf_evsel__read_size(evsel); 338 int *fd = FD(evsel, cpu_map_idx, thread); 339 u64 read_format = evsel->attr.read_format; 340 u64 *data; 341 int idx = 1; 342 343 if (fd == NULL || *fd < 0) 344 return -EINVAL; 345 346 data = calloc(1, size); 347 if (data == NULL) 348 return -ENOMEM; 349 350 if (readn(*fd, data, size) <= 0) { 351 free(data); 352 return -errno; 353 } 354 355 /* 356 * This reads only the leader event intentionally since we don't have 357 * perf counts values for sibling events. 358 */ 359 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 360 count->ena = data[idx++]; 361 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 362 count->run = data[idx++]; 363 364 /* value is always available */ 365 count->val = data[idx++]; 366 if (read_format & PERF_FORMAT_ID) 367 count->id = data[idx++]; 368 if (read_format & PERF_FORMAT_LOST) 369 count->lost = data[idx++]; 370 371 free(data); 372 return 0; 373 } 374 375 /* 376 * The perf read format is very flexible. It needs to set the proper 377 * values according to the read format. 378 */ 379 static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, 380 struct perf_counts_values *count) 381 { 382 u64 read_format = evsel->attr.read_format; 383 int n = 0; 384 385 count->val = buf[n++]; 386 387 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 388 count->ena = buf[n++]; 389 390 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 391 count->run = buf[n++]; 392 393 if (read_format & PERF_FORMAT_ID) 394 count->id = buf[n++]; 395 396 if (read_format & PERF_FORMAT_LOST) 397 count->lost = buf[n++]; 398 } 399 400 int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, 401 struct perf_counts_values *count) 402 { 403 size_t size = perf_evsel__read_size(evsel); 404 int *fd = FD(evsel, cpu_map_idx, thread); 405 u64 read_format = evsel->attr.read_format; 406 struct perf_counts_values buf; 407 408 memset(count, 0, sizeof(*count)); 409 410 if (fd == NULL || *fd < 0) 411 return -EINVAL; 412 413 if (read_format & PERF_FORMAT_GROUP) 414 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); 415 416 if (MMAP(evsel, cpu_map_idx, thread) && 417 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && 418 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) 419 return 0; 420 421 if (readn(*fd, buf.values, size) <= 0) 422 return -errno; 423 424 perf_evsel__adjust_values(evsel, buf.values, count); 425 return 0; 426 } 427 428 static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, 429 int cpu_map_idx, int thread) 430 { 431 int *fd = FD(evsel, cpu_map_idx, thread); 432 433 if (fd == NULL || *fd < 0) 434 return -1; 435 436 return ioctl(*fd, ioc, arg); 437 } 438 439 static int perf_evsel__run_ioctl(struct perf_evsel *evsel, 440 int ioc, void *arg, 441 int cpu_map_idx) 442 { 443 int thread; 444 445 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 446 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); 447 448 if (err) 449 return err; 450 } 451 452 return 0; 453 } 454 455 int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 456 { 457 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); 458 } 459 460 int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) 461 { 462 struct perf_cpu cpu __maybe_unused; 463 int idx; 464 int err; 465 466 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { 467 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread); 468 if (err) 469 return err; 470 } 471 472 return 0; 473 } 474 475 int perf_evsel__enable(struct perf_evsel *evsel) 476 { 477 int i; 478 int err = 0; 479 480 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 481 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i); 482 return err; 483 } 484 485 int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 486 { 487 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx); 488 } 489 490 int perf_evsel__disable(struct perf_evsel *evsel) 491 { 492 int i; 493 int err = 0; 494 495 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 496 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i); 497 return err; 498 } 499 500 int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) 501 { 502 int err = 0, i; 503 504 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) 505 err = perf_evsel__run_ioctl(evsel, 506 PERF_EVENT_IOC_SET_FILTER, 507 (void *)filter, i); 508 return err; 509 } 510 511 struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel) 512 { 513 return evsel->cpus; 514 } 515 516 struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel) 517 { 518 return evsel->threads; 519 } 520 521 struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel) 522 { 523 return &evsel->attr; 524 } 525 526 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) 527 { 528 if (ncpus == 0 || nthreads == 0) 529 return 0; 530 531 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); 532 if (evsel->sample_id == NULL) 533 return -ENOMEM; 534 535 evsel->id = zalloc(ncpus * nthreads * sizeof(u64)); 536 if (evsel->id == NULL) { 537 xyarray__delete(evsel->sample_id); 538 evsel->sample_id = NULL; 539 return -ENOMEM; 540 } 541 542 return 0; 543 } 544 545 void perf_evsel__free_id(struct perf_evsel *evsel) 546 { 547 struct perf_sample_id_period *pos, *n; 548 549 xyarray__delete(evsel->sample_id); 550 evsel->sample_id = NULL; 551 zfree(&evsel->id); 552 evsel->ids = 0; 553 554 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) { 555 list_del_init(&pos->node); 556 free(pos); 557 } 558 } 559 560 bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel) 561 { 562 return (evsel->attr.sample_type & PERF_SAMPLE_READ) && 563 (evsel->attr.sample_type & PERF_SAMPLE_TID) && 564 evsel->attr.inherit; 565 } 566 567 u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread) 568 { 569 struct hlist_head *head; 570 struct perf_sample_id_period *res; 571 int hash; 572 573 if (!per_thread) 574 return &sid->period; 575 576 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS); 577 head = &sid->periods[hash]; 578 579 hlist_for_each_entry(res, head, hnode) 580 if (res->tid == tid) 581 return &res->period; 582 583 if (sid->evsel == NULL) 584 return NULL; 585 586 res = zalloc(sizeof(struct perf_sample_id_period)); 587 if (res == NULL) 588 return NULL; 589 590 INIT_LIST_HEAD(&res->node); 591 res->tid = tid; 592 593 list_add_tail(&res->node, &sid->evsel->per_stream_periods); 594 hlist_add_head(&res->hnode, &sid->periods[hash]); 595 596 return &res->period; 597 } 598 599 void perf_counts_values__scale(struct perf_counts_values *count, 600 bool scale, __s8 *pscaled) 601 { 602 s8 scaled = 0; 603 604 if (scale) { 605 if (count->run == 0) { 606 scaled = -1; 607 count->val = 0; 608 } else if (count->run < count->ena) { 609 scaled = 1; 610 count->val = (u64)((double)count->val * count->ena / count->run); 611 } 612 } 613 614 if (pscaled) 615 *pscaled = scaled; 616 } 617