1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <unistd.h> 4 #include <sys/syscall.h> 5 #include <perf/evsel.h> 6 #include <perf/cpumap.h> 7 #include <perf/threadmap.h> 8 #include <linux/hash.h> 9 #include <linux/list.h> 10 #include <internal/evsel.h> 11 #include <linux/zalloc.h> 12 #include <stdlib.h> 13 #include <internal/xyarray.h> 14 #include <internal/cpumap.h> 15 #include <internal/mmap.h> 16 #include <internal/threadmap.h> 17 #include <internal/lib.h> 18 #include <linux/string.h> 19 #include <sys/ioctl.h> 20 #include <sys/mman.h> 21 #include <asm/bug.h> 22 23 void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, 24 int idx) 25 { 26 INIT_LIST_HEAD(&evsel->node); 27 INIT_LIST_HEAD(&evsel->per_stream_periods); 28 evsel->attr = *attr; 29 evsel->idx = idx; 30 evsel->leader = evsel; 31 } 32 33 struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr) 34 { 35 struct perf_evsel *evsel = zalloc(sizeof(*evsel)); 36 37 if (evsel != NULL) 38 perf_evsel__init(evsel, attr, 0); 39 40 return evsel; 41 } 42 43 void perf_evsel__delete(struct perf_evsel *evsel) 44 { 45 assert(evsel->fd == NULL); /* If not fds were not closed. */ 46 assert(evsel->mmap == NULL); /* If not munmap wasn't called. */ 47 assert(evsel->sample_id == NULL); /* If not free_id wasn't called. */ 48 perf_cpu_map__put(evsel->cpus); 49 perf_cpu_map__put(evsel->own_cpus); 50 perf_thread_map__put(evsel->threads); 51 free(evsel); 52 } 53 54 #define FD(_evsel, _cpu_map_idx, _thread) \ 55 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread)) 56 #define MMAP(_evsel, _cpu_map_idx, _thread) \ 57 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \ 58 : NULL) 59 60 int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads) 61 { 62 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int)); 63 64 if (evsel->fd) { 65 int idx, thread; 66 67 for (idx = 0; idx < ncpus; idx++) { 68 for (thread = 0; thread < nthreads; thread++) { 69 int *fd = FD(evsel, idx, thread); 70 71 if (fd) 72 *fd = -1; 73 } 74 } 75 } 76 77 return evsel->fd != NULL ? 0 : -ENOMEM; 78 } 79 80 static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads) 81 { 82 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap)); 83 84 return evsel->mmap != NULL ? 0 : -ENOMEM; 85 } 86 87 static int 88 sys_perf_event_open(struct perf_event_attr *attr, 89 pid_t pid, struct perf_cpu cpu, int group_fd, 90 unsigned long flags) 91 { 92 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags); 93 } 94 95 static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd) 96 { 97 struct perf_evsel *leader = evsel->leader; 98 int *fd; 99 100 if (evsel == leader) { 101 *group_fd = -1; 102 return 0; 103 } 104 105 /* 106 * Leader must be already processed/open, 107 * if not it's a bug. 108 */ 109 if (!leader->fd) 110 return -ENOTCONN; 111 112 fd = FD(leader, cpu_map_idx, thread); 113 if (fd == NULL || *fd == -1) 114 return -EBADF; 115 116 *group_fd = *fd; 117 118 return 0; 119 } 120 121 int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, 122 struct perf_thread_map *threads) 123 { 124 struct perf_cpu cpu; 125 int idx, thread, err = 0; 126 127 if (cpus == NULL) { 128 static struct perf_cpu_map *empty_cpu_map; 129 130 if (empty_cpu_map == NULL) { 131 empty_cpu_map = perf_cpu_map__new_any_cpu(); 132 if (empty_cpu_map == NULL) 133 return -ENOMEM; 134 } 135 136 cpus = empty_cpu_map; 137 } 138 139 if (threads == NULL) { 140 static struct perf_thread_map *empty_thread_map; 141 142 if (empty_thread_map == NULL) { 143 empty_thread_map = perf_thread_map__new_dummy(); 144 if (empty_thread_map == NULL) 145 return -ENOMEM; 146 } 147 148 threads = empty_thread_map; 149 } 150 151 if (evsel->fd == NULL && 152 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 153 return -ENOMEM; 154 155 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 156 for (thread = 0; thread < threads->nr; thread++) { 157 int fd, group_fd, *evsel_fd; 158 159 evsel_fd = FD(evsel, idx, thread); 160 if (evsel_fd == NULL) { 161 err = -EINVAL; 162 goto out; 163 } 164 165 err = get_group_fd(evsel, idx, thread, &group_fd); 166 if (err < 0) 167 goto out; 168 169 fd = sys_perf_event_open(&evsel->attr, 170 threads->map[thread].pid, 171 cpu, group_fd, 0); 172 173 if (fd < 0) { 174 err = -errno; 175 goto out; 176 } 177 178 *evsel_fd = fd; 179 } 180 } 181 out: 182 if (err) 183 perf_evsel__close(evsel); 184 185 return err; 186 } 187 188 static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx) 189 { 190 int thread; 191 192 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) { 193 int *fd = FD(evsel, cpu_map_idx, thread); 194 195 if (fd && *fd >= 0) { 196 close(*fd); 197 *fd = -1; 198 } 199 } 200 } 201 202 void perf_evsel__close_fd(struct perf_evsel *evsel) 203 { 204 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++) 205 perf_evsel__close_fd_cpu(evsel, idx); 206 } 207 208 void perf_evsel__free_fd(struct perf_evsel *evsel) 209 { 210 xyarray__delete(evsel->fd); 211 evsel->fd = NULL; 212 } 213 214 void perf_evsel__close(struct perf_evsel *evsel) 215 { 216 if (evsel->fd == NULL) 217 return; 218 219 perf_evsel__close_fd(evsel); 220 perf_evsel__free_fd(evsel); 221 } 222 223 void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx) 224 { 225 if (evsel->fd == NULL) 226 return; 227 228 perf_evsel__close_fd_cpu(evsel, cpu_map_idx); 229 } 230 231 void perf_evsel__munmap(struct perf_evsel *evsel) 232 { 233 int idx, thread; 234 235 if (evsel->fd == NULL || evsel->mmap == NULL) 236 return; 237 238 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 239 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 240 int *fd = FD(evsel, idx, thread); 241 242 if (fd == NULL || *fd < 0) 243 continue; 244 245 perf_mmap__munmap(MMAP(evsel, idx, thread)); 246 } 247 } 248 249 xyarray__delete(evsel->mmap); 250 evsel->mmap = NULL; 251 } 252 253 int perf_evsel__mmap(struct perf_evsel *evsel, int pages) 254 { 255 int ret, idx, thread; 256 struct perf_mmap_param mp = { 257 .prot = PROT_READ | PROT_WRITE, 258 .mask = (pages * page_size) - 1, 259 }; 260 261 if (evsel->fd == NULL || evsel->mmap) 262 return -EINVAL; 263 264 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0) 265 return -ENOMEM; 266 267 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 268 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 269 int *fd = FD(evsel, idx, thread); 270 struct perf_mmap *map; 271 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx); 272 273 if (fd == NULL || *fd < 0) 274 continue; 275 276 map = MMAP(evsel, idx, thread); 277 perf_mmap__init(map, NULL, false, NULL); 278 279 ret = perf_mmap__mmap(map, &mp, *fd, cpu); 280 if (ret) { 281 perf_evsel__munmap(evsel); 282 return ret; 283 } 284 } 285 } 286 287 return 0; 288 } 289 290 void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread) 291 { 292 int *fd = FD(evsel, cpu_map_idx, thread); 293 294 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL) 295 return NULL; 296 297 return MMAP(evsel, cpu_map_idx, thread)->base; 298 } 299 300 int perf_evsel__read_size(struct perf_evsel *evsel) 301 { 302 u64 read_format = evsel->attr.read_format; 303 int entry = sizeof(u64); /* value */ 304 int size = 0; 305 int nr = 1; 306 307 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 308 size += sizeof(u64); 309 310 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 311 size += sizeof(u64); 312 313 if (read_format & PERF_FORMAT_ID) 314 entry += sizeof(u64); 315 316 if (read_format & PERF_FORMAT_LOST) 317 entry += sizeof(u64); 318 319 if (read_format & PERF_FORMAT_GROUP) { 320 nr = evsel->nr_members; 321 size += sizeof(u64); 322 } 323 324 size += entry * nr; 325 return size; 326 } 327 328 /* This only reads values for the leader */ 329 static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, 330 int thread, struct perf_counts_values *count) 331 { 332 size_t size = perf_evsel__read_size(evsel); 333 int *fd = FD(evsel, cpu_map_idx, thread); 334 u64 read_format = evsel->attr.read_format; 335 u64 *data; 336 int idx = 1; 337 338 if (fd == NULL || *fd < 0) 339 return -EINVAL; 340 341 data = calloc(1, size); 342 if (data == NULL) 343 return -ENOMEM; 344 345 if (readn(*fd, data, size) <= 0) { 346 free(data); 347 return -errno; 348 } 349 350 /* 351 * This reads only the leader event intentionally since we don't have 352 * perf counts values for sibling events. 353 */ 354 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 355 count->ena = data[idx++]; 356 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 357 count->run = data[idx++]; 358 359 /* value is always available */ 360 count->val = data[idx++]; 361 if (read_format & PERF_FORMAT_ID) 362 count->id = data[idx++]; 363 if (read_format & PERF_FORMAT_LOST) 364 count->lost = data[idx++]; 365 366 free(data); 367 return 0; 368 } 369 370 /* 371 * The perf read format is very flexible. It needs to set the proper 372 * values according to the read format. 373 */ 374 static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, 375 struct perf_counts_values *count) 376 { 377 u64 read_format = evsel->attr.read_format; 378 int n = 0; 379 380 count->val = buf[n++]; 381 382 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 383 count->ena = buf[n++]; 384 385 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 386 count->run = buf[n++]; 387 388 if (read_format & PERF_FORMAT_ID) 389 count->id = buf[n++]; 390 391 if (read_format & PERF_FORMAT_LOST) 392 count->lost = buf[n++]; 393 } 394 395 int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, 396 struct perf_counts_values *count) 397 { 398 size_t size = perf_evsel__read_size(evsel); 399 int *fd = FD(evsel, cpu_map_idx, thread); 400 u64 read_format = evsel->attr.read_format; 401 struct perf_counts_values buf; 402 403 memset(count, 0, sizeof(*count)); 404 405 if (fd == NULL || *fd < 0) 406 return -EINVAL; 407 408 if (read_format & PERF_FORMAT_GROUP) 409 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); 410 411 if (MMAP(evsel, cpu_map_idx, thread) && 412 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && 413 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) 414 return 0; 415 416 if (readn(*fd, buf.values, size) <= 0) 417 return -errno; 418 419 perf_evsel__adjust_values(evsel, buf.values, count); 420 return 0; 421 } 422 423 static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, 424 int cpu_map_idx, int thread) 425 { 426 int *fd = FD(evsel, cpu_map_idx, thread); 427 428 if (fd == NULL || *fd < 0) 429 return -1; 430 431 return ioctl(*fd, ioc, arg); 432 } 433 434 static int perf_evsel__run_ioctl(struct perf_evsel *evsel, 435 int ioc, void *arg, 436 int cpu_map_idx) 437 { 438 int thread; 439 440 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 441 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); 442 443 if (err) 444 return err; 445 } 446 447 return 0; 448 } 449 450 int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 451 { 452 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); 453 } 454 455 int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) 456 { 457 struct perf_cpu cpu __maybe_unused; 458 int idx; 459 int err; 460 461 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { 462 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread); 463 if (err) 464 return err; 465 } 466 467 return 0; 468 } 469 470 int perf_evsel__enable(struct perf_evsel *evsel) 471 { 472 int i; 473 int err = 0; 474 475 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 476 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i); 477 return err; 478 } 479 480 int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 481 { 482 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx); 483 } 484 485 int perf_evsel__disable(struct perf_evsel *evsel) 486 { 487 int i; 488 int err = 0; 489 490 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 491 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i); 492 return err; 493 } 494 495 int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) 496 { 497 int err = 0, i; 498 499 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) 500 err = perf_evsel__run_ioctl(evsel, 501 PERF_EVENT_IOC_SET_FILTER, 502 (void *)filter, i); 503 return err; 504 } 505 506 struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel) 507 { 508 return evsel->cpus; 509 } 510 511 struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel) 512 { 513 return evsel->threads; 514 } 515 516 struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel) 517 { 518 return &evsel->attr; 519 } 520 521 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) 522 { 523 if (ncpus == 0 || nthreads == 0) 524 return 0; 525 526 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); 527 if (evsel->sample_id == NULL) 528 return -ENOMEM; 529 530 evsel->id = zalloc(ncpus * nthreads * sizeof(u64)); 531 if (evsel->id == NULL) { 532 xyarray__delete(evsel->sample_id); 533 evsel->sample_id = NULL; 534 return -ENOMEM; 535 } 536 537 return 0; 538 } 539 540 void perf_evsel__free_id(struct perf_evsel *evsel) 541 { 542 struct perf_sample_id_period *pos, *n; 543 544 xyarray__delete(evsel->sample_id); 545 evsel->sample_id = NULL; 546 zfree(&evsel->id); 547 evsel->ids = 0; 548 549 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) { 550 list_del_init(&pos->node); 551 free(pos); 552 } 553 } 554 555 bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel) 556 { 557 return (evsel->attr.sample_type & PERF_SAMPLE_READ) && 558 (evsel->attr.sample_type & PERF_SAMPLE_TID) && 559 evsel->attr.inherit; 560 } 561 562 u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread) 563 { 564 struct hlist_head *head; 565 struct perf_sample_id_period *res; 566 int hash; 567 568 if (!per_thread) 569 return &sid->period; 570 571 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS); 572 head = &sid->periods[hash]; 573 574 hlist_for_each_entry(res, head, hnode) 575 if (res->tid == tid) 576 return &res->period; 577 578 if (sid->evsel == NULL) 579 return NULL; 580 581 res = zalloc(sizeof(struct perf_sample_id_period)); 582 if (res == NULL) 583 return NULL; 584 585 INIT_LIST_HEAD(&res->node); 586 res->tid = tid; 587 588 list_add_tail(&res->node, &sid->evsel->per_stream_periods); 589 hlist_add_head(&res->hnode, &sid->periods[hash]); 590 591 return &res->period; 592 } 593 594 void perf_counts_values__scale(struct perf_counts_values *count, 595 bool scale, __s8 *pscaled) 596 { 597 s8 scaled = 0; 598 599 if (scale) { 600 if (count->run == 0) { 601 scaled = -1; 602 count->val = 0; 603 } else if (count->run < count->ena) { 604 scaled = 1; 605 count->val = (u64)((double)count->val * count->ena / count->run); 606 } 607 } 608 609 if (pscaled) 610 *pscaled = scaled; 611 } 612