1 // SPDX-License-Identifier: GPL-2.0 2 #include <errno.h> 3 #include <unistd.h> 4 #include <sys/syscall.h> 5 #include <perf/evsel.h> 6 #include <perf/cpumap.h> 7 #include <perf/threadmap.h> 8 #include <linux/hash.h> 9 #include <linux/list.h> 10 #include <internal/evsel.h> 11 #include <linux/zalloc.h> 12 #include <stdlib.h> 13 #include <internal/xyarray.h> 14 #include <internal/cpumap.h> 15 #include <internal/mmap.h> 16 #include <internal/threadmap.h> 17 #include <internal/lib.h> 18 #include <linux/string.h> 19 #include <sys/ioctl.h> 20 #include <sys/mman.h> 21 #include <asm/bug.h> 22 23 void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr, 24 int idx) 25 { 26 INIT_LIST_HEAD(&evsel->node); 27 INIT_LIST_HEAD(&evsel->per_stream_periods); 28 evsel->attr = *attr; 29 evsel->idx = idx; 30 evsel->leader = evsel; 31 } 32 33 struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr) 34 { 35 struct perf_evsel *evsel = zalloc(sizeof(*evsel)); 36 37 if (evsel != NULL) 38 perf_evsel__init(evsel, attr, 0); 39 40 return evsel; 41 } 42 43 void perf_evsel__delete(struct perf_evsel *evsel) 44 { 45 free(evsel); 46 } 47 48 #define FD(_evsel, _cpu_map_idx, _thread) \ 49 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread)) 50 #define MMAP(_evsel, _cpu_map_idx, _thread) \ 51 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \ 52 : NULL) 53 54 int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads) 55 { 56 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int)); 57 58 if (evsel->fd) { 59 int idx, thread; 60 61 for (idx = 0; idx < ncpus; idx++) { 62 for (thread = 0; thread < nthreads; thread++) { 63 int *fd = FD(evsel, idx, thread); 64 65 if (fd) 66 *fd = -1; 67 } 68 } 69 } 70 71 return evsel->fd != NULL ? 0 : -ENOMEM; 72 } 73 74 static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads) 75 { 76 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap)); 77 78 return evsel->mmap != NULL ? 0 : -ENOMEM; 79 } 80 81 static int 82 sys_perf_event_open(struct perf_event_attr *attr, 83 pid_t pid, struct perf_cpu cpu, int group_fd, 84 unsigned long flags) 85 { 86 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags); 87 } 88 89 static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd) 90 { 91 struct perf_evsel *leader = evsel->leader; 92 int *fd; 93 94 if (evsel == leader) { 95 *group_fd = -1; 96 return 0; 97 } 98 99 /* 100 * Leader must be already processed/open, 101 * if not it's a bug. 102 */ 103 if (!leader->fd) 104 return -ENOTCONN; 105 106 fd = FD(leader, cpu_map_idx, thread); 107 if (fd == NULL || *fd == -1) 108 return -EBADF; 109 110 *group_fd = *fd; 111 112 return 0; 113 } 114 115 int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, 116 struct perf_thread_map *threads) 117 { 118 struct perf_cpu cpu; 119 int idx, thread, err = 0; 120 121 if (cpus == NULL) { 122 static struct perf_cpu_map *empty_cpu_map; 123 124 if (empty_cpu_map == NULL) { 125 empty_cpu_map = perf_cpu_map__new_any_cpu(); 126 if (empty_cpu_map == NULL) 127 return -ENOMEM; 128 } 129 130 cpus = empty_cpu_map; 131 } 132 133 if (threads == NULL) { 134 static struct perf_thread_map *empty_thread_map; 135 136 if (empty_thread_map == NULL) { 137 empty_thread_map = perf_thread_map__new_dummy(); 138 if (empty_thread_map == NULL) 139 return -ENOMEM; 140 } 141 142 threads = empty_thread_map; 143 } 144 145 if (evsel->fd == NULL && 146 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0) 147 return -ENOMEM; 148 149 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 150 for (thread = 0; thread < threads->nr; thread++) { 151 int fd, group_fd, *evsel_fd; 152 153 evsel_fd = FD(evsel, idx, thread); 154 if (evsel_fd == NULL) { 155 err = -EINVAL; 156 goto out; 157 } 158 159 err = get_group_fd(evsel, idx, thread, &group_fd); 160 if (err < 0) 161 goto out; 162 163 fd = sys_perf_event_open(&evsel->attr, 164 threads->map[thread].pid, 165 cpu, group_fd, 0); 166 167 if (fd < 0) { 168 err = -errno; 169 goto out; 170 } 171 172 *evsel_fd = fd; 173 } 174 } 175 out: 176 if (err) 177 perf_evsel__close(evsel); 178 179 return err; 180 } 181 182 static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx) 183 { 184 int thread; 185 186 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) { 187 int *fd = FD(evsel, cpu_map_idx, thread); 188 189 if (fd && *fd >= 0) { 190 close(*fd); 191 *fd = -1; 192 } 193 } 194 } 195 196 void perf_evsel__close_fd(struct perf_evsel *evsel) 197 { 198 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++) 199 perf_evsel__close_fd_cpu(evsel, idx); 200 } 201 202 void perf_evsel__free_fd(struct perf_evsel *evsel) 203 { 204 xyarray__delete(evsel->fd); 205 evsel->fd = NULL; 206 } 207 208 void perf_evsel__close(struct perf_evsel *evsel) 209 { 210 if (evsel->fd == NULL) 211 return; 212 213 perf_evsel__close_fd(evsel); 214 perf_evsel__free_fd(evsel); 215 } 216 217 void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx) 218 { 219 if (evsel->fd == NULL) 220 return; 221 222 perf_evsel__close_fd_cpu(evsel, cpu_map_idx); 223 } 224 225 void perf_evsel__munmap(struct perf_evsel *evsel) 226 { 227 int idx, thread; 228 229 if (evsel->fd == NULL || evsel->mmap == NULL) 230 return; 231 232 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 233 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 234 int *fd = FD(evsel, idx, thread); 235 236 if (fd == NULL || *fd < 0) 237 continue; 238 239 perf_mmap__munmap(MMAP(evsel, idx, thread)); 240 } 241 } 242 243 xyarray__delete(evsel->mmap); 244 evsel->mmap = NULL; 245 } 246 247 int perf_evsel__mmap(struct perf_evsel *evsel, int pages) 248 { 249 int ret, idx, thread; 250 struct perf_mmap_param mp = { 251 .prot = PROT_READ | PROT_WRITE, 252 .mask = (pages * page_size) - 1, 253 }; 254 255 if (evsel->fd == NULL || evsel->mmap) 256 return -EINVAL; 257 258 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0) 259 return -ENOMEM; 260 261 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) { 262 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 263 int *fd = FD(evsel, idx, thread); 264 struct perf_mmap *map; 265 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx); 266 267 if (fd == NULL || *fd < 0) 268 continue; 269 270 map = MMAP(evsel, idx, thread); 271 perf_mmap__init(map, NULL, false, NULL); 272 273 ret = perf_mmap__mmap(map, &mp, *fd, cpu); 274 if (ret) { 275 perf_evsel__munmap(evsel); 276 return ret; 277 } 278 } 279 } 280 281 return 0; 282 } 283 284 void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread) 285 { 286 int *fd = FD(evsel, cpu_map_idx, thread); 287 288 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL) 289 return NULL; 290 291 return MMAP(evsel, cpu_map_idx, thread)->base; 292 } 293 294 int perf_evsel__read_size(struct perf_evsel *evsel) 295 { 296 u64 read_format = evsel->attr.read_format; 297 int entry = sizeof(u64); /* value */ 298 int size = 0; 299 int nr = 1; 300 301 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 302 size += sizeof(u64); 303 304 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 305 size += sizeof(u64); 306 307 if (read_format & PERF_FORMAT_ID) 308 entry += sizeof(u64); 309 310 if (read_format & PERF_FORMAT_LOST) 311 entry += sizeof(u64); 312 313 if (read_format & PERF_FORMAT_GROUP) { 314 nr = evsel->nr_members; 315 size += sizeof(u64); 316 } 317 318 size += entry * nr; 319 return size; 320 } 321 322 /* This only reads values for the leader */ 323 static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, 324 int thread, struct perf_counts_values *count) 325 { 326 size_t size = perf_evsel__read_size(evsel); 327 int *fd = FD(evsel, cpu_map_idx, thread); 328 u64 read_format = evsel->attr.read_format; 329 u64 *data; 330 int idx = 1; 331 332 if (fd == NULL || *fd < 0) 333 return -EINVAL; 334 335 data = calloc(1, size); 336 if (data == NULL) 337 return -ENOMEM; 338 339 if (readn(*fd, data, size) <= 0) { 340 free(data); 341 return -errno; 342 } 343 344 /* 345 * This reads only the leader event intentionally since we don't have 346 * perf counts values for sibling events. 347 */ 348 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 349 count->ena = data[idx++]; 350 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 351 count->run = data[idx++]; 352 353 /* value is always available */ 354 count->val = data[idx++]; 355 if (read_format & PERF_FORMAT_ID) 356 count->id = data[idx++]; 357 if (read_format & PERF_FORMAT_LOST) 358 count->lost = data[idx++]; 359 360 free(data); 361 return 0; 362 } 363 364 /* 365 * The perf read format is very flexible. It needs to set the proper 366 * values according to the read format. 367 */ 368 static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, 369 struct perf_counts_values *count) 370 { 371 u64 read_format = evsel->attr.read_format; 372 int n = 0; 373 374 count->val = buf[n++]; 375 376 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 377 count->ena = buf[n++]; 378 379 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 380 count->run = buf[n++]; 381 382 if (read_format & PERF_FORMAT_ID) 383 count->id = buf[n++]; 384 385 if (read_format & PERF_FORMAT_LOST) 386 count->lost = buf[n++]; 387 } 388 389 int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, 390 struct perf_counts_values *count) 391 { 392 size_t size = perf_evsel__read_size(evsel); 393 int *fd = FD(evsel, cpu_map_idx, thread); 394 u64 read_format = evsel->attr.read_format; 395 struct perf_counts_values buf; 396 397 memset(count, 0, sizeof(*count)); 398 399 if (fd == NULL || *fd < 0) 400 return -EINVAL; 401 402 if (read_format & PERF_FORMAT_GROUP) 403 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); 404 405 if (MMAP(evsel, cpu_map_idx, thread) && 406 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && 407 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) 408 return 0; 409 410 if (readn(*fd, buf.values, size) <= 0) 411 return -errno; 412 413 perf_evsel__adjust_values(evsel, buf.values, count); 414 return 0; 415 } 416 417 static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, 418 int cpu_map_idx, int thread) 419 { 420 int *fd = FD(evsel, cpu_map_idx, thread); 421 422 if (fd == NULL || *fd < 0) 423 return -1; 424 425 return ioctl(*fd, ioc, arg); 426 } 427 428 static int perf_evsel__run_ioctl(struct perf_evsel *evsel, 429 int ioc, void *arg, 430 int cpu_map_idx) 431 { 432 int thread; 433 434 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { 435 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); 436 437 if (err) 438 return err; 439 } 440 441 return 0; 442 } 443 444 int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 445 { 446 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); 447 } 448 449 int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) 450 { 451 struct perf_cpu cpu __maybe_unused; 452 int idx; 453 int err; 454 455 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { 456 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread); 457 if (err) 458 return err; 459 } 460 461 return 0; 462 } 463 464 int perf_evsel__enable(struct perf_evsel *evsel) 465 { 466 int i; 467 int err = 0; 468 469 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 470 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i); 471 return err; 472 } 473 474 int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx) 475 { 476 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx); 477 } 478 479 int perf_evsel__disable(struct perf_evsel *evsel) 480 { 481 int i; 482 int err = 0; 483 484 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++) 485 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i); 486 return err; 487 } 488 489 int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter) 490 { 491 int err = 0, i; 492 493 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++) 494 err = perf_evsel__run_ioctl(evsel, 495 PERF_EVENT_IOC_SET_FILTER, 496 (void *)filter, i); 497 return err; 498 } 499 500 struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel) 501 { 502 return evsel->cpus; 503 } 504 505 struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel) 506 { 507 return evsel->threads; 508 } 509 510 struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel) 511 { 512 return &evsel->attr; 513 } 514 515 int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) 516 { 517 if (ncpus == 0 || nthreads == 0) 518 return 0; 519 520 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); 521 if (evsel->sample_id == NULL) 522 return -ENOMEM; 523 524 evsel->id = zalloc(ncpus * nthreads * sizeof(u64)); 525 if (evsel->id == NULL) { 526 xyarray__delete(evsel->sample_id); 527 evsel->sample_id = NULL; 528 return -ENOMEM; 529 } 530 531 return 0; 532 } 533 534 void perf_evsel__free_id(struct perf_evsel *evsel) 535 { 536 struct perf_sample_id_period *pos, *n; 537 538 xyarray__delete(evsel->sample_id); 539 evsel->sample_id = NULL; 540 zfree(&evsel->id); 541 evsel->ids = 0; 542 543 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) { 544 list_del_init(&pos->node); 545 free(pos); 546 } 547 } 548 549 bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel) 550 { 551 return (evsel->attr.sample_type & PERF_SAMPLE_READ) && 552 (evsel->attr.sample_type & PERF_SAMPLE_TID) && 553 evsel->attr.inherit; 554 } 555 556 u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread) 557 { 558 struct hlist_head *head; 559 struct perf_sample_id_period *res; 560 int hash; 561 562 if (!per_thread) 563 return &sid->period; 564 565 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS); 566 head = &sid->periods[hash]; 567 568 hlist_for_each_entry(res, head, hnode) 569 if (res->tid == tid) 570 return &res->period; 571 572 if (sid->evsel == NULL) 573 return NULL; 574 575 res = zalloc(sizeof(struct perf_sample_id_period)); 576 if (res == NULL) 577 return NULL; 578 579 INIT_LIST_HEAD(&res->node); 580 res->tid = tid; 581 582 list_add_tail(&res->node, &sid->evsel->per_stream_periods); 583 hlist_add_head(&res->hnode, &sid->periods[hash]); 584 585 return &res->period; 586 } 587 588 void perf_counts_values__scale(struct perf_counts_values *count, 589 bool scale, __s8 *pscaled) 590 { 591 s8 scaled = 0; 592 593 if (scale) { 594 if (count->run == 0) { 595 scaled = -1; 596 count->val = 0; 597 } else if (count->run < count->ena) { 598 scaled = 1; 599 count->val = (u64)((double)count->val * count->ena / count->run); 600 } 601 } 602 603 if (pscaled) 604 *pscaled = scaled; 605 } 606