1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * intel_tpebs.c: Intel TPEBS support 4 */ 5 6 #include <api/fs/fs.h> 7 #include <sys/param.h> 8 #include <subcmd/run-command.h> 9 #include <thread.h> 10 #include "intel-tpebs.h" 11 #include <linux/list.h> 12 #include <linux/zalloc.h> 13 #include <linux/err.h> 14 #include "sample.h" 15 #include "counts.h" 16 #include "debug.h" 17 #include "evlist.h" 18 #include "evsel.h" 19 #include "mutex.h" 20 #include "session.h" 21 #include "stat.h" 22 #include "tool.h" 23 #include "cpumap.h" 24 #include "metricgroup.h" 25 #include <sys/stat.h> 26 #include <sys/file.h> 27 #include <errno.h> 28 #include <poll.h> 29 #include <math.h> 30 31 #define PERF_DATA "-" 32 33 bool tpebs_recording; 34 enum tpebs_mode tpebs_mode; 35 static LIST_HEAD(tpebs_results); 36 static pthread_t tpebs_reader_thread; 37 static struct child_process tpebs_cmd; 38 static int control_fd[2], ack_fd[2]; 39 static struct mutex tpebs_mtx; 40 static bool tpebs_stopping; 41 42 struct tpebs_retire_lat { 43 struct list_head nd; 44 /** @evsel: The evsel that opened the retire_lat event. */ 45 struct evsel *evsel; 46 /** @event: Event passed to perf record. */ 47 char *event; 48 /** @stats: Recorded retirement latency stats. */ 49 struct stats stats; 50 /** @last: Last retirement latency read. */ 51 uint64_t last; 52 /* Has the event been sent to perf record? */ 53 bool started; 54 }; 55 56 static void tpebs_init(void) 57 { 58 mutex_init(&tpebs_mtx); 59 control_fd[0] = control_fd[1] = -1; 60 ack_fd[0] = ack_fd[1] = -1; 61 } 62 63 static struct mutex *tpebs_mtx_get(void) 64 { 65 static pthread_once_t tpebs_once = PTHREAD_ONCE_INIT; 66 67 pthread_once(&tpebs_once, tpebs_init); 68 return &tpebs_mtx; 69 } 70 71 static struct tpebs_retire_lat *tpebs_retire_lat__find(struct evsel *evsel) 72 EXCLUSIVE_LOCKS_REQUIRED(tpebs_mtx_get()); 73 74 static int evsel__tpebs_start_perf_record(struct evsel *evsel) 75 { 76 const char **record_argv; 77 int tpebs_event_size = 0, i = 0, ret; 78 char control_fd_buf[32]; 79 char cpumap_buf[50]; 80 struct tpebs_retire_lat *t; 81 82 list_for_each_entry(t, &tpebs_results, nd) 83 tpebs_event_size++; 84 85 record_argv = malloc((10 + 2 * tpebs_event_size) * sizeof(*record_argv)); 86 if (!record_argv) 87 return -ENOMEM; 88 89 record_argv[i++] = "perf"; 90 record_argv[i++] = "record"; 91 record_argv[i++] = "-W"; 92 record_argv[i++] = "--synth=no"; 93 94 scnprintf(control_fd_buf, sizeof(control_fd_buf), "--control=fd:%d,%d", 95 control_fd[0], ack_fd[1]); 96 record_argv[i++] = control_fd_buf; 97 98 record_argv[i++] = "-o"; 99 record_argv[i++] = PERF_DATA; 100 101 if (!perf_cpu_map__is_any_cpu_or_is_empty( 102 evlist__core(evsel->evlist)->user_requested_cpus)) { 103 cpu_map__snprint(evlist__core(evsel->evlist)->user_requested_cpus, cpumap_buf, 104 sizeof(cpumap_buf)); 105 record_argv[i++] = "-C"; 106 record_argv[i++] = cpumap_buf; 107 } 108 109 list_for_each_entry(t, &tpebs_results, nd) { 110 record_argv[i++] = "-e"; 111 record_argv[i++] = t->event; 112 } 113 record_argv[i++] = NULL; 114 assert(i == 10 + 2 * tpebs_event_size || i == 8 + 2 * tpebs_event_size); 115 /* Note, no workload given so system wide is implied. */ 116 117 assert(tpebs_cmd.pid == 0); 118 memset(&tpebs_cmd, 0, sizeof(tpebs_cmd)); 119 tpebs_cmd.argv = record_argv; 120 tpebs_cmd.out = -1; 121 ret = start_command(&tpebs_cmd); 122 zfree(&tpebs_cmd.argv); 123 list_for_each_entry(t, &tpebs_results, nd) 124 t->started = true; 125 126 return ret; 127 } 128 129 static bool is_child_pid(pid_t parent, pid_t child) 130 { 131 if (parent < 0 || child < 0) 132 return false; 133 134 while (true) { 135 char path[PATH_MAX]; 136 char line[256]; 137 FILE *fp; 138 139 new_child: 140 if (parent == child) 141 return true; 142 143 if (child <= 0) 144 return false; 145 146 scnprintf(path, sizeof(path), "%s/%d/status", procfs__mountpoint(), child); 147 fp = fopen(path, "r"); 148 if (!fp) { 149 /* Presumably the process went away. Assume not a child. */ 150 return false; 151 } 152 while (fgets(line, sizeof(line), fp) != NULL) { 153 if (strncmp(line, "PPid:", 5) == 0) { 154 fclose(fp); 155 if (sscanf(line + 5, "%d", &child) != 1) { 156 /* Unexpected error parsing. */ 157 return false; 158 } 159 goto new_child; 160 } 161 } 162 /* Unexpected EOF. */ 163 fclose(fp); 164 return false; 165 } 166 } 167 168 static bool should_ignore_sample(const struct perf_sample *sample, const struct tpebs_retire_lat *t) 169 { 170 pid_t workload_pid, sample_pid = sample->pid; 171 172 /* 173 * During evlist__purge the evlist will be removed prior to the 174 * evsel__exit calling evsel__tpebs_close and taking the 175 * tpebs_mtx. Avoid a segfault by ignoring samples in this case. 176 */ 177 if (t->evsel->evlist == NULL) 178 return true; 179 180 workload_pid = evlist__workload_pid(t->evsel->evlist); 181 if (workload_pid < 0 || workload_pid == sample_pid) 182 return false; 183 184 if (!t->evsel->core.attr.inherit) 185 return true; 186 187 return !is_child_pid(workload_pid, sample_pid); 188 } 189 190 static int process_sample_event(const struct perf_tool *tool __maybe_unused, 191 union perf_event *event __maybe_unused, 192 struct perf_sample *sample, 193 struct machine *machine __maybe_unused) 194 { 195 struct tpebs_retire_lat *t; 196 197 mutex_lock(tpebs_mtx_get()); 198 if (tpebs_cmd.pid == 0) { 199 /* Record has terminated. */ 200 mutex_unlock(tpebs_mtx_get()); 201 return 0; 202 } 203 t = tpebs_retire_lat__find(sample->evsel); 204 if (!t) { 205 mutex_unlock(tpebs_mtx_get()); 206 return -EINVAL; 207 } 208 if (should_ignore_sample(sample, t)) { 209 mutex_unlock(tpebs_mtx_get()); 210 return 0; 211 } 212 /* 213 * Need to handle per core results? We are assuming average retire 214 * latency value will be used. Save the number of samples and the sum of 215 * retire latency value for each event. 216 */ 217 t->last = sample->weight3; 218 update_stats(&t->stats, sample->weight3); 219 mutex_unlock(tpebs_mtx_get()); 220 return 0; 221 } 222 223 static void *__sample_reader(void *arg __maybe_unused) 224 { 225 struct perf_session *session; 226 struct perf_data data = { 227 .mode = PERF_DATA_MODE_READ, 228 .path = PERF_DATA, 229 .file.fd = tpebs_cmd.out, 230 }; 231 struct perf_tool tool; 232 233 perf_tool__init(&tool, /*ordered_events=*/false); 234 tool.sample = process_sample_event; 235 tool.feature = perf_event__process_feature; 236 tool.attr = perf_event__process_attr; 237 238 session = perf_session__new(&data, &tool); 239 if (IS_ERR(session)) 240 return NULL; 241 perf_session__process_events(session); 242 perf_session__delete(session); 243 244 return NULL; 245 } 246 247 static int tpebs_send_record_cmd(const char *msg) EXCLUSIVE_LOCKS_REQUIRED(tpebs_mtx_get()) 248 { 249 struct pollfd pollfd = { .events = POLLIN, }; 250 int ret, len, retries = 0; 251 char ack_buf[8]; 252 253 /* Check if the command exited before the send, done with the lock held. */ 254 if (tpebs_cmd.pid == 0) 255 return 0; 256 257 /* 258 * Let go of the lock while sending/receiving as blocking can starve the 259 * sample reading thread. 260 */ 261 mutex_unlock(tpebs_mtx_get()); 262 263 /* Send perf record command.*/ 264 len = strlen(msg); 265 ret = write(control_fd[1], msg, len); 266 if (ret != len) { 267 pr_err("perf record control write control message '%s' failed\n", msg); 268 ret = -EPIPE; 269 goto out; 270 } 271 272 if (!strcmp(msg, EVLIST_CTL_CMD_STOP_TAG)) { 273 ret = 0; 274 goto out; 275 } 276 277 /* Wait for an ack. */ 278 pollfd.fd = ack_fd[0]; 279 280 /* 281 * We need this poll to ensure the ack_fd PIPE will not hang 282 * when perf record failed for any reason. The timeout value 283 * 3000ms is an empirical selection. 284 */ 285 again: 286 if (!poll(&pollfd, 1, 500)) { 287 if (check_if_command_finished(&tpebs_cmd)) { 288 ret = 0; 289 goto out; 290 } 291 292 if (retries++ < 6) 293 goto again; 294 pr_err("tpebs failed: perf record ack timeout for '%s'\n", msg); 295 ret = -ETIMEDOUT; 296 goto out; 297 } 298 299 if (!(pollfd.revents & POLLIN)) { 300 if (check_if_command_finished(&tpebs_cmd)) { 301 ret = 0; 302 goto out; 303 } 304 305 pr_err("tpebs failed: did not received an ack for '%s'\n", msg); 306 ret = -EPIPE; 307 goto out; 308 } 309 310 ret = read(ack_fd[0], ack_buf, sizeof(ack_buf)); 311 if (ret > 0) 312 ret = strcmp(ack_buf, EVLIST_CTL_CMD_ACK_TAG); 313 else 314 pr_err("tpebs: perf record control ack failed\n"); 315 out: 316 /* Re-take lock as expected by caller. */ 317 mutex_lock(tpebs_mtx_get()); 318 return ret; 319 } 320 321 /* 322 * tpebs_stop - stop the sample data read thread and the perf record process. 323 */ 324 static int tpebs_stop(void) EXCLUSIVE_LOCKS_REQUIRED(tpebs_mtx_get()) 325 { 326 int ret = 0; 327 328 if (tpebs_stopping) 329 return 0; 330 331 /* Like tpebs_start, we should only run tpebs_end once. */ 332 if (tpebs_cmd.pid != 0) { 333 pid_t actual_pid = tpebs_cmd.pid; 334 335 tpebs_stopping = true; 336 tpebs_send_record_cmd(EVLIST_CTL_CMD_STOP_TAG); 337 tpebs_cmd.pid = 0; 338 mutex_unlock(tpebs_mtx_get()); 339 pthread_join(tpebs_reader_thread, NULL); 340 mutex_lock(tpebs_mtx_get()); 341 if (control_fd[0] >= 0) { 342 close(control_fd[0]); 343 control_fd[0] = -1; 344 } 345 if (control_fd[1] >= 0) { 346 close(control_fd[1]); 347 control_fd[1] = -1; 348 } 349 if (ack_fd[0] >= 0) { 350 close(ack_fd[0]); 351 ack_fd[0] = -1; 352 } 353 if (ack_fd[1] >= 0) { 354 close(ack_fd[1]); 355 ack_fd[1] = -1; 356 } 357 if (tpebs_cmd.out >= 0) { 358 close(tpebs_cmd.out); 359 tpebs_cmd.out = -1; 360 } 361 tpebs_cmd.pid = actual_pid; 362 ret = finish_command(&tpebs_cmd); 363 tpebs_cmd.pid = 0; 364 tpebs_stopping = false; 365 if (ret == -ERR_RUN_COMMAND_WAITPID_SIGNAL) 366 ret = 0; 367 } 368 return ret; 369 } 370 371 /** 372 * evsel__tpebs_event() - Create string event encoding to pass to `perf record`. 373 */ 374 static int evsel__tpebs_event(struct evsel *evsel, char **event) 375 { 376 char *name, *modifier; 377 int ret; 378 379 name = strdup(evsel->name); 380 if (!name) 381 return -ENOMEM; 382 383 modifier = strrchr(name, 'R'); 384 if (!modifier) { 385 ret = -EINVAL; 386 goto out; 387 } 388 *modifier = 'p'; 389 modifier = strchr(name, ':'); 390 if (!modifier) 391 modifier = strrchr(name, '/'); 392 if (!modifier) { 393 ret = -EINVAL; 394 goto out; 395 } 396 *modifier = '\0'; 397 if (asprintf(event, "%s/name=tpebs_event_%p/%s", name, evsel, modifier + 1) > 0) 398 ret = 0; 399 else 400 ret = -ENOMEM; 401 out: 402 if (ret) 403 pr_err("Tpebs event modifier broken '%s'\n", evsel->name); 404 free(name); 405 return ret; 406 } 407 408 static struct tpebs_retire_lat *tpebs_retire_lat__new(struct evsel *evsel) 409 { 410 struct tpebs_retire_lat *result = zalloc(sizeof(*result)); 411 int ret; 412 413 if (!result) 414 return NULL; 415 416 ret = evsel__tpebs_event(evsel, &result->event); 417 if (ret) { 418 free(result); 419 return NULL; 420 } 421 result->evsel = evsel; 422 return result; 423 } 424 425 static void tpebs_retire_lat__delete(struct tpebs_retire_lat *r) 426 { 427 zfree(&r->event); 428 free(r); 429 } 430 431 static struct tpebs_retire_lat *tpebs_retire_lat__find(struct evsel *evsel) 432 { 433 struct tpebs_retire_lat *t; 434 unsigned long num; 435 const char *evsel_name; 436 437 /* 438 * Evsels will match for evlist with the retirement latency event. The 439 * name with "tpebs_event_" prefix will be present on events being read 440 * from `perf record`. 441 */ 442 if (evsel__is_retire_lat(evsel)) { 443 list_for_each_entry(t, &tpebs_results, nd) { 444 if (t->evsel == evsel) 445 return t; 446 } 447 return NULL; 448 } 449 evsel_name = strstr(evsel->name, "tpebs_event_"); 450 if (!evsel_name) { 451 /* Unexpected that the perf record should have other events. */ 452 return NULL; 453 } 454 errno = 0; 455 num = strtoull(evsel_name + 12, NULL, 16); 456 if (errno) { 457 pr_err("Bad evsel for tpebs find '%s'\n", evsel->name); 458 return NULL; 459 } 460 list_for_each_entry(t, &tpebs_results, nd) { 461 if ((unsigned long)t->evsel == num) 462 return t; 463 } 464 return NULL; 465 } 466 467 /** 468 * evsel__tpebs_prepare - create tpebs data structures ready for opening. 469 * @evsel: retire_latency evsel, all evsels on its list will be prepared. 470 */ 471 static int evsel__tpebs_prepare(struct evsel *evsel) 472 { 473 struct evsel *pos; 474 struct tpebs_retire_lat *tpebs_event; 475 476 mutex_lock(tpebs_mtx_get()); 477 tpebs_event = tpebs_retire_lat__find(evsel); 478 if (tpebs_event) { 479 /* evsel, or an identically named one, was already prepared. */ 480 mutex_unlock(tpebs_mtx_get()); 481 return 0; 482 } 483 tpebs_event = tpebs_retire_lat__new(evsel); 484 if (!tpebs_event) { 485 mutex_unlock(tpebs_mtx_get()); 486 return -ENOMEM; 487 } 488 list_add_tail(&tpebs_event->nd, &tpebs_results); 489 mutex_unlock(tpebs_mtx_get()); 490 491 /* 492 * Eagerly prepare all other evsels on the list to try to ensure that by 493 * open they are all known. 494 */ 495 evlist__for_each_entry(evsel->evlist, pos) { 496 int ret; 497 498 if (pos == evsel || !pos->retire_lat) 499 continue; 500 501 ret = evsel__tpebs_prepare(pos); 502 if (ret) 503 return ret; 504 } 505 return 0; 506 } 507 508 /** 509 * evsel__tpebs_open - starts tpebs execution. 510 * @evsel: retire_latency evsel, all evsels on its list will be selected. Each 511 * evsel is sampled to get the average retire_latency value. 512 */ 513 int evsel__tpebs_open(struct evsel *evsel) 514 { 515 int ret; 516 bool tpebs_empty; 517 bool started_process = false; 518 519 /* We should only run tpebs_start when tpebs_recording is enabled. */ 520 if (!tpebs_recording) 521 return 0; 522 523 mutex_lock(tpebs_mtx_get()); 524 if (tpebs_stopping) { 525 mutex_unlock(tpebs_mtx_get()); 526 return -EBUSY; 527 } 528 /* Only start the events once. */ 529 if (tpebs_cmd.pid != 0) { 530 struct tpebs_retire_lat *t; 531 bool valid; 532 533 t = tpebs_retire_lat__find(evsel); 534 valid = t && t->started; 535 mutex_unlock(tpebs_mtx_get()); 536 /* May fail as the event wasn't started. */ 537 return valid ? 0 : -EBUSY; 538 } 539 mutex_unlock(tpebs_mtx_get()); 540 541 ret = evsel__tpebs_prepare(evsel); 542 if (ret) 543 return ret; 544 545 mutex_lock(tpebs_mtx_get()); 546 if (tpebs_stopping || tpebs_cmd.pid != 0) { 547 ret = -EBUSY; 548 goto out; 549 } 550 tpebs_empty = list_empty(&tpebs_results); 551 if (!tpebs_empty) { 552 started_process = true; 553 /*Create control and ack fd for --control*/ 554 if (pipe(control_fd) < 0) { 555 pr_err("tpebs: Failed to create control fifo"); 556 ret = -1; 557 goto out; 558 } 559 if (pipe(ack_fd) < 0) { 560 pr_err("tpebs: Failed to create control fifo"); 561 ret = -1; 562 goto out; 563 } 564 565 ret = evsel__tpebs_start_perf_record(evsel); 566 if (ret) 567 goto out; 568 569 if (pthread_create(&tpebs_reader_thread, /*attr=*/NULL, __sample_reader, 570 /*arg=*/NULL)) { 571 kill(tpebs_cmd.pid, SIGTERM); 572 pr_err("Could not create thread to process sample data.\n"); 573 ret = -1; 574 goto out; 575 } 576 ret = tpebs_send_record_cmd(EVLIST_CTL_CMD_ENABLE_TAG); 577 } 578 out: 579 if (ret) { 580 struct tpebs_retire_lat *t = tpebs_retire_lat__find(evsel); 581 582 if (t) { 583 list_del_init(&t->nd); 584 tpebs_retire_lat__delete(t); 585 } 586 587 if (started_process) { 588 if (tpebs_cmd.pid > 0) { 589 kill(tpebs_cmd.pid, SIGTERM); 590 finish_command(&tpebs_cmd); 591 tpebs_cmd.pid = 0; 592 } 593 if (tpebs_cmd.out >= 0) { 594 close(tpebs_cmd.out); 595 tpebs_cmd.out = -1; 596 } 597 if (control_fd[0] >= 0) { 598 close(control_fd[0]); 599 control_fd[0] = -1; 600 } 601 if (control_fd[1] >= 0) { 602 close(control_fd[1]); 603 control_fd[1] = -1; 604 } 605 if (ack_fd[0] >= 0) { 606 close(ack_fd[0]); 607 ack_fd[0] = -1; 608 } 609 if (ack_fd[1] >= 0) { 610 close(ack_fd[1]); 611 ack_fd[1] = -1; 612 } 613 } 614 } 615 mutex_unlock(tpebs_mtx_get()); 616 return ret; 617 } 618 619 int evsel__tpebs_read(struct evsel *evsel, int cpu_map_idx, int thread) 620 { 621 struct perf_counts_values *count, *old_count = NULL; 622 struct tpebs_retire_lat *t; 623 uint64_t val; 624 int ret; 625 626 /* Only set retire_latency value to the first CPU and thread. */ 627 if (cpu_map_idx != 0 || thread != 0) 628 return 0; 629 630 if (evsel->prev_raw_counts) 631 old_count = perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); 632 633 count = perf_counts(evsel->counts, cpu_map_idx, thread); 634 635 mutex_lock(tpebs_mtx_get()); 636 t = tpebs_retire_lat__find(evsel); 637 /* 638 * If reading the first tpebs result, send a ping to the record 639 * process. Allow the sample reader a chance to read by releasing and 640 * reacquiring the lock. 641 */ 642 if (t && &t->nd == tpebs_results.next) { 643 ret = tpebs_send_record_cmd(EVLIST_CTL_CMD_PING_TAG); 644 mutex_unlock(tpebs_mtx_get()); 645 if (ret) 646 return ret; 647 mutex_lock(tpebs_mtx_get()); 648 } 649 if (t == NULL || t->stats.n == 0) { 650 /* No sample data, use default. */ 651 if (tpebs_recording) { 652 pr_warning_once( 653 "Using precomputed retirement latency data as no samples\n"); 654 } 655 val = 0; 656 switch (tpebs_mode) { 657 case TPEBS_MODE__MIN: 658 val = rint(evsel->retirement_latency.min); 659 break; 660 case TPEBS_MODE__MAX: 661 val = rint(evsel->retirement_latency.max); 662 break; 663 default: 664 case TPEBS_MODE__LAST: 665 case TPEBS_MODE__MEAN: 666 val = rint(evsel->retirement_latency.mean); 667 break; 668 } 669 } else { 670 switch (tpebs_mode) { 671 case TPEBS_MODE__MIN: 672 val = t->stats.min; 673 break; 674 case TPEBS_MODE__MAX: 675 val = t->stats.max; 676 break; 677 case TPEBS_MODE__LAST: 678 val = t->last; 679 break; 680 default: 681 case TPEBS_MODE__MEAN: 682 val = rint(t->stats.mean); 683 break; 684 } 685 } 686 mutex_unlock(tpebs_mtx_get()); 687 688 if (old_count) { 689 count->val = old_count->val + val; 690 count->run = old_count->run + 1; 691 count->ena = old_count->ena + 1; 692 } else { 693 count->val = val; 694 count->run++; 695 count->ena++; 696 } 697 return 0; 698 } 699 700 /** 701 * evsel__tpebs_close() - delete tpebs related data. If the last event, stop the 702 * created thread and process by calling tpebs_stop(). 703 * 704 * This function is called in evsel__close() to be symmetric with 705 * evsel__tpebs_open() being called in evsel__open(). 706 */ 707 void evsel__tpebs_close(struct evsel *evsel) 708 { 709 struct tpebs_retire_lat *t; 710 711 mutex_lock(tpebs_mtx_get()); 712 t = tpebs_retire_lat__find(evsel); 713 if (t) { 714 list_del_init(&t->nd); 715 tpebs_retire_lat__delete(t); 716 717 if (list_empty(&tpebs_results)) 718 tpebs_stop(); 719 } 720 mutex_unlock(tpebs_mtx_get()); 721 } 722