1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 // Copyright (c) 2019 Facebook 3 #include <argp.h> 4 #include <stdio.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <time.h> 8 #include <bpf/libbpf.h> 9 #include <bpf/bpf.h> 10 #include "runqslower.h" 11 #include "runqslower.skel.h" 12 13 struct env { 14 pid_t pid; 15 __u64 min_us; 16 bool verbose; 17 } env = { 18 .min_us = 10000, 19 }; 20 21 const char *argp_program_version = "runqslower 0.1"; 22 const char *argp_program_bug_address = "<bpf@vger.kernel.org>"; 23 const char argp_program_doc[] = 24 "runqslower Trace long process scheduling delays.\n" 25 " For Linux, uses eBPF, BPF CO-RE, libbpf, BTF.\n" 26 "\n" 27 "This script traces high scheduling delays between tasks being\n" 28 "ready to run and them running on CPU after that.\n" 29 "\n" 30 "USAGE: runqslower [-p PID] [min_us]\n" 31 "\n" 32 "EXAMPLES:\n" 33 " runqslower # trace run queue latency higher than 10000 us (default)\n" 34 " runqslower 1000 # trace run queue latency higher than 1000 us\n" 35 " runqslower -p 123 # trace pid 123 only\n"; 36 37 static const struct argp_option opts[] = { 38 { "pid", 'p', "PID", 0, "Process PID to trace"}, 39 { "verbose", 'v', NULL, 0, "Verbose debug output" }, 40 {}, 41 }; 42 43 static error_t parse_arg(int key, char *arg, struct argp_state *state) 44 { 45 static int pos_args; 46 int pid; 47 long long min_us; 48 49 switch (key) { 50 case 'v': 51 env.verbose = true; 52 break; 53 case 'p': 54 errno = 0; 55 pid = strtol(arg, NULL, 10); 56 if (errno || pid <= 0) { 57 fprintf(stderr, "Invalid PID: %s\n", arg); 58 argp_usage(state); 59 } 60 env.pid = pid; 61 break; 62 case ARGP_KEY_ARG: 63 if (pos_args++) { 64 fprintf(stderr, 65 "Unrecognized positional argument: %s\n", arg); 66 argp_usage(state); 67 } 68 errno = 0; 69 min_us = strtoll(arg, NULL, 10); 70 if (errno || min_us <= 0) { 71 fprintf(stderr, "Invalid delay (in us): %s\n", arg); 72 argp_usage(state); 73 } 74 env.min_us = min_us; 75 break; 76 default: 77 return ARGP_ERR_UNKNOWN; 78 } 79 return 0; 80 } 81 82 int libbpf_print_fn(enum libbpf_print_level level, 83 const char *format, va_list args) 84 { 85 if (level == LIBBPF_DEBUG && !env.verbose) 86 return 0; 87 return vfprintf(stderr, format, args); 88 } 89 90 void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) 91 { 92 const struct runq_event *e = data; 93 struct tm *tm; 94 char ts[32]; 95 time_t t; 96 97 time(&t); 98 tm = localtime(&t); 99 strftime(ts, sizeof(ts), "%H:%M:%S", tm); 100 printf("%-8s %-16s %-6d %14llu\n", ts, e->task, e->pid, e->delta_us); 101 } 102 103 void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt) 104 { 105 printf("Lost %llu events on CPU #%d!\n", lost_cnt, cpu); 106 } 107 108 int main(int argc, char **argv) 109 { 110 static const struct argp argp = { 111 .options = opts, 112 .parser = parse_arg, 113 .doc = argp_program_doc, 114 }; 115 struct perf_buffer *pb = NULL; 116 struct runqslower_bpf *obj; 117 int err; 118 119 err = argp_parse(&argp, argc, argv, 0, NULL, NULL); 120 if (err) 121 return err; 122 123 libbpf_set_print(libbpf_print_fn); 124 125 /* Use libbpf 1.0 API mode */ 126 libbpf_set_strict_mode(LIBBPF_STRICT_ALL); 127 128 obj = runqslower_bpf__open(); 129 if (!obj) { 130 fprintf(stderr, "failed to open and/or load BPF object\n"); 131 return 1; 132 } 133 134 /* initialize global data (filtering options) */ 135 obj->rodata->targ_pid = env.pid; 136 obj->rodata->min_us = env.min_us; 137 138 err = runqslower_bpf__load(obj); 139 if (err) { 140 fprintf(stderr, "failed to load BPF object: %d\n", err); 141 goto cleanup; 142 } 143 144 err = runqslower_bpf__attach(obj); 145 if (err) { 146 fprintf(stderr, "failed to attach BPF programs\n"); 147 goto cleanup; 148 } 149 150 printf("Tracing run queue latency higher than %llu us\n", env.min_us); 151 printf("%-8s %-16s %-6s %14s\n", "TIME", "COMM", "PID", "LAT(us)"); 152 153 pb = perf_buffer__new(bpf_map__fd(obj->maps.events), 64, 154 handle_event, handle_lost_events, NULL, NULL); 155 err = libbpf_get_error(pb); 156 if (err) { 157 pb = NULL; 158 fprintf(stderr, "failed to open perf buffer: %d\n", err); 159 goto cleanup; 160 } 161 162 while ((err = perf_buffer__poll(pb, 100)) >= 0) 163 ; 164 printf("Error polling perf buffer: %d\n", err); 165 166 cleanup: 167 perf_buffer__free(pb); 168 runqslower_bpf__destroy(obj); 169 170 return err != 0; 171 } 172