1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2021 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 7 // This should be in sync with "util/ftrace.h" 8 #define NUM_BUCKET 22 9 10 struct { 11 __uint(type, BPF_MAP_TYPE_HASH); 12 __uint(key_size, sizeof(__u64)); 13 __uint(value_size, sizeof(__u64)); 14 __uint(max_entries, 10000); 15 } functime SEC(".maps"); 16 17 struct { 18 __uint(type, BPF_MAP_TYPE_HASH); 19 __uint(key_size, sizeof(__u32)); 20 __uint(value_size, sizeof(__u8)); 21 __uint(max_entries, 1); 22 } cpu_filter SEC(".maps"); 23 24 struct { 25 __uint(type, BPF_MAP_TYPE_HASH); 26 __uint(key_size, sizeof(__u32)); 27 __uint(value_size, sizeof(__u8)); 28 __uint(max_entries, 1); 29 } task_filter SEC(".maps"); 30 31 struct { 32 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 33 __uint(key_size, sizeof(__u32)); 34 __uint(value_size, sizeof(__u64)); 35 __uint(max_entries, NUM_BUCKET); 36 } latency SEC(".maps"); 37 38 39 int enabled = 0; 40 41 // stats 42 __s64 total; 43 __s64 count; 44 __s64 max; 45 __s64 min; 46 47 const volatile int has_cpu = 0; 48 const volatile int has_task = 0; 49 const volatile int use_nsec = 0; 50 const volatile unsigned int bucket_range; 51 const volatile unsigned int min_latency; 52 const volatile unsigned int max_latency; 53 const volatile unsigned int bucket_num = NUM_BUCKET; 54 55 static bool can_record(void) 56 { 57 if (has_cpu) { 58 __u32 cpu = bpf_get_smp_processor_id(); 59 __u8 *ok; 60 61 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 62 if (!ok) 63 return false; 64 } 65 66 if (has_task) { 67 __u32 pid = bpf_get_current_pid_tgid(); 68 __u8 *ok; 69 70 ok = bpf_map_lookup_elem(&task_filter, &pid); 71 if (!ok) 72 return false; 73 } 74 return true; 75 } 76 77 static void update_latency(__s64 delta) 78 { 79 __u64 val = delta; 80 __u32 key = 0; 81 __u64 *hist; 82 __u64 cmp_base = use_nsec ? 1 : 1000; 83 84 if (delta < 0) 85 return; 86 87 if (bucket_range != 0) { 88 val = delta / cmp_base; 89 90 if (min_latency > 0) { 91 if (val > min_latency) 92 val -= min_latency; 93 else 94 goto do_lookup; 95 } 96 97 // Less than 1 unit (ms or ns), or, in the future, 98 // than the min latency desired. 99 if (val > 0) { // 1st entry: [ 1 unit .. bucket_range units ) 100 key = val / bucket_range + 1; 101 if (key >= bucket_num) 102 key = bucket_num - 1; 103 } 104 105 goto do_lookup; 106 } 107 // calculate index using delta 108 for (key = 0; key < (bucket_num - 1); key++) { 109 if (delta < (cmp_base << key)) 110 break; 111 } 112 113 do_lookup: 114 hist = bpf_map_lookup_elem(&latency, &key); 115 if (!hist) 116 return; 117 118 __sync_fetch_and_add(hist, 1); 119 120 __sync_fetch_and_add(&total, delta); // always in nsec 121 __sync_fetch_and_add(&count, 1); 122 123 if (delta > max) 124 max = delta; 125 if (delta < min) 126 min = delta; 127 } 128 129 SEC("kprobe/func") 130 int BPF_PROG(func_begin) 131 { 132 __u64 key, now; 133 134 if (!enabled || !can_record()) 135 return 0; 136 137 key = bpf_get_current_pid_tgid(); 138 now = bpf_ktime_get_ns(); 139 140 // overwrite timestamp for nested functions 141 bpf_map_update_elem(&functime, &key, &now, BPF_ANY); 142 return 0; 143 } 144 145 SEC("kretprobe/func") 146 int BPF_PROG(func_end) 147 { 148 __u64 tid; 149 __u64 *start; 150 151 if (!enabled) 152 return 0; 153 154 tid = bpf_get_current_pid_tgid(); 155 156 start = bpf_map_lookup_elem(&functime, &tid); 157 if (start) { 158 update_latency(bpf_ktime_get_ns() - *start); 159 bpf_map_delete_elem(&functime, &tid); 160 } 161 162 return 0; 163 } 164 165 SEC("raw_tp") 166 int BPF_PROG(event_begin) 167 { 168 __u64 key, now; 169 170 if (!enabled || !can_record()) 171 return 0; 172 173 key = bpf_get_current_pid_tgid(); 174 now = bpf_ktime_get_ns(); 175 176 // overwrite timestamp for nested events 177 bpf_map_update_elem(&functime, &key, &now, BPF_ANY); 178 return 0; 179 } 180 181 SEC("raw_tp") 182 int BPF_PROG(event_end) 183 { 184 __u64 tid; 185 __u64 *start; 186 187 if (!enabled) 188 return 0; 189 190 tid = bpf_get_current_pid_tgid(); 191 192 start = bpf_map_lookup_elem(&functime, &tid); 193 if (start) { 194 update_latency(bpf_ktime_get_ns() - *start); 195 bpf_map_delete_elem(&functime, &tid); 196 } 197 198 return 0; 199 } 200