1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2021 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 7 // This should be in sync with "util/ftrace.h" 8 #define NUM_BUCKET 22 9 10 struct { 11 __uint(type, BPF_MAP_TYPE_HASH); 12 __uint(key_size, sizeof(__u64)); 13 __uint(value_size, sizeof(__u64)); 14 __uint(max_entries, 10000); 15 } functime SEC(".maps"); 16 17 struct { 18 __uint(type, BPF_MAP_TYPE_HASH); 19 __uint(key_size, sizeof(__u32)); 20 __uint(value_size, sizeof(__u8)); 21 __uint(max_entries, 1); 22 } cpu_filter SEC(".maps"); 23 24 struct { 25 __uint(type, BPF_MAP_TYPE_HASH); 26 __uint(key_size, sizeof(__u32)); 27 __uint(value_size, sizeof(__u8)); 28 __uint(max_entries, 1); 29 } task_filter SEC(".maps"); 30 31 struct { 32 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 33 __uint(key_size, sizeof(__u32)); 34 __uint(value_size, sizeof(__u64)); 35 __uint(max_entries, NUM_BUCKET); 36 } latency SEC(".maps"); 37 38 39 int enabled = 0; 40 41 // stats 42 __s64 total; 43 __s64 count; 44 __s64 max; 45 __s64 min; 46 47 const volatile int has_cpu = 0; 48 const volatile int has_task = 0; 49 const volatile int use_nsec = 0; 50 const volatile unsigned int bucket_range; 51 const volatile unsigned int min_latency; 52 const volatile unsigned int max_latency; 53 54 SEC("kprobe/func") 55 int BPF_PROG(func_begin) 56 { 57 __u64 key, now; 58 59 if (!enabled) 60 return 0; 61 62 key = bpf_get_current_pid_tgid(); 63 64 if (has_cpu) { 65 __u32 cpu = bpf_get_smp_processor_id(); 66 __u8 *ok; 67 68 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 69 if (!ok) 70 return 0; 71 } 72 73 if (has_task) { 74 __u32 pid = key & 0xffffffff; 75 __u8 *ok; 76 77 ok = bpf_map_lookup_elem(&task_filter, &pid); 78 if (!ok) 79 return 0; 80 } 81 82 now = bpf_ktime_get_ns(); 83 84 // overwrite timestamp for nested functions 85 bpf_map_update_elem(&functime, &key, &now, BPF_ANY); 86 return 0; 87 } 88 89 SEC("kretprobe/func") 90 int BPF_PROG(func_end) 91 { 92 __u64 tid; 93 __u64 *start; 94 __u64 cmp_base = use_nsec ? 1 : 1000; 95 96 if (!enabled) 97 return 0; 98 99 tid = bpf_get_current_pid_tgid(); 100 101 start = bpf_map_lookup_elem(&functime, &tid); 102 if (start) { 103 __s64 delta = bpf_ktime_get_ns() - *start; 104 __u32 key = 0; 105 __u64 *hist; 106 107 bpf_map_delete_elem(&functime, &tid); 108 109 if (delta < 0) 110 return 0; 111 112 if (bucket_range != 0) { 113 delta /= cmp_base; 114 115 if (min_latency > 0) { 116 if (delta > min_latency) 117 delta -= min_latency; 118 else 119 goto do_lookup; 120 } 121 122 // Less than 1 unit (ms or ns), or, in the future, 123 // than the min latency desired. 124 if (delta > 0) { // 1st entry: [ 1 unit .. bucket_range units ) 125 // clang 12 doesn't like s64 / u32 division 126 key = (__u64)delta / bucket_range + 1; 127 if (key >= NUM_BUCKET || 128 delta >= max_latency - min_latency) 129 key = NUM_BUCKET - 1; 130 } 131 132 delta += min_latency; 133 goto do_lookup; 134 } 135 // calculate index using delta 136 for (key = 0; key < (NUM_BUCKET - 1); key++) { 137 if (delta < (cmp_base << key)) 138 break; 139 } 140 141 do_lookup: 142 hist = bpf_map_lookup_elem(&latency, &key); 143 if (!hist) 144 return 0; 145 146 *hist += 1; 147 148 if (bucket_range == 0) 149 delta /= cmp_base; 150 151 __sync_fetch_and_add(&total, delta); 152 __sync_fetch_and_add(&count, 1); 153 154 if (delta > max) 155 max = delta; 156 if (delta < min) 157 min = delta; 158 } 159 160 return 0; 161 } 162