1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (c) 2021 Google
3 #include "vmlinux.h"
4 #include <bpf/bpf_helpers.h>
5 #include <bpf/bpf_tracing.h>
6
7 // This should be in sync with "util/ftrace.h"
8 #define NUM_BUCKET 22
9
10 struct {
11 __uint(type, BPF_MAP_TYPE_HASH);
12 __uint(key_size, sizeof(__u64));
13 __uint(value_size, sizeof(__u64));
14 __uint(max_entries, 10000);
15 } functime SEC(".maps");
16
17 struct {
18 __uint(type, BPF_MAP_TYPE_HASH);
19 __uint(key_size, sizeof(__u32));
20 __uint(value_size, sizeof(__u8));
21 __uint(max_entries, 1);
22 } cpu_filter SEC(".maps");
23
24 struct {
25 __uint(type, BPF_MAP_TYPE_HASH);
26 __uint(key_size, sizeof(__u32));
27 __uint(value_size, sizeof(__u8));
28 __uint(max_entries, 1);
29 } task_filter SEC(".maps");
30
31 struct {
32 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
33 __uint(key_size, sizeof(__u32));
34 __uint(value_size, sizeof(__u64));
35 __uint(max_entries, NUM_BUCKET);
36 } latency SEC(".maps");
37
38
39 int enabled = 0;
40
41 // stats
42 __s64 total;
43 __s64 count;
44 __s64 max;
45 __s64 min;
46
47 const volatile int has_cpu = 0;
48 const volatile int has_task = 0;
49 const volatile int use_nsec = 0;
50 const volatile unsigned int bucket_range;
51 const volatile unsigned int min_latency;
52 const volatile unsigned int max_latency;
53 const volatile unsigned int bucket_num = NUM_BUCKET;
54
can_record(void)55 static bool can_record(void)
56 {
57 if (has_cpu) {
58 __u32 cpu = bpf_get_smp_processor_id();
59 __u8 *ok;
60
61 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
62 if (!ok)
63 return false;
64 }
65
66 if (has_task) {
67 __u32 pid = bpf_get_current_pid_tgid();
68 __u8 *ok;
69
70 ok = bpf_map_lookup_elem(&task_filter, &pid);
71 if (!ok)
72 return false;
73 }
74 return true;
75 }
76
update_latency(__s64 delta)77 static void update_latency(__s64 delta)
78 {
79 __u64 val = delta;
80 __u32 key = 0;
81 __u64 *hist;
82 __u64 cmp_base = use_nsec ? 1 : 1000;
83
84 if (delta < 0)
85 return;
86
87 if (bucket_range != 0) {
88 val = delta / cmp_base;
89
90 if (min_latency > 0) {
91 if (val > min_latency)
92 val -= min_latency;
93 else
94 goto do_lookup;
95 }
96
97 // Less than 1 unit (ms or ns), or, in the future,
98 // than the min latency desired.
99 if (val > 0) { // 1st entry: [ 1 unit .. bucket_range units )
100 key = val / bucket_range + 1;
101 if (key >= bucket_num)
102 key = bucket_num - 1;
103 }
104
105 goto do_lookup;
106 }
107 // calculate index using delta
108 for (key = 0; key < (bucket_num - 1); key++) {
109 if (delta < (cmp_base << key))
110 break;
111 }
112
113 do_lookup:
114 hist = bpf_map_lookup_elem(&latency, &key);
115 if (!hist)
116 return;
117
118 __sync_fetch_and_add(hist, 1);
119
120 __sync_fetch_and_add(&total, delta); // always in nsec
121 __sync_fetch_and_add(&count, 1);
122
123 if (delta > max)
124 max = delta;
125 if (delta < min)
126 min = delta;
127 }
128
129 SEC("kprobe/func")
BPF_PROG(func_begin)130 int BPF_PROG(func_begin)
131 {
132 __u64 key, now;
133
134 if (!enabled || !can_record())
135 return 0;
136
137 key = bpf_get_current_pid_tgid();
138 now = bpf_ktime_get_ns();
139
140 // overwrite timestamp for nested functions
141 bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
142 return 0;
143 }
144
145 SEC("kretprobe/func")
BPF_PROG(func_end)146 int BPF_PROG(func_end)
147 {
148 __u64 tid;
149 __u64 *start;
150
151 if (!enabled)
152 return 0;
153
154 tid = bpf_get_current_pid_tgid();
155
156 start = bpf_map_lookup_elem(&functime, &tid);
157 if (start) {
158 update_latency(bpf_ktime_get_ns() - *start);
159 bpf_map_delete_elem(&functime, &tid);
160 }
161
162 return 0;
163 }
164
165 SEC("raw_tp")
BPF_PROG(event_begin)166 int BPF_PROG(event_begin)
167 {
168 __u64 key, now;
169
170 if (!enabled || !can_record())
171 return 0;
172
173 key = bpf_get_current_pid_tgid();
174 now = bpf_ktime_get_ns();
175
176 // overwrite timestamp for nested events
177 bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
178 return 0;
179 }
180
181 SEC("raw_tp")
BPF_PROG(event_end)182 int BPF_PROG(event_end)
183 {
184 __u64 tid;
185 __u64 *start;
186
187 if (!enabled)
188 return 0;
189
190 tid = bpf_get_current_pid_tgid();
191
192 start = bpf_map_lookup_elem(&functime, &tid);
193 if (start) {
194 update_latency(bpf_ktime_get_ns() - *start);
195 bpf_map_delete_elem(&functime, &tid);
196 }
197
198 return 0;
199 }
200