1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2022 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_core_read.h> 7 8 #include "lock_data.h" 9 10 /* default buffer size */ 11 #define MAX_ENTRIES 10240 12 13 struct tstamp_data { 14 __u64 timestamp; 15 __u64 lock; 16 __u32 flags; 17 __s32 stack_id; 18 }; 19 20 /* callstack storage */ 21 struct { 22 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 23 __uint(key_size, sizeof(__u32)); 24 __uint(value_size, sizeof(__u64)); 25 __uint(max_entries, MAX_ENTRIES); 26 } stacks SEC(".maps"); 27 28 /* maintain timestamp at the beginning of contention */ 29 struct { 30 __uint(type, BPF_MAP_TYPE_HASH); 31 __type(key, int); 32 __type(value, struct tstamp_data); 33 __uint(max_entries, MAX_ENTRIES); 34 } tstamp SEC(".maps"); 35 36 /* actual lock contention statistics */ 37 struct { 38 __uint(type, BPF_MAP_TYPE_HASH); 39 __uint(key_size, sizeof(struct contention_key)); 40 __uint(value_size, sizeof(struct contention_data)); 41 __uint(max_entries, MAX_ENTRIES); 42 } lock_stat SEC(".maps"); 43 44 struct { 45 __uint(type, BPF_MAP_TYPE_HASH); 46 __uint(key_size, sizeof(__u32)); 47 __uint(value_size, sizeof(struct contention_task_data)); 48 __uint(max_entries, MAX_ENTRIES); 49 } task_data SEC(".maps"); 50 51 struct { 52 __uint(type, BPF_MAP_TYPE_HASH); 53 __uint(key_size, sizeof(__u32)); 54 __uint(value_size, sizeof(__u8)); 55 __uint(max_entries, 1); 56 } cpu_filter SEC(".maps"); 57 58 struct { 59 __uint(type, BPF_MAP_TYPE_HASH); 60 __uint(key_size, sizeof(__u32)); 61 __uint(value_size, sizeof(__u8)); 62 __uint(max_entries, 1); 63 } task_filter SEC(".maps"); 64 65 /* control flags */ 66 int enabled; 67 int has_cpu; 68 int has_task; 69 int stack_skip; 70 71 /* determine the key of lock stat */ 72 int aggr_mode; 73 74 /* error stat */ 75 int lost; 76 77 static inline int can_record(void) 78 { 79 if (has_cpu) { 80 __u32 cpu = bpf_get_smp_processor_id(); 81 __u8 *ok; 82 83 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 84 if (!ok) 85 return 0; 86 } 87 88 if (has_task) { 89 __u8 *ok; 90 __u32 pid = bpf_get_current_pid_tgid(); 91 92 ok = bpf_map_lookup_elem(&task_filter, &pid); 93 if (!ok) 94 return 0; 95 } 96 97 return 1; 98 } 99 100 static inline void update_task_data(__u32 pid) 101 { 102 struct contention_task_data *p; 103 104 p = bpf_map_lookup_elem(&task_data, &pid); 105 if (p == NULL) { 106 struct contention_task_data data; 107 108 bpf_get_current_comm(data.comm, sizeof(data.comm)); 109 bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST); 110 } 111 } 112 113 SEC("tp_btf/contention_begin") 114 int contention_begin(u64 *ctx) 115 { 116 __u32 pid; 117 struct tstamp_data *pelem; 118 119 if (!enabled || !can_record()) 120 return 0; 121 122 pid = bpf_get_current_pid_tgid(); 123 pelem = bpf_map_lookup_elem(&tstamp, &pid); 124 if (pelem && pelem->lock) 125 return 0; 126 127 if (pelem == NULL) { 128 struct tstamp_data zero = {}; 129 130 bpf_map_update_elem(&tstamp, &pid, &zero, BPF_ANY); 131 pelem = bpf_map_lookup_elem(&tstamp, &pid); 132 if (pelem == NULL) { 133 lost++; 134 return 0; 135 } 136 } 137 138 pelem->timestamp = bpf_ktime_get_ns(); 139 pelem->lock = (__u64)ctx[0]; 140 pelem->flags = (__u32)ctx[1]; 141 142 if (aggr_mode == LOCK_AGGR_CALLER) { 143 pelem->stack_id = bpf_get_stackid(ctx, &stacks, 144 BPF_F_FAST_STACK_CMP | stack_skip); 145 if (pelem->stack_id < 0) 146 lost++; 147 } 148 149 return 0; 150 } 151 152 SEC("tp_btf/contention_end") 153 int contention_end(u64 *ctx) 154 { 155 __u32 pid; 156 struct tstamp_data *pelem; 157 struct contention_key key; 158 struct contention_data *data; 159 __u64 duration; 160 161 if (!enabled) 162 return 0; 163 164 pid = bpf_get_current_pid_tgid(); 165 pelem = bpf_map_lookup_elem(&tstamp, &pid); 166 if (!pelem || pelem->lock != ctx[0]) 167 return 0; 168 169 duration = bpf_ktime_get_ns() - pelem->timestamp; 170 171 switch (aggr_mode) { 172 case LOCK_AGGR_CALLER: 173 key.aggr_key = pelem->stack_id; 174 break; 175 case LOCK_AGGR_TASK: 176 key.aggr_key = pid; 177 update_task_data(pid); 178 break; 179 case LOCK_AGGR_ADDR: 180 key.aggr_key = pelem->lock; 181 break; 182 default: 183 /* should not happen */ 184 return 0; 185 } 186 187 data = bpf_map_lookup_elem(&lock_stat, &key); 188 if (!data) { 189 struct contention_data first = { 190 .total_time = duration, 191 .max_time = duration, 192 .min_time = duration, 193 .count = 1, 194 .flags = pelem->flags, 195 }; 196 197 bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); 198 bpf_map_delete_elem(&tstamp, &pid); 199 return 0; 200 } 201 202 __sync_fetch_and_add(&data->total_time, duration); 203 __sync_fetch_and_add(&data->count, 1); 204 205 /* FIXME: need atomic operations */ 206 if (data->max_time < duration) 207 data->max_time = duration; 208 if (data->min_time > duration) 209 data->min_time = duration; 210 211 bpf_map_delete_elem(&tstamp, &pid); 212 return 0; 213 } 214 215 char LICENSE[] SEC("license") = "Dual BSD/GPL"; 216