1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2022 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_core_read.h> 7 #include <asm-generic/errno-base.h> 8 9 #include "lock_data.h" 10 11 /* for collect_lock_syms(). 4096 was rejected by the verifier */ 12 #define MAX_CPUS 1024 13 14 /* lock contention flags from include/trace/events/lock.h */ 15 #define LCB_F_SPIN (1U << 0) 16 #define LCB_F_READ (1U << 1) 17 #define LCB_F_WRITE (1U << 2) 18 #define LCB_F_RT (1U << 3) 19 #define LCB_F_PERCPU (1U << 4) 20 #define LCB_F_MUTEX (1U << 5) 21 22 /* callstack storage */ 23 struct { 24 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 25 __uint(key_size, sizeof(__u32)); 26 __uint(value_size, sizeof(__u64)); 27 __uint(max_entries, MAX_ENTRIES); 28 } stacks SEC(".maps"); 29 30 /* maintain timestamp at the beginning of contention */ 31 struct { 32 __uint(type, BPF_MAP_TYPE_HASH); 33 __type(key, int); 34 __type(value, struct tstamp_data); 35 __uint(max_entries, MAX_ENTRIES); 36 } tstamp SEC(".maps"); 37 38 /* maintain per-CPU timestamp at the beginning of contention */ 39 struct { 40 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 41 __uint(key_size, sizeof(__u32)); 42 __uint(value_size, sizeof(struct tstamp_data)); 43 __uint(max_entries, 1); 44 } tstamp_cpu SEC(".maps"); 45 46 /* actual lock contention statistics */ 47 struct { 48 __uint(type, BPF_MAP_TYPE_HASH); 49 __uint(key_size, sizeof(struct contention_key)); 50 __uint(value_size, sizeof(struct contention_data)); 51 __uint(max_entries, MAX_ENTRIES); 52 } lock_stat SEC(".maps"); 53 54 struct { 55 __uint(type, BPF_MAP_TYPE_HASH); 56 __uint(key_size, sizeof(__u32)); 57 __uint(value_size, sizeof(struct contention_task_data)); 58 __uint(max_entries, MAX_ENTRIES); 59 } task_data SEC(".maps"); 60 61 struct { 62 __uint(type, BPF_MAP_TYPE_HASH); 63 __uint(key_size, sizeof(__u64)); 64 __uint(value_size, sizeof(__u32)); 65 __uint(max_entries, MAX_ENTRIES); 66 } lock_syms SEC(".maps"); 67 68 struct { 69 __uint(type, BPF_MAP_TYPE_HASH); 70 __uint(key_size, sizeof(__u32)); 71 __uint(value_size, sizeof(__u8)); 72 __uint(max_entries, 1); 73 } cpu_filter SEC(".maps"); 74 75 struct { 76 __uint(type, BPF_MAP_TYPE_HASH); 77 __uint(key_size, sizeof(__u32)); 78 __uint(value_size, sizeof(__u8)); 79 __uint(max_entries, 1); 80 } task_filter SEC(".maps"); 81 82 struct { 83 __uint(type, BPF_MAP_TYPE_HASH); 84 __uint(key_size, sizeof(__u32)); 85 __uint(value_size, sizeof(__u8)); 86 __uint(max_entries, 1); 87 } type_filter SEC(".maps"); 88 89 struct { 90 __uint(type, BPF_MAP_TYPE_HASH); 91 __uint(key_size, sizeof(__u64)); 92 __uint(value_size, sizeof(__u8)); 93 __uint(max_entries, 1); 94 } addr_filter SEC(".maps"); 95 96 struct { 97 __uint(type, BPF_MAP_TYPE_HASH); 98 __uint(key_size, sizeof(__u64)); 99 __uint(value_size, sizeof(__u8)); 100 __uint(max_entries, 1); 101 } cgroup_filter SEC(".maps"); 102 103 struct rw_semaphore___old { 104 struct task_struct *owner; 105 } __attribute__((preserve_access_index)); 106 107 struct rw_semaphore___new { 108 atomic_long_t owner; 109 } __attribute__((preserve_access_index)); 110 111 struct mm_struct___old { 112 struct rw_semaphore mmap_sem; 113 } __attribute__((preserve_access_index)); 114 115 struct mm_struct___new { 116 struct rw_semaphore mmap_lock; 117 } __attribute__((preserve_access_index)); 118 119 /* control flags */ 120 int enabled; 121 int has_cpu; 122 int has_task; 123 int has_type; 124 int has_addr; 125 int has_cgroup; 126 int needs_callstack; 127 int stack_skip; 128 int lock_owner; 129 130 int use_cgroup_v2; 131 int perf_subsys_id = -1; 132 133 /* determine the key of lock stat */ 134 int aggr_mode; 135 136 __u64 end_ts; 137 138 /* error stat */ 139 int task_fail; 140 int stack_fail; 141 int time_fail; 142 int data_fail; 143 144 int task_map_full; 145 int data_map_full; 146 147 static inline __u64 get_current_cgroup_id(void) 148 { 149 struct task_struct *task; 150 struct cgroup *cgrp; 151 152 if (use_cgroup_v2) 153 return bpf_get_current_cgroup_id(); 154 155 task = bpf_get_current_task_btf(); 156 157 if (perf_subsys_id == -1) { 158 #if __has_builtin(__builtin_preserve_enum_value) 159 perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, 160 perf_event_cgrp_id); 161 #else 162 perf_subsys_id = perf_event_cgrp_id; 163 #endif 164 } 165 166 cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); 167 return BPF_CORE_READ(cgrp, kn, id); 168 } 169 170 static inline int can_record(u64 *ctx) 171 { 172 if (has_cpu) { 173 __u32 cpu = bpf_get_smp_processor_id(); 174 __u8 *ok; 175 176 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 177 if (!ok) 178 return 0; 179 } 180 181 if (has_task) { 182 __u8 *ok; 183 __u32 pid = bpf_get_current_pid_tgid(); 184 185 ok = bpf_map_lookup_elem(&task_filter, &pid); 186 if (!ok) 187 return 0; 188 } 189 190 if (has_type) { 191 __u8 *ok; 192 __u32 flags = (__u32)ctx[1]; 193 194 ok = bpf_map_lookup_elem(&type_filter, &flags); 195 if (!ok) 196 return 0; 197 } 198 199 if (has_addr) { 200 __u8 *ok; 201 __u64 addr = ctx[0]; 202 203 ok = bpf_map_lookup_elem(&addr_filter, &addr); 204 if (!ok) 205 return 0; 206 } 207 208 if (has_cgroup) { 209 __u8 *ok; 210 __u64 cgrp = get_current_cgroup_id(); 211 212 ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp); 213 if (!ok) 214 return 0; 215 } 216 217 return 1; 218 } 219 220 static inline int update_task_data(struct task_struct *task) 221 { 222 struct contention_task_data *p; 223 int pid, err; 224 225 err = bpf_core_read(&pid, sizeof(pid), &task->pid); 226 if (err) 227 return -1; 228 229 p = bpf_map_lookup_elem(&task_data, &pid); 230 if (p == NULL && !task_map_full) { 231 struct contention_task_data data = {}; 232 233 BPF_CORE_READ_STR_INTO(&data.comm, task, comm); 234 if (bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST) == -E2BIG) 235 task_map_full = 1; 236 } 237 238 return 0; 239 } 240 241 #ifndef __has_builtin 242 # define __has_builtin(x) 0 243 #endif 244 245 static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags) 246 { 247 struct task_struct *task; 248 __u64 owner = 0; 249 250 if (flags & LCB_F_MUTEX) { 251 struct mutex *mutex = (void *)lock; 252 owner = BPF_CORE_READ(mutex, owner.counter); 253 } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) { 254 /* 255 * Support for the BPF_TYPE_MATCHES argument to the 256 * __builtin_preserve_type_info builtin was added at some point during 257 * development of clang 15 and it's what is needed for 258 * bpf_core_type_matches. 259 */ 260 #if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15 261 if (bpf_core_type_matches(struct rw_semaphore___old)) { 262 struct rw_semaphore___old *rwsem = (void *)lock; 263 owner = (unsigned long)BPF_CORE_READ(rwsem, owner); 264 } else if (bpf_core_type_matches(struct rw_semaphore___new)) { 265 struct rw_semaphore___new *rwsem = (void *)lock; 266 owner = BPF_CORE_READ(rwsem, owner.counter); 267 } 268 #else 269 /* assume new struct */ 270 struct rw_semaphore *rwsem = (void *)lock; 271 owner = BPF_CORE_READ(rwsem, owner.counter); 272 #endif 273 } 274 275 if (!owner) 276 return NULL; 277 278 task = (void *)(owner & ~7UL); 279 return task; 280 } 281 282 static inline __u32 check_lock_type(__u64 lock, __u32 flags) 283 { 284 struct task_struct *curr; 285 struct mm_struct___old *mm_old; 286 struct mm_struct___new *mm_new; 287 struct sighand_struct *sighand; 288 289 switch (flags) { 290 case LCB_F_READ: /* rwsem */ 291 case LCB_F_WRITE: 292 curr = bpf_get_current_task_btf(); 293 if (curr->mm == NULL) 294 break; 295 mm_new = (void *)curr->mm; 296 if (bpf_core_field_exists(mm_new->mmap_lock)) { 297 if (&mm_new->mmap_lock == (void *)lock) 298 return LCD_F_MMAP_LOCK; 299 break; 300 } 301 mm_old = (void *)curr->mm; 302 if (bpf_core_field_exists(mm_old->mmap_sem)) { 303 if (&mm_old->mmap_sem == (void *)lock) 304 return LCD_F_MMAP_LOCK; 305 } 306 break; 307 case LCB_F_SPIN: /* spinlock */ 308 curr = bpf_get_current_task_btf(); 309 sighand = curr->sighand; 310 311 if (sighand && &sighand->siglock == (void *)lock) 312 return LCD_F_SIGHAND_LOCK; 313 break; 314 default: 315 break; 316 } 317 return 0; 318 } 319 320 static inline struct tstamp_data *get_tstamp_elem(__u32 flags) 321 { 322 __u32 pid; 323 struct tstamp_data *pelem; 324 325 /* Use per-cpu array map for spinlock and rwlock */ 326 if ((flags & (LCB_F_SPIN | LCB_F_MUTEX)) == LCB_F_SPIN) { 327 __u32 idx = 0; 328 329 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); 330 /* Do not update the element for nested locks */ 331 if (pelem && pelem->lock) 332 pelem = NULL; 333 return pelem; 334 } 335 336 pid = bpf_get_current_pid_tgid(); 337 pelem = bpf_map_lookup_elem(&tstamp, &pid); 338 /* Do not update the element for nested locks */ 339 if (pelem && pelem->lock) 340 return NULL; 341 342 if (pelem == NULL) { 343 struct tstamp_data zero = {}; 344 345 if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) { 346 __sync_fetch_and_add(&task_fail, 1); 347 return NULL; 348 } 349 350 pelem = bpf_map_lookup_elem(&tstamp, &pid); 351 if (pelem == NULL) { 352 __sync_fetch_and_add(&task_fail, 1); 353 return NULL; 354 } 355 } 356 return pelem; 357 } 358 359 SEC("tp_btf/contention_begin") 360 int contention_begin(u64 *ctx) 361 { 362 struct tstamp_data *pelem; 363 364 if (!enabled || !can_record(ctx)) 365 return 0; 366 367 pelem = get_tstamp_elem(ctx[1]); 368 if (pelem == NULL) 369 return 0; 370 371 pelem->timestamp = bpf_ktime_get_ns(); 372 pelem->lock = (__u64)ctx[0]; 373 pelem->flags = (__u32)ctx[1]; 374 375 if (needs_callstack) { 376 pelem->stack_id = bpf_get_stackid(ctx, &stacks, 377 BPF_F_FAST_STACK_CMP | stack_skip); 378 if (pelem->stack_id < 0) 379 __sync_fetch_and_add(&stack_fail, 1); 380 } else if (aggr_mode == LOCK_AGGR_TASK) { 381 struct task_struct *task; 382 383 if (lock_owner) { 384 task = get_lock_owner(pelem->lock, pelem->flags); 385 386 /* The flags is not used anymore. Pass the owner pid. */ 387 if (task) 388 pelem->flags = BPF_CORE_READ(task, pid); 389 else 390 pelem->flags = -1U; 391 392 } else { 393 task = bpf_get_current_task_btf(); 394 } 395 396 if (task) { 397 if (update_task_data(task) < 0 && lock_owner) 398 pelem->flags = -1U; 399 } 400 } 401 402 return 0; 403 } 404 405 SEC("tp_btf/contention_end") 406 int contention_end(u64 *ctx) 407 { 408 __u32 pid = 0, idx = 0; 409 struct tstamp_data *pelem; 410 struct contention_key key = {}; 411 struct contention_data *data; 412 __u64 duration; 413 bool need_delete = false; 414 415 if (!enabled) 416 return 0; 417 418 /* 419 * For spinlock and rwlock, it needs to get the timestamp for the 420 * per-cpu map. However, contention_end does not have the flags 421 * so it cannot know whether it reads percpu or hash map. 422 * 423 * Try per-cpu map first and check if there's active contention. 424 * If it is, do not read hash map because it cannot go to sleeping 425 * locks before releasing the spinning locks. 426 */ 427 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); 428 if (pelem && pelem->lock) { 429 if (pelem->lock != ctx[0]) 430 return 0; 431 } else { 432 pid = bpf_get_current_pid_tgid(); 433 pelem = bpf_map_lookup_elem(&tstamp, &pid); 434 if (!pelem || pelem->lock != ctx[0]) 435 return 0; 436 need_delete = true; 437 } 438 439 duration = bpf_ktime_get_ns() - pelem->timestamp; 440 if ((__s64)duration < 0) { 441 __sync_fetch_and_add(&time_fail, 1); 442 goto out; 443 } 444 445 switch (aggr_mode) { 446 case LOCK_AGGR_CALLER: 447 key.stack_id = pelem->stack_id; 448 break; 449 case LOCK_AGGR_TASK: 450 if (lock_owner) 451 key.pid = pelem->flags; 452 else { 453 if (!need_delete) 454 pid = bpf_get_current_pid_tgid(); 455 key.pid = pid; 456 } 457 if (needs_callstack) 458 key.stack_id = pelem->stack_id; 459 break; 460 case LOCK_AGGR_ADDR: 461 key.lock_addr_or_cgroup = pelem->lock; 462 if (needs_callstack) 463 key.stack_id = pelem->stack_id; 464 break; 465 case LOCK_AGGR_CGROUP: 466 key.lock_addr_or_cgroup = get_current_cgroup_id(); 467 break; 468 default: 469 /* should not happen */ 470 return 0; 471 } 472 473 data = bpf_map_lookup_elem(&lock_stat, &key); 474 if (!data) { 475 if (data_map_full) { 476 __sync_fetch_and_add(&data_fail, 1); 477 goto out; 478 } 479 480 struct contention_data first = { 481 .total_time = duration, 482 .max_time = duration, 483 .min_time = duration, 484 .count = 1, 485 .flags = pelem->flags, 486 }; 487 int err; 488 489 if (aggr_mode == LOCK_AGGR_ADDR) 490 first.flags |= check_lock_type(pelem->lock, pelem->flags); 491 492 err = bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); 493 if (err < 0) { 494 if (err == -EEXIST) { 495 /* it lost the race, try to get it again */ 496 data = bpf_map_lookup_elem(&lock_stat, &key); 497 if (data != NULL) 498 goto found; 499 } 500 if (err == -E2BIG) 501 data_map_full = 1; 502 __sync_fetch_and_add(&data_fail, 1); 503 } 504 goto out; 505 } 506 507 found: 508 __sync_fetch_and_add(&data->total_time, duration); 509 __sync_fetch_and_add(&data->count, 1); 510 511 /* FIXME: need atomic operations */ 512 if (data->max_time < duration) 513 data->max_time = duration; 514 if (data->min_time > duration) 515 data->min_time = duration; 516 517 out: 518 pelem->lock = 0; 519 if (need_delete) 520 bpf_map_delete_elem(&tstamp, &pid); 521 return 0; 522 } 523 524 extern struct rq runqueues __ksym; 525 526 struct rq___old { 527 raw_spinlock_t lock; 528 } __attribute__((preserve_access_index)); 529 530 struct rq___new { 531 raw_spinlock_t __lock; 532 } __attribute__((preserve_access_index)); 533 534 SEC("raw_tp/bpf_test_finish") 535 int BPF_PROG(collect_lock_syms) 536 { 537 __u64 lock_addr, lock_off; 538 __u32 lock_flag; 539 540 if (bpf_core_field_exists(struct rq___new, __lock)) 541 lock_off = offsetof(struct rq___new, __lock); 542 else 543 lock_off = offsetof(struct rq___old, lock); 544 545 for (int i = 0; i < MAX_CPUS; i++) { 546 struct rq *rq = bpf_per_cpu_ptr(&runqueues, i); 547 548 if (rq == NULL) 549 break; 550 551 lock_addr = (__u64)(void *)rq + lock_off; 552 lock_flag = LOCK_CLASS_RQLOCK; 553 bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); 554 } 555 return 0; 556 } 557 558 SEC("raw_tp/bpf_test_finish") 559 int BPF_PROG(end_timestamp) 560 { 561 end_ts = bpf_ktime_get_ns(); 562 return 0; 563 } 564 565 char LICENSE[] SEC("license") = "Dual BSD/GPL"; 566