1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 // Copyright (c) 2022 Google 3 #include "vmlinux.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_core_read.h> 7 #include <asm-generic/errno-base.h> 8 9 #include "lock_data.h" 10 11 /* for collect_lock_syms(). 4096 was rejected by the verifier */ 12 #define MAX_CPUS 1024 13 14 /* lock contention flags from include/trace/events/lock.h */ 15 #define LCB_F_SPIN (1U << 0) 16 #define LCB_F_READ (1U << 1) 17 #define LCB_F_WRITE (1U << 2) 18 #define LCB_F_RT (1U << 3) 19 #define LCB_F_PERCPU (1U << 4) 20 #define LCB_F_MUTEX (1U << 5) 21 22 struct tstamp_data { 23 __u64 timestamp; 24 __u64 lock; 25 __u32 flags; 26 __s32 stack_id; 27 }; 28 29 /* callstack storage */ 30 struct { 31 __uint(type, BPF_MAP_TYPE_STACK_TRACE); 32 __uint(key_size, sizeof(__u32)); 33 __uint(value_size, sizeof(__u64)); 34 __uint(max_entries, MAX_ENTRIES); 35 } stacks SEC(".maps"); 36 37 /* maintain timestamp at the beginning of contention */ 38 struct { 39 __uint(type, BPF_MAP_TYPE_HASH); 40 __type(key, int); 41 __type(value, struct tstamp_data); 42 __uint(max_entries, MAX_ENTRIES); 43 } tstamp SEC(".maps"); 44 45 /* maintain per-CPU timestamp at the beginning of contention */ 46 struct { 47 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 48 __uint(key_size, sizeof(__u32)); 49 __uint(value_size, sizeof(struct tstamp_data)); 50 __uint(max_entries, 1); 51 } tstamp_cpu SEC(".maps"); 52 53 /* actual lock contention statistics */ 54 struct { 55 __uint(type, BPF_MAP_TYPE_HASH); 56 __uint(key_size, sizeof(struct contention_key)); 57 __uint(value_size, sizeof(struct contention_data)); 58 __uint(max_entries, MAX_ENTRIES); 59 } lock_stat SEC(".maps"); 60 61 struct { 62 __uint(type, BPF_MAP_TYPE_HASH); 63 __uint(key_size, sizeof(__u32)); 64 __uint(value_size, sizeof(struct contention_task_data)); 65 __uint(max_entries, MAX_ENTRIES); 66 } task_data SEC(".maps"); 67 68 struct { 69 __uint(type, BPF_MAP_TYPE_HASH); 70 __uint(key_size, sizeof(__u64)); 71 __uint(value_size, sizeof(__u32)); 72 __uint(max_entries, MAX_ENTRIES); 73 } lock_syms SEC(".maps"); 74 75 struct { 76 __uint(type, BPF_MAP_TYPE_HASH); 77 __uint(key_size, sizeof(__u32)); 78 __uint(value_size, sizeof(__u8)); 79 __uint(max_entries, 1); 80 } cpu_filter SEC(".maps"); 81 82 struct { 83 __uint(type, BPF_MAP_TYPE_HASH); 84 __uint(key_size, sizeof(__u32)); 85 __uint(value_size, sizeof(__u8)); 86 __uint(max_entries, 1); 87 } task_filter SEC(".maps"); 88 89 struct { 90 __uint(type, BPF_MAP_TYPE_HASH); 91 __uint(key_size, sizeof(__u32)); 92 __uint(value_size, sizeof(__u8)); 93 __uint(max_entries, 1); 94 } type_filter SEC(".maps"); 95 96 struct { 97 __uint(type, BPF_MAP_TYPE_HASH); 98 __uint(key_size, sizeof(__u64)); 99 __uint(value_size, sizeof(__u8)); 100 __uint(max_entries, 1); 101 } addr_filter SEC(".maps"); 102 103 struct { 104 __uint(type, BPF_MAP_TYPE_HASH); 105 __uint(key_size, sizeof(__u64)); 106 __uint(value_size, sizeof(__u8)); 107 __uint(max_entries, 1); 108 } cgroup_filter SEC(".maps"); 109 110 struct rw_semaphore___old { 111 struct task_struct *owner; 112 } __attribute__((preserve_access_index)); 113 114 struct rw_semaphore___new { 115 atomic_long_t owner; 116 } __attribute__((preserve_access_index)); 117 118 struct mm_struct___old { 119 struct rw_semaphore mmap_sem; 120 } __attribute__((preserve_access_index)); 121 122 struct mm_struct___new { 123 struct rw_semaphore mmap_lock; 124 } __attribute__((preserve_access_index)); 125 126 /* control flags */ 127 int enabled; 128 int has_cpu; 129 int has_task; 130 int has_type; 131 int has_addr; 132 int has_cgroup; 133 int needs_callstack; 134 int stack_skip; 135 int lock_owner; 136 137 int use_cgroup_v2; 138 int perf_subsys_id = -1; 139 140 /* determine the key of lock stat */ 141 int aggr_mode; 142 143 /* error stat */ 144 int task_fail; 145 int stack_fail; 146 int time_fail; 147 int data_fail; 148 149 int task_map_full; 150 int data_map_full; 151 152 static inline __u64 get_current_cgroup_id(void) 153 { 154 struct task_struct *task; 155 struct cgroup *cgrp; 156 157 if (use_cgroup_v2) 158 return bpf_get_current_cgroup_id(); 159 160 task = bpf_get_current_task_btf(); 161 162 if (perf_subsys_id == -1) { 163 #if __has_builtin(__builtin_preserve_enum_value) 164 perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, 165 perf_event_cgrp_id); 166 #else 167 perf_subsys_id = perf_event_cgrp_id; 168 #endif 169 } 170 171 cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); 172 return BPF_CORE_READ(cgrp, kn, id); 173 } 174 175 static inline int can_record(u64 *ctx) 176 { 177 if (has_cpu) { 178 __u32 cpu = bpf_get_smp_processor_id(); 179 __u8 *ok; 180 181 ok = bpf_map_lookup_elem(&cpu_filter, &cpu); 182 if (!ok) 183 return 0; 184 } 185 186 if (has_task) { 187 __u8 *ok; 188 __u32 pid = bpf_get_current_pid_tgid(); 189 190 ok = bpf_map_lookup_elem(&task_filter, &pid); 191 if (!ok) 192 return 0; 193 } 194 195 if (has_type) { 196 __u8 *ok; 197 __u32 flags = (__u32)ctx[1]; 198 199 ok = bpf_map_lookup_elem(&type_filter, &flags); 200 if (!ok) 201 return 0; 202 } 203 204 if (has_addr) { 205 __u8 *ok; 206 __u64 addr = ctx[0]; 207 208 ok = bpf_map_lookup_elem(&addr_filter, &addr); 209 if (!ok) 210 return 0; 211 } 212 213 if (has_cgroup) { 214 __u8 *ok; 215 __u64 cgrp = get_current_cgroup_id(); 216 217 ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp); 218 if (!ok) 219 return 0; 220 } 221 222 return 1; 223 } 224 225 static inline int update_task_data(struct task_struct *task) 226 { 227 struct contention_task_data *p; 228 int pid, err; 229 230 err = bpf_core_read(&pid, sizeof(pid), &task->pid); 231 if (err) 232 return -1; 233 234 p = bpf_map_lookup_elem(&task_data, &pid); 235 if (p == NULL && !task_map_full) { 236 struct contention_task_data data = {}; 237 238 BPF_CORE_READ_STR_INTO(&data.comm, task, comm); 239 if (bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST) == -E2BIG) 240 task_map_full = 1; 241 } 242 243 return 0; 244 } 245 246 #ifndef __has_builtin 247 # define __has_builtin(x) 0 248 #endif 249 250 static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags) 251 { 252 struct task_struct *task; 253 __u64 owner = 0; 254 255 if (flags & LCB_F_MUTEX) { 256 struct mutex *mutex = (void *)lock; 257 owner = BPF_CORE_READ(mutex, owner.counter); 258 } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) { 259 /* 260 * Support for the BPF_TYPE_MATCHES argument to the 261 * __builtin_preserve_type_info builtin was added at some point during 262 * development of clang 15 and it's what is needed for 263 * bpf_core_type_matches. 264 */ 265 #if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15 266 if (bpf_core_type_matches(struct rw_semaphore___old)) { 267 struct rw_semaphore___old *rwsem = (void *)lock; 268 owner = (unsigned long)BPF_CORE_READ(rwsem, owner); 269 } else if (bpf_core_type_matches(struct rw_semaphore___new)) { 270 struct rw_semaphore___new *rwsem = (void *)lock; 271 owner = BPF_CORE_READ(rwsem, owner.counter); 272 } 273 #else 274 /* assume new struct */ 275 struct rw_semaphore *rwsem = (void *)lock; 276 owner = BPF_CORE_READ(rwsem, owner.counter); 277 #endif 278 } 279 280 if (!owner) 281 return NULL; 282 283 task = (void *)(owner & ~7UL); 284 return task; 285 } 286 287 static inline __u32 check_lock_type(__u64 lock, __u32 flags) 288 { 289 struct task_struct *curr; 290 struct mm_struct___old *mm_old; 291 struct mm_struct___new *mm_new; 292 293 switch (flags) { 294 case LCB_F_READ: /* rwsem */ 295 case LCB_F_WRITE: 296 curr = bpf_get_current_task_btf(); 297 if (curr->mm == NULL) 298 break; 299 mm_new = (void *)curr->mm; 300 if (bpf_core_field_exists(mm_new->mmap_lock)) { 301 if (&mm_new->mmap_lock == (void *)lock) 302 return LCD_F_MMAP_LOCK; 303 break; 304 } 305 mm_old = (void *)curr->mm; 306 if (bpf_core_field_exists(mm_old->mmap_sem)) { 307 if (&mm_old->mmap_sem == (void *)lock) 308 return LCD_F_MMAP_LOCK; 309 } 310 break; 311 case LCB_F_SPIN: /* spinlock */ 312 curr = bpf_get_current_task_btf(); 313 if (&curr->sighand->siglock == (void *)lock) 314 return LCD_F_SIGHAND_LOCK; 315 break; 316 default: 317 break; 318 } 319 return 0; 320 } 321 322 static inline struct tstamp_data *get_tstamp_elem(__u32 flags) 323 { 324 __u32 pid; 325 struct tstamp_data *pelem; 326 327 /* Use per-cpu array map for spinlock and rwlock */ 328 if (flags == (LCB_F_SPIN | LCB_F_READ) || flags == LCB_F_SPIN || 329 flags == (LCB_F_SPIN | LCB_F_WRITE)) { 330 __u32 idx = 0; 331 332 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); 333 /* Do not update the element for nested locks */ 334 if (pelem && pelem->lock) 335 pelem = NULL; 336 return pelem; 337 } 338 339 pid = bpf_get_current_pid_tgid(); 340 pelem = bpf_map_lookup_elem(&tstamp, &pid); 341 /* Do not update the element for nested locks */ 342 if (pelem && pelem->lock) 343 return NULL; 344 345 if (pelem == NULL) { 346 struct tstamp_data zero = {}; 347 348 if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) { 349 __sync_fetch_and_add(&task_fail, 1); 350 return NULL; 351 } 352 353 pelem = bpf_map_lookup_elem(&tstamp, &pid); 354 if (pelem == NULL) { 355 __sync_fetch_and_add(&task_fail, 1); 356 return NULL; 357 } 358 } 359 return pelem; 360 } 361 362 SEC("tp_btf/contention_begin") 363 int contention_begin(u64 *ctx) 364 { 365 struct tstamp_data *pelem; 366 367 if (!enabled || !can_record(ctx)) 368 return 0; 369 370 pelem = get_tstamp_elem(ctx[1]); 371 if (pelem == NULL) 372 return 0; 373 374 pelem->timestamp = bpf_ktime_get_ns(); 375 pelem->lock = (__u64)ctx[0]; 376 pelem->flags = (__u32)ctx[1]; 377 378 if (needs_callstack) { 379 pelem->stack_id = bpf_get_stackid(ctx, &stacks, 380 BPF_F_FAST_STACK_CMP | stack_skip); 381 if (pelem->stack_id < 0) 382 __sync_fetch_and_add(&stack_fail, 1); 383 } else if (aggr_mode == LOCK_AGGR_TASK) { 384 struct task_struct *task; 385 386 if (lock_owner) { 387 task = get_lock_owner(pelem->lock, pelem->flags); 388 389 /* The flags is not used anymore. Pass the owner pid. */ 390 if (task) 391 pelem->flags = BPF_CORE_READ(task, pid); 392 else 393 pelem->flags = -1U; 394 395 } else { 396 task = bpf_get_current_task_btf(); 397 } 398 399 if (task) { 400 if (update_task_data(task) < 0 && lock_owner) 401 pelem->flags = -1U; 402 } 403 } 404 405 return 0; 406 } 407 408 SEC("tp_btf/contention_end") 409 int contention_end(u64 *ctx) 410 { 411 __u32 pid = 0, idx = 0; 412 struct tstamp_data *pelem; 413 struct contention_key key = {}; 414 struct contention_data *data; 415 __u64 duration; 416 bool need_delete = false; 417 418 if (!enabled) 419 return 0; 420 421 /* 422 * For spinlock and rwlock, it needs to get the timestamp for the 423 * per-cpu map. However, contention_end does not have the flags 424 * so it cannot know whether it reads percpu or hash map. 425 * 426 * Try per-cpu map first and check if there's active contention. 427 * If it is, do not read hash map because it cannot go to sleeping 428 * locks before releasing the spinning locks. 429 */ 430 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); 431 if (pelem && pelem->lock) { 432 if (pelem->lock != ctx[0]) 433 return 0; 434 } else { 435 pid = bpf_get_current_pid_tgid(); 436 pelem = bpf_map_lookup_elem(&tstamp, &pid); 437 if (!pelem || pelem->lock != ctx[0]) 438 return 0; 439 need_delete = true; 440 } 441 442 duration = bpf_ktime_get_ns() - pelem->timestamp; 443 if ((__s64)duration < 0) { 444 pelem->lock = 0; 445 if (need_delete) 446 bpf_map_delete_elem(&tstamp, &pid); 447 __sync_fetch_and_add(&time_fail, 1); 448 return 0; 449 } 450 451 switch (aggr_mode) { 452 case LOCK_AGGR_CALLER: 453 key.stack_id = pelem->stack_id; 454 break; 455 case LOCK_AGGR_TASK: 456 if (lock_owner) 457 key.pid = pelem->flags; 458 else { 459 if (!need_delete) 460 pid = bpf_get_current_pid_tgid(); 461 key.pid = pid; 462 } 463 if (needs_callstack) 464 key.stack_id = pelem->stack_id; 465 break; 466 case LOCK_AGGR_ADDR: 467 key.lock_addr_or_cgroup = pelem->lock; 468 if (needs_callstack) 469 key.stack_id = pelem->stack_id; 470 break; 471 case LOCK_AGGR_CGROUP: 472 key.lock_addr_or_cgroup = get_current_cgroup_id(); 473 break; 474 default: 475 /* should not happen */ 476 return 0; 477 } 478 479 data = bpf_map_lookup_elem(&lock_stat, &key); 480 if (!data) { 481 if (data_map_full) { 482 pelem->lock = 0; 483 if (need_delete) 484 bpf_map_delete_elem(&tstamp, &pid); 485 __sync_fetch_and_add(&data_fail, 1); 486 return 0; 487 } 488 489 struct contention_data first = { 490 .total_time = duration, 491 .max_time = duration, 492 .min_time = duration, 493 .count = 1, 494 .flags = pelem->flags, 495 }; 496 int err; 497 498 if (aggr_mode == LOCK_AGGR_ADDR) 499 first.flags |= check_lock_type(pelem->lock, pelem->flags); 500 501 err = bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); 502 if (err < 0) { 503 if (err == -E2BIG) 504 data_map_full = 1; 505 __sync_fetch_and_add(&data_fail, 1); 506 } 507 pelem->lock = 0; 508 if (need_delete) 509 bpf_map_delete_elem(&tstamp, &pid); 510 return 0; 511 } 512 513 __sync_fetch_and_add(&data->total_time, duration); 514 __sync_fetch_and_add(&data->count, 1); 515 516 /* FIXME: need atomic operations */ 517 if (data->max_time < duration) 518 data->max_time = duration; 519 if (data->min_time > duration) 520 data->min_time = duration; 521 522 pelem->lock = 0; 523 if (need_delete) 524 bpf_map_delete_elem(&tstamp, &pid); 525 return 0; 526 } 527 528 extern struct rq runqueues __ksym; 529 530 struct rq___old { 531 raw_spinlock_t lock; 532 } __attribute__((preserve_access_index)); 533 534 struct rq___new { 535 raw_spinlock_t __lock; 536 } __attribute__((preserve_access_index)); 537 538 SEC("raw_tp/bpf_test_finish") 539 int BPF_PROG(collect_lock_syms) 540 { 541 __u64 lock_addr, lock_off; 542 __u32 lock_flag; 543 544 if (bpf_core_field_exists(struct rq___new, __lock)) 545 lock_off = offsetof(struct rq___new, __lock); 546 else 547 lock_off = offsetof(struct rq___old, lock); 548 549 for (int i = 0; i < MAX_CPUS; i++) { 550 struct rq *rq = bpf_per_cpu_ptr(&runqueues, i); 551 552 if (rq == NULL) 553 break; 554 555 lock_addr = (__u64)(void *)rq + lock_off; 556 lock_flag = LOCK_CLASS_RQLOCK; 557 bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); 558 } 559 return 0; 560 } 561 562 char LICENSE[] SEC("license") = "Dual BSD/GPL"; 563