1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 4 #include <vmlinux.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_helpers.h> 7 8 #include "../bpf_experimental.h" 9 #include "bpf_misc.h" 10 #include "task_kfunc_common.h" 11 12 char _license[] SEC("license") = "GPL"; 13 14 int err, pid; 15 16 /* Prototype for all of the program trace events below: 17 * 18 * TRACE_EVENT(task_newtask, 19 * TP_PROTO(struct task_struct *p, u64 clone_flags) 20 */ 21 22 struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym __weak; 23 24 struct task_struct *bpf_task_acquire___one(struct task_struct *task) __ksym __weak; 25 /* The two-param bpf_task_acquire doesn't exist */ 26 struct task_struct *bpf_task_acquire___two(struct task_struct *p, void *ctx) __ksym __weak; 27 /* Incorrect type for first param */ 28 struct task_struct *bpf_task_acquire___three(void *ctx) __ksym __weak; 29 30 void invalid_kfunc(void) __ksym __weak; 31 void bpf_testmod_test_mod_kfunc(int i) __ksym __weak; 32 33 static bool is_test_kfunc_task(void) 34 { 35 int cur_pid = bpf_get_current_pid_tgid() >> 32; 36 37 return pid == cur_pid; 38 } 39 40 static int test_acquire_release(struct task_struct *task) 41 { 42 struct task_struct *acquired = NULL; 43 44 if (!bpf_ksym_exists(bpf_task_acquire)) { 45 err = 3; 46 return 0; 47 } 48 if (!bpf_ksym_exists(bpf_testmod_test_mod_kfunc)) { 49 err = 4; 50 return 0; 51 } 52 if (bpf_ksym_exists(invalid_kfunc)) { 53 /* the verifier's dead code elimination should remove this */ 54 err = 5; 55 asm volatile ("goto -1"); /* for (;;); */ 56 } 57 58 acquired = bpf_task_acquire(task); 59 if (acquired) 60 bpf_task_release(acquired); 61 else 62 err = 6; 63 64 return 0; 65 } 66 67 SEC("tp_btf/task_newtask") 68 int BPF_PROG(test_task_kfunc_flavor_relo, struct task_struct *task, u64 clone_flags) 69 { 70 struct task_struct *acquired = NULL; 71 int fake_ctx = 42; 72 73 if (bpf_ksym_exists(bpf_task_acquire___one)) { 74 acquired = bpf_task_acquire___one(task); 75 } else if (bpf_ksym_exists(bpf_task_acquire___two)) { 76 /* Here, bpf_object__resolve_ksym_func_btf_id's find_ksym_btf_id 77 * call will find vmlinux's bpf_task_acquire, but subsequent 78 * bpf_core_types_are_compat will fail 79 */ 80 acquired = bpf_task_acquire___two(task, &fake_ctx); 81 err = 3; 82 return 0; 83 } else if (bpf_ksym_exists(bpf_task_acquire___three)) { 84 /* bpf_core_types_are_compat will fail similarly to above case */ 85 acquired = bpf_task_acquire___three(&fake_ctx); 86 err = 4; 87 return 0; 88 } 89 90 if (acquired) 91 bpf_task_release(acquired); 92 else 93 err = 5; 94 return 0; 95 } 96 97 SEC("tp_btf/task_newtask") 98 int BPF_PROG(test_task_kfunc_flavor_relo_not_found, struct task_struct *task, u64 clone_flags) 99 { 100 /* Neither symbol should successfully resolve. 101 * Success or failure of one ___flavor should not affect others 102 */ 103 if (bpf_ksym_exists(bpf_task_acquire___two)) 104 err = 1; 105 else if (bpf_ksym_exists(bpf_task_acquire___three)) 106 err = 2; 107 108 return 0; 109 } 110 111 SEC("tp_btf/task_newtask") 112 int BPF_PROG(test_task_acquire_release_argument, struct task_struct *task, u64 clone_flags) 113 { 114 if (!is_test_kfunc_task()) 115 return 0; 116 117 return test_acquire_release(task); 118 } 119 120 SEC("tp_btf/task_newtask") 121 int BPF_PROG(test_task_acquire_release_current, struct task_struct *task, u64 clone_flags) 122 { 123 if (!is_test_kfunc_task()) 124 return 0; 125 126 return test_acquire_release(bpf_get_current_task_btf()); 127 } 128 129 SEC("tp_btf/task_newtask") 130 int BPF_PROG(test_task_acquire_leave_in_map, struct task_struct *task, u64 clone_flags) 131 { 132 long status; 133 134 if (!is_test_kfunc_task()) 135 return 0; 136 137 status = tasks_kfunc_map_insert(task); 138 if (status) 139 err = 1; 140 141 return 0; 142 } 143 144 SEC("syscall") 145 int test_task_xchg_release(const void *ctx) 146 { 147 struct task_struct *task, *kptr, *acquired; 148 struct __tasks_kfunc_map_value *v, *local; 149 int refcnt, refcnt_after_drop; 150 long status; 151 152 (void)ctx; 153 154 task = bpf_get_current_task_btf(); 155 status = tasks_kfunc_map_insert(task); 156 if (status) { 157 err = 1; 158 return 0; 159 } 160 161 v = tasks_kfunc_map_value_lookup(task); 162 if (!v) { 163 err = 2; 164 return 0; 165 } 166 167 kptr = bpf_kptr_xchg(&v->task, NULL); 168 if (!kptr) { 169 err = 3; 170 return 0; 171 } 172 173 local = bpf_obj_new(typeof(*local)); 174 if (!local) { 175 err = 4; 176 bpf_task_release(kptr); 177 return 0; 178 } 179 180 kptr = bpf_kptr_xchg(&local->task, kptr); 181 if (kptr) { 182 err = 5; 183 bpf_obj_drop(local); 184 bpf_task_release(kptr); 185 return 0; 186 } 187 188 kptr = bpf_kptr_xchg(&local->task, NULL); 189 if (!kptr) { 190 err = 6; 191 bpf_obj_drop(local); 192 return 0; 193 } 194 195 /* Stash a copy into local kptr and check if it is released recursively. */ 196 acquired = bpf_task_acquire(kptr); 197 if (!acquired) { 198 err = 7; 199 bpf_obj_drop(local); 200 bpf_task_release(kptr); 201 return 0; 202 } 203 bpf_probe_read_kernel(&refcnt, sizeof(refcnt), &acquired->rcu_users); 204 205 acquired = bpf_kptr_xchg(&local->task, acquired); 206 if (acquired) { 207 err = 8; 208 bpf_obj_drop(local); 209 bpf_task_release(kptr); 210 bpf_task_release(acquired); 211 return 0; 212 } 213 214 bpf_obj_drop(local); 215 216 bpf_probe_read_kernel(&refcnt_after_drop, sizeof(refcnt_after_drop), &kptr->rcu_users); 217 if (refcnt != refcnt_after_drop + 1) { 218 err = 9; 219 bpf_task_release(kptr); 220 return 0; 221 } 222 223 bpf_task_release(kptr); 224 return 0; 225 } 226 227 SEC("tp_btf/task_newtask") 228 int BPF_PROG(test_task_map_acquire_release, struct task_struct *task, u64 clone_flags) 229 { 230 struct task_struct *kptr; 231 struct __tasks_kfunc_map_value *v; 232 long status; 233 234 if (!is_test_kfunc_task()) 235 return 0; 236 237 status = tasks_kfunc_map_insert(task); 238 if (status) { 239 err = 1; 240 return 0; 241 } 242 243 v = tasks_kfunc_map_value_lookup(task); 244 if (!v) { 245 err = 2; 246 return 0; 247 } 248 249 bpf_rcu_read_lock(); 250 kptr = v->task; 251 if (!kptr) { 252 err = 3; 253 } else { 254 kptr = bpf_task_acquire(kptr); 255 if (!kptr) 256 err = 4; 257 else 258 bpf_task_release(kptr); 259 } 260 bpf_rcu_read_unlock(); 261 262 return 0; 263 } 264 265 SEC("tp_btf/task_newtask") 266 int BPF_PROG(test_task_current_acquire_release, struct task_struct *task, u64 clone_flags) 267 { 268 struct task_struct *current, *acquired; 269 270 if (!is_test_kfunc_task()) 271 return 0; 272 273 current = bpf_get_current_task_btf(); 274 acquired = bpf_task_acquire(current); 275 if (acquired) 276 bpf_task_release(acquired); 277 else 278 err = 1; 279 280 return 0; 281 } 282 283 static void lookup_compare_pid(const struct task_struct *p) 284 { 285 struct task_struct *acquired; 286 287 acquired = bpf_task_from_pid(p->pid); 288 if (!acquired) { 289 err = 1; 290 return; 291 } 292 293 if (acquired->pid != p->pid) 294 err = 2; 295 bpf_task_release(acquired); 296 } 297 298 SEC("tp_btf/task_newtask") 299 int BPF_PROG(test_task_from_pid_arg, struct task_struct *task, u64 clone_flags) 300 { 301 if (!is_test_kfunc_task()) 302 return 0; 303 304 lookup_compare_pid(task); 305 return 0; 306 } 307 308 SEC("tp_btf/task_newtask") 309 int BPF_PROG(test_task_from_pid_current, struct task_struct *task, u64 clone_flags) 310 { 311 if (!is_test_kfunc_task()) 312 return 0; 313 314 lookup_compare_pid(bpf_get_current_task_btf()); 315 return 0; 316 } 317 318 static int is_pid_lookup_valid(s32 pid) 319 { 320 struct task_struct *acquired; 321 322 acquired = bpf_task_from_pid(pid); 323 if (acquired) { 324 bpf_task_release(acquired); 325 return 1; 326 } 327 328 return 0; 329 } 330 331 SEC("tp_btf/task_newtask") 332 int BPF_PROG(test_task_from_pid_invalid, struct task_struct *task, u64 clone_flags) 333 { 334 if (!is_test_kfunc_task()) 335 return 0; 336 337 bpf_strncmp(task->comm, 12, "foo"); 338 bpf_strncmp(task->comm, 16, "foo"); 339 bpf_strncmp(&task->comm[8], 4, "foo"); 340 341 if (is_pid_lookup_valid(-1)) { 342 err = 1; 343 return 0; 344 } 345 346 if (is_pid_lookup_valid(0xcafef00d)) { 347 err = 2; 348 return 0; 349 } 350 351 return 0; 352 } 353 354 SEC("tp_btf/task_newtask") 355 int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 clone_flags) 356 { 357 struct task_struct *acquired; 358 359 /* task->group_leader is listed as a trusted, non-NULL field of task struct. */ 360 acquired = bpf_task_acquire(task->group_leader); 361 if (acquired) 362 bpf_task_release(acquired); 363 else 364 err = 1; 365 366 367 return 0; 368 } 369 370 SEC("fentry/" SYS_PREFIX "sys_getpgid") 371 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_non_sleepable) 372 { 373 struct task_kptr_lock_value *v; 374 struct task_struct *task, *acquired; 375 int key = 0; 376 377 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 378 if (!v) 379 return 0; 380 381 bpf_spin_lock(&v->lock); 382 task = v->task; 383 bpf_spin_unlock(&v->lock); 384 if (!task) 385 return 0; 386 387 acquired = bpf_task_acquire(task); 388 if (acquired) 389 bpf_task_release(acquired); 390 return 0; 391 } 392 393 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 394 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_explicit_rcu) 395 { 396 struct task_kptr_lock_value *v; 397 struct task_struct *task, *acquired; 398 int key = 0; 399 400 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 401 if (!v) 402 return 0; 403 404 bpf_rcu_read_lock(); 405 bpf_spin_lock(&v->lock); 406 task = v->task; 407 bpf_spin_unlock(&v->lock); 408 if (task) { 409 acquired = bpf_task_acquire(task); 410 if (acquired) 411 bpf_task_release(acquired); 412 } 413 bpf_rcu_read_unlock(); 414 return 0; 415 } 416 417 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 418 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_preempt_disabled) 419 { 420 struct task_kptr_lock_value *v; 421 struct task_struct *task, *acquired; 422 int key = 0; 423 424 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 425 if (!v) 426 return 0; 427 428 bpf_preempt_disable(); 429 bpf_spin_lock(&v->lock); 430 task = v->task; 431 bpf_spin_unlock(&v->lock); 432 if (task) { 433 acquired = bpf_task_acquire(task); 434 if (acquired) 435 bpf_task_release(acquired); 436 } 437 bpf_preempt_enable(); 438 return 0; 439 } 440 441 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 442 int BPF_PROG(task_kfunc_acquire_after_spin_unlock_irq_disabled) 443 { 444 struct task_kptr_lock_value *v; 445 struct task_struct *task, *acquired; 446 unsigned long flags; 447 int key = 0; 448 449 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 450 if (!v) 451 return 0; 452 453 bpf_local_irq_save(&flags); 454 bpf_spin_lock(&v->lock); 455 task = v->task; 456 bpf_spin_unlock(&v->lock); 457 if (task) { 458 acquired = bpf_task_acquire(task); 459 if (acquired) 460 bpf_task_release(acquired); 461 } 462 bpf_local_irq_restore(&flags); 463 return 0; 464 } 465 466 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 467 int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_preempt_disabled) 468 { 469 struct task_kptr_lock_value *v; 470 struct task_struct *task, *acquired; 471 int key = 0; 472 473 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 474 if (!v) 475 return 0; 476 477 bpf_preempt_disable(); 478 bpf_rcu_read_lock(); 479 task = v->task; 480 bpf_rcu_read_unlock(); 481 if (task) { 482 acquired = bpf_task_acquire(task); 483 if (acquired) 484 bpf_task_release(acquired); 485 } 486 bpf_preempt_enable(); 487 return 0; 488 } 489 490 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 491 int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_irq_disabled) 492 { 493 struct task_kptr_lock_value *v; 494 struct task_struct *task, *acquired; 495 unsigned long flags; 496 int key = 0; 497 498 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 499 if (!v) 500 return 0; 501 502 bpf_local_irq_save(&flags); 503 bpf_rcu_read_lock(); 504 task = v->task; 505 bpf_rcu_read_unlock(); 506 if (task) { 507 acquired = bpf_task_acquire(task); 508 if (acquired) 509 bpf_task_release(acquired); 510 } 511 bpf_local_irq_restore(&flags); 512 return 0; 513 } 514 515 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 516 int BPF_PROG(task_kfunc_acquire_after_preempt_enable_explicit_rcu) 517 { 518 struct task_kptr_lock_value *v; 519 struct task_struct *task, *acquired; 520 int key = 0; 521 522 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 523 if (!v) 524 return 0; 525 526 bpf_preempt_disable(); 527 task = v->task; 528 bpf_rcu_read_lock(); 529 bpf_preempt_enable(); 530 if (task) { 531 acquired = bpf_task_acquire(task); 532 if (acquired) 533 bpf_task_release(acquired); 534 } 535 bpf_rcu_read_unlock(); 536 return 0; 537 } 538 539 SEC("fentry.s/" SYS_PREFIX "sys_getpgid") 540 int BPF_PROG(task_kfunc_acquire_after_irq_restore_explicit_rcu) 541 { 542 struct task_kptr_lock_value *v; 543 struct task_struct *task, *acquired; 544 unsigned long flags; 545 int key = 0; 546 547 v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); 548 if (!v) 549 return 0; 550 551 bpf_local_irq_save(&flags); 552 task = v->task; 553 bpf_rcu_read_lock(); 554 bpf_local_irq_restore(&flags); 555 if (task) { 556 acquired = bpf_task_acquire(task); 557 if (acquired) 558 bpf_task_release(acquired); 559 } 560 bpf_rcu_read_unlock(); 561 return 0; 562 } 563 564 SEC("syscall") 565 int test_task_from_vpid_current(const void *ctx) 566 { 567 struct task_struct *current, *v_task; 568 569 v_task = bpf_task_from_vpid(1); 570 if (!v_task) { 571 err = 1; 572 return 0; 573 } 574 575 current = bpf_get_current_task_btf(); 576 577 /* The current process should be the init process (pid 1) in the new pid namespace. */ 578 if (current != v_task) 579 err = 2; 580 581 bpf_task_release(v_task); 582 return 0; 583 } 584 585 SEC("syscall") 586 int test_task_from_vpid_invalid(const void *ctx) 587 { 588 struct task_struct *v_task; 589 590 v_task = bpf_task_from_vpid(-1); 591 if (v_task) { 592 err = 1; 593 goto err; 594 } 595 596 /* There should be only one process (current process) in the new pid namespace. */ 597 v_task = bpf_task_from_vpid(2); 598 if (v_task) { 599 err = 2; 600 goto err; 601 } 602 603 v_task = bpf_task_from_vpid(9999); 604 if (v_task) { 605 err = 3; 606 goto err; 607 } 608 609 return 0; 610 err: 611 bpf_task_release(v_task); 612 return 0; 613 } 614