1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <test_progs.h> 4 5 #ifdef __x86_64__ 6 7 #define _GNU_SOURCE 8 #include <sched.h> 9 #include <unistd.h> 10 #include <asm/ptrace.h> 11 #include <linux/compiler.h> 12 #include <linux/stringify.h> 13 #include <linux/kernel.h> 14 #include <sys/wait.h> 15 #include <sys/syscall.h> 16 #include <sys/prctl.h> 17 #include <asm/prctl.h> 18 #include <stdnoreturn.h> 19 #include "uprobe_syscall.skel.h" 20 #include "uprobe_syscall_executed.skel.h" 21 #include "bpf/libbpf_internal.h" 22 23 #define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 24 #include "usdt.h" 25 26 #pragma GCC diagnostic ignored "-Wattributes" 27 28 __attribute__((aligned(16))) 29 __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void) 30 { 31 asm volatile ( 32 ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ 33 "movq $0xdeadbeef, %rax\n" 34 "ret\n" 35 ); 36 } 37 38 __naked void uprobe_regs(struct pt_regs *before, struct pt_regs *after) 39 { 40 asm volatile ( 41 "movq %r15, 0(%rdi)\n" 42 "movq %r14, 8(%rdi)\n" 43 "movq %r13, 16(%rdi)\n" 44 "movq %r12, 24(%rdi)\n" 45 "movq %rbp, 32(%rdi)\n" 46 "movq %rbx, 40(%rdi)\n" 47 "movq %r11, 48(%rdi)\n" 48 "movq %r10, 56(%rdi)\n" 49 "movq %r9, 64(%rdi)\n" 50 "movq %r8, 72(%rdi)\n" 51 "movq %rax, 80(%rdi)\n" 52 "movq %rcx, 88(%rdi)\n" 53 "movq %rdx, 96(%rdi)\n" 54 "movq %rsi, 104(%rdi)\n" 55 "movq %rdi, 112(%rdi)\n" 56 "movq $0, 120(%rdi)\n" /* orig_rax */ 57 "movq $0, 128(%rdi)\n" /* rip */ 58 "movq $0, 136(%rdi)\n" /* cs */ 59 "pushq %rax\n" 60 "pushf\n" 61 "pop %rax\n" 62 "movq %rax, 144(%rdi)\n" /* eflags */ 63 "pop %rax\n" 64 "movq %rsp, 152(%rdi)\n" /* rsp */ 65 "movq $0, 160(%rdi)\n" /* ss */ 66 67 /* save 2nd argument */ 68 "pushq %rsi\n" 69 "call uprobe_regs_trigger\n" 70 71 /* save return value and load 2nd argument pointer to rax */ 72 "pushq %rax\n" 73 "movq 8(%rsp), %rax\n" 74 75 "movq %r15, 0(%rax)\n" 76 "movq %r14, 8(%rax)\n" 77 "movq %r13, 16(%rax)\n" 78 "movq %r12, 24(%rax)\n" 79 "movq %rbp, 32(%rax)\n" 80 "movq %rbx, 40(%rax)\n" 81 "movq %r11, 48(%rax)\n" 82 "movq %r10, 56(%rax)\n" 83 "movq %r9, 64(%rax)\n" 84 "movq %r8, 72(%rax)\n" 85 "movq %rcx, 88(%rax)\n" 86 "movq %rdx, 96(%rax)\n" 87 "movq %rsi, 104(%rax)\n" 88 "movq %rdi, 112(%rax)\n" 89 "movq $0, 120(%rax)\n" /* orig_rax */ 90 "movq $0, 128(%rax)\n" /* rip */ 91 "movq $0, 136(%rax)\n" /* cs */ 92 93 /* restore return value and 2nd argument */ 94 "pop %rax\n" 95 "pop %rsi\n" 96 97 "movq %rax, 80(%rsi)\n" 98 99 "pushf\n" 100 "pop %rax\n" 101 102 "movq %rax, 144(%rsi)\n" /* eflags */ 103 "movq %rsp, 152(%rsi)\n" /* rsp */ 104 "movq $0, 160(%rsi)\n" /* ss */ 105 "ret\n" 106 ); 107 } 108 109 static void test_uprobe_regs_equal(bool retprobe) 110 { 111 LIBBPF_OPTS(bpf_uprobe_opts, opts, 112 .retprobe = retprobe, 113 ); 114 struct uprobe_syscall *skel = NULL; 115 struct pt_regs before = {}, after = {}; 116 unsigned long *pb = (unsigned long *) &before; 117 unsigned long *pa = (unsigned long *) &after; 118 unsigned long *pp; 119 unsigned long offset; 120 unsigned int i, cnt; 121 122 offset = get_uprobe_offset(&uprobe_regs_trigger); 123 if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) 124 return; 125 126 skel = uprobe_syscall__open_and_load(); 127 if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load")) 128 goto cleanup; 129 130 skel->links.probe = bpf_program__attach_uprobe_opts(skel->progs.probe, 131 0, "/proc/self/exe", offset, &opts); 132 if (!ASSERT_OK_PTR(skel->links.probe, "bpf_program__attach_uprobe_opts")) 133 goto cleanup; 134 135 /* make sure uprobe gets optimized */ 136 if (!retprobe) 137 uprobe_regs_trigger(); 138 139 uprobe_regs(&before, &after); 140 141 pp = (unsigned long *) &skel->bss->regs; 142 cnt = sizeof(before)/sizeof(*pb); 143 144 for (i = 0; i < cnt; i++) { 145 unsigned int offset = i * sizeof(unsigned long); 146 147 /* 148 * Check register before and after uprobe_regs_trigger call 149 * that triggers the uretprobe. 150 */ 151 switch (offset) { 152 case offsetof(struct pt_regs, rax): 153 ASSERT_EQ(pa[i], 0xdeadbeef, "return value"); 154 break; 155 default: 156 if (!ASSERT_EQ(pb[i], pa[i], "register before-after value check")) 157 fprintf(stdout, "failed register offset %u\n", offset); 158 } 159 160 /* 161 * Check register seen from bpf program and register after 162 * uprobe_regs_trigger call (with rax exception, check below). 163 */ 164 switch (offset) { 165 /* 166 * These values will be different (not set in uretprobe_regs), 167 * we don't care. 168 */ 169 case offsetof(struct pt_regs, orig_rax): 170 case offsetof(struct pt_regs, rip): 171 case offsetof(struct pt_regs, cs): 172 case offsetof(struct pt_regs, rsp): 173 case offsetof(struct pt_regs, ss): 174 break; 175 /* 176 * uprobe does not see return value in rax, it needs to see the 177 * original (before) rax value 178 */ 179 case offsetof(struct pt_regs, rax): 180 if (!retprobe) { 181 ASSERT_EQ(pp[i], pb[i], "uprobe rax prog-before value check"); 182 break; 183 } 184 default: 185 if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check")) 186 fprintf(stdout, "failed register offset %u\n", offset); 187 } 188 } 189 190 cleanup: 191 uprobe_syscall__destroy(skel); 192 } 193 194 #define BPF_TESTMOD_UPROBE_TEST_FILE "/sys/kernel/bpf_testmod_uprobe" 195 196 static int write_bpf_testmod_uprobe(unsigned long offset) 197 { 198 size_t n, ret; 199 char buf[30]; 200 int fd; 201 202 n = sprintf(buf, "%lu", offset); 203 204 fd = open(BPF_TESTMOD_UPROBE_TEST_FILE, O_WRONLY); 205 if (fd < 0) 206 return -errno; 207 208 ret = write(fd, buf, n); 209 close(fd); 210 return ret != n ? (int) ret : 0; 211 } 212 213 static void test_regs_change(void) 214 { 215 struct pt_regs before = {}, after = {}; 216 unsigned long *pb = (unsigned long *) &before; 217 unsigned long *pa = (unsigned long *) &after; 218 unsigned long cnt = sizeof(before)/sizeof(*pb); 219 unsigned int i, err, offset; 220 221 offset = get_uprobe_offset(uprobe_regs_trigger); 222 223 err = write_bpf_testmod_uprobe(offset); 224 if (!ASSERT_OK(err, "register_uprobe")) 225 return; 226 227 /* make sure uprobe gets optimized */ 228 uprobe_regs_trigger(); 229 230 uprobe_regs(&before, &after); 231 232 err = write_bpf_testmod_uprobe(0); 233 if (!ASSERT_OK(err, "unregister_uprobe")) 234 return; 235 236 for (i = 0; i < cnt; i++) { 237 unsigned int offset = i * sizeof(unsigned long); 238 239 switch (offset) { 240 case offsetof(struct pt_regs, rax): 241 ASSERT_EQ(pa[i], 0x12345678deadbeef, "rax"); 242 break; 243 case offsetof(struct pt_regs, rcx): 244 ASSERT_EQ(pa[i], 0x87654321feebdaed, "rcx"); 245 break; 246 case offsetof(struct pt_regs, r11): 247 ASSERT_EQ(pa[i], (__u64) -1, "r11"); 248 break; 249 default: 250 if (!ASSERT_EQ(pa[i], pb[i], "register before-after value check")) 251 fprintf(stdout, "failed register offset %u\n", offset); 252 } 253 } 254 } 255 256 #ifndef __NR_uretprobe 257 #define __NR_uretprobe 335 258 #endif 259 260 __naked unsigned long uretprobe_syscall_call_1(void) 261 { 262 /* 263 * Pretend we are uretprobe trampoline to trigger the return 264 * probe invocation in order to verify we get SIGILL. 265 */ 266 asm volatile ( 267 "pushq %rax\n" 268 "pushq %rcx\n" 269 "pushq %r11\n" 270 "movq $" __stringify(__NR_uretprobe) ", %rax\n" 271 "syscall\n" 272 "popq %r11\n" 273 "popq %rcx\n" 274 "retq\n" 275 ); 276 } 277 278 __naked unsigned long uretprobe_syscall_call(void) 279 { 280 asm volatile ( 281 "call uretprobe_syscall_call_1\n" 282 "retq\n" 283 ); 284 } 285 286 static void test_uretprobe_syscall_call(void) 287 { 288 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts, 289 .retprobe = true, 290 ); 291 struct uprobe_syscall_executed *skel; 292 int pid, status, err, go[2], c = 0; 293 struct bpf_link *link; 294 295 if (!ASSERT_OK(pipe(go), "pipe")) 296 return; 297 298 skel = uprobe_syscall_executed__open_and_load(); 299 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load")) 300 goto cleanup; 301 302 pid = fork(); 303 if (!ASSERT_GE(pid, 0, "fork")) 304 goto cleanup; 305 306 /* child */ 307 if (pid == 0) { 308 close(go[1]); 309 310 /* wait for parent's kick */ 311 err = read(go[0], &c, 1); 312 if (err != 1) 313 exit(-1); 314 315 uretprobe_syscall_call(); 316 _exit(0); 317 } 318 319 skel->bss->pid = pid; 320 321 link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi, 322 pid, "/proc/self/exe", 323 "uretprobe_syscall_call", &opts); 324 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 325 goto cleanup; 326 skel->links.test_uretprobe_multi = link; 327 328 /* kick the child */ 329 write(go[1], &c, 1); 330 err = waitpid(pid, &status, 0); 331 ASSERT_EQ(err, pid, "waitpid"); 332 333 /* verify the child got killed with SIGILL */ 334 ASSERT_EQ(WIFSIGNALED(status), 1, "WIFSIGNALED"); 335 ASSERT_EQ(WTERMSIG(status), SIGILL, "WTERMSIG"); 336 337 /* verify the uretprobe program wasn't called */ 338 ASSERT_EQ(skel->bss->executed, 0, "executed"); 339 340 cleanup: 341 uprobe_syscall_executed__destroy(skel); 342 close(go[1]); 343 close(go[0]); 344 } 345 346 #define TRAMP "[uprobes-trampoline]" 347 348 __attribute__((aligned(16))) 349 __nocf_check __weak __naked void uprobe_test(void) 350 { 351 asm volatile ( 352 ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ 353 "ret\n" 354 ); 355 } 356 357 __attribute__((aligned(16))) 358 __nocf_check __weak void usdt_test(void) 359 { 360 USDT(optimized_uprobe, usdt); 361 } 362 363 /* 364 * Assembly-level red zone clobbering test. Stores known values in the 365 * red zone (below RSP), executes a nop10 (uprobe site), and checks that 366 * the values survived. Returns 0 if intact, 1 if clobbered. 367 * 368 * The nop5 optimization used CALL (which pushes a return address to 369 * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization 370 * should escape that by moving stackpointer below the redzone before 371 * doing the CALL. 372 * 373 * Align the code at 64 bytes, to make sure nop10 is not on page boundary. 374 */ 375 __attribute__((aligned(64))) 376 __nocf_check __weak __naked unsigned long uprobe_red_zone_test(void) 377 { 378 asm volatile ( 379 "movabs $0x1111111111111111, %%rax\n" 380 "movq %%rax, -8(%%rsp)\n" 381 "movabs $0x2222222222222222, %%rax\n" 382 "movq %%rax, -16(%%rsp)\n" 383 "movabs $0x3333333333333333, %%rax\n" 384 "movq %%rax, -24(%%rsp)\n" 385 386 ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */ 387 388 "movabs $0x1111111111111111, %%rax\n" 389 "cmpq %%rax, -8(%%rsp)\n" 390 "jne 1f\n" 391 "movabs $0x2222222222222222, %%rax\n" 392 "cmpq %%rax, -16(%%rsp)\n" 393 "jne 1f\n" 394 "movabs $0x3333333333333333, %%rax\n" 395 "cmpq %%rax, -24(%%rsp)\n" 396 "jne 1f\n" 397 398 "xorl %%eax, %%eax\n" 399 "retq\n" 400 "1:\n" 401 "movl $1, %%eax\n" 402 "retq\n" 403 ::: "rax", "memory" 404 ); 405 } 406 407 static int find_uprobes_trampoline(void *tramp_addr) 408 { 409 void *start, *end; 410 char line[128]; 411 int ret = -1; 412 FILE *maps; 413 414 maps = fopen("/proc/self/maps", "r"); 415 if (!maps) { 416 fprintf(stderr, "cannot open maps\n"); 417 return -1; 418 } 419 420 while (fgets(line, sizeof(line), maps)) { 421 int m = -1; 422 423 /* We care only about private r-x mappings. */ 424 if (sscanf(line, "%p-%p r-xp %*x %*x:%*x %*u %n", &start, &end, &m) != 2) 425 continue; 426 if (m < 0) 427 continue; 428 if (!strncmp(&line[m], TRAMP, sizeof(TRAMP)-1) && (start == tramp_addr)) { 429 ret = 0; 430 break; 431 } 432 } 433 434 fclose(maps); 435 return ret; 436 } 437 438 static unsigned char nop10[10] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; 439 static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; 440 441 static void *find_nop10(void *fn) 442 { 443 int i; 444 445 for (i = 0; i < 128; i++) { 446 if (!memcmp(nop10, fn + i, 10)) 447 return fn + i; 448 } 449 return NULL; 450 } 451 452 typedef void (__attribute__((nocf_check)) *trigger_t)(void); 453 454 static void check_attach_notrigger(struct uprobe_syscall_executed *skel, 455 void *addr, int executed) 456 { 457 unsigned char *op = addr; 458 459 /* Make sure bpf program was not executed. */ 460 ASSERT_EQ(skel->bss->executed, executed, "executed"); 461 ASSERT_EQ(*op, 0xcc, "int3"); 462 } 463 464 static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger, 465 void *addr, int executed) 466 { 467 struct __arch_relative_insn { 468 __u8 op; 469 __s32 raddr; 470 } __packed *call; 471 void *tramp = NULL; 472 473 /* Uprobe gets optimized after first trigger, so let's press twice. */ 474 trigger(); 475 trigger(); 476 477 /* Make sure bpf program got executed.. */ 478 ASSERT_EQ(skel->bss->executed, executed, "executed"); 479 480 /* .. and check the trampoline is as expected. */ 481 ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp"); 482 call = (struct __arch_relative_insn *)(addr + 5); 483 tramp = (void *) (call + 1) + call->raddr; 484 ASSERT_EQ(call->op, 0xe8, "call"); 485 ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); 486 487 return tramp; 488 } 489 490 static bool check_detach(void *addr, void *tramp) 491 { 492 static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 }; 493 bool ok = true; 494 495 /* [uprobes_trampoline] stays after detach */ 496 ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); 497 ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix"); 498 return ok; 499 } 500 501 static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link, 502 trigger_t trigger, void *addr, int executed) 503 { 504 void *tramp; 505 506 tramp = check_attach(skel, trigger, addr, executed); 507 bpf_link__destroy(link); 508 check_detach(addr, tramp); 509 return tramp; 510 } 511 512 static void test_uprobe_legacy(void) 513 { 514 struct uprobe_syscall_executed *skel = NULL; 515 LIBBPF_OPTS(bpf_uprobe_opts, opts, 516 .retprobe = true, 517 ); 518 struct bpf_link *link; 519 unsigned long offset; 520 void *tramp; 521 522 offset = get_uprobe_offset(&uprobe_test); 523 if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) 524 goto cleanup; 525 526 /* uprobe */ 527 skel = uprobe_syscall_executed__open_and_load(); 528 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load")) 529 return; 530 531 skel->bss->pid = getpid(); 532 533 link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, 534 0, "/proc/self/exe", offset, NULL); 535 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) 536 goto cleanup; 537 538 tramp = check(skel, link, uprobe_test, uprobe_test, 2); 539 540 /* reattach and detach without triggering optimization */ 541 link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, 542 0, "/proc/self/exe", offset, NULL); 543 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) 544 goto cleanup; 545 546 check_attach_notrigger(skel, uprobe_test, 2); 547 548 bpf_link__destroy(link); 549 if (!check_detach(uprobe_test, tramp)) 550 goto cleanup; 551 552 uprobe_test(); 553 ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); 554 555 /* reattach with triggering optimization */ 556 link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, 557 0, "/proc/self/exe", offset, NULL); 558 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) 559 goto cleanup; 560 561 check(skel, link, uprobe_test, uprobe_test, 4); 562 563 /* uretprobe */ 564 skel->bss->executed = 0; 565 566 link = bpf_program__attach_uprobe_opts(skel->progs.test_uretprobe, 567 0, "/proc/self/exe", offset, &opts); 568 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) 569 goto cleanup; 570 571 check(skel, link, uprobe_test, uprobe_test, 2); 572 573 cleanup: 574 uprobe_syscall_executed__destroy(skel); 575 } 576 577 static void test_uprobe_multi(void) 578 { 579 struct uprobe_syscall_executed *skel = NULL; 580 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); 581 struct bpf_link *link; 582 unsigned long offset; 583 void *tramp; 584 585 offset = get_uprobe_offset(&uprobe_test); 586 if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) 587 goto cleanup; 588 589 opts.offsets = &offset; 590 opts.cnt = 1; 591 592 skel = uprobe_syscall_executed__open_and_load(); 593 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load")) 594 return; 595 596 skel->bss->pid = getpid(); 597 598 /* uprobe.multi */ 599 link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, 600 0, "/proc/self/exe", NULL, &opts); 601 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 602 goto cleanup; 603 604 tramp = check(skel, link, uprobe_test, uprobe_test, 2); 605 606 /* reattach and detach without triggering optimization */ 607 link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, 608 0, "/proc/self/exe", NULL, &opts); 609 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 610 goto cleanup; 611 612 check_attach_notrigger(skel, uprobe_test, 2); 613 614 bpf_link__destroy(link); 615 if (!check_detach(uprobe_test, tramp)) 616 goto cleanup; 617 618 uprobe_test(); 619 ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); 620 621 /* reattach with triggering optimization */ 622 link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, 623 0, "/proc/self/exe", NULL, &opts); 624 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 625 goto cleanup; 626 627 check(skel, link, uprobe_test, uprobe_test, 4); 628 629 /* uretprobe.multi */ 630 skel->bss->executed = 0; 631 opts.retprobe = true; 632 link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi, 633 0, "/proc/self/exe", NULL, &opts); 634 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 635 goto cleanup; 636 637 check(skel, link, uprobe_test, uprobe_test, 2); 638 639 cleanup: 640 uprobe_syscall_executed__destroy(skel); 641 } 642 643 static void test_uprobe_session(void) 644 { 645 struct uprobe_syscall_executed *skel = NULL; 646 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts, 647 .session = true, 648 ); 649 struct bpf_link *link; 650 unsigned long offset; 651 void *tramp; 652 653 offset = get_uprobe_offset(&uprobe_test); 654 if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) 655 goto cleanup; 656 657 opts.offsets = &offset; 658 opts.cnt = 1; 659 660 skel = uprobe_syscall_executed__open_and_load(); 661 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load")) 662 return; 663 664 skel->bss->pid = getpid(); 665 666 link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, 667 0, "/proc/self/exe", NULL, &opts); 668 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 669 goto cleanup; 670 671 tramp = check(skel, link, uprobe_test, uprobe_test, 4); 672 673 /* reattach and detach without triggering optimization */ 674 link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, 675 0, "/proc/self/exe", NULL, &opts); 676 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 677 goto cleanup; 678 679 check_attach_notrigger(skel, uprobe_test, 4); 680 681 bpf_link__destroy(link); 682 if (!check_detach(uprobe_test, tramp)) 683 goto cleanup; 684 685 uprobe_test(); 686 ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe"); 687 688 /* reattach with triggering optimization */ 689 link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, 690 0, "/proc/self/exe", NULL, &opts); 691 if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) 692 goto cleanup; 693 694 check(skel, link, uprobe_test, uprobe_test, 8); 695 696 cleanup: 697 uprobe_syscall_executed__destroy(skel); 698 } 699 700 static void test_uprobe_usdt(void) 701 { 702 struct uprobe_syscall_executed *skel; 703 struct bpf_link *link; 704 void *addr, *tramp; 705 706 errno = 0; 707 addr = find_nop10(usdt_test); 708 if (!ASSERT_OK_PTR(addr, "find_nop10")) 709 return; 710 711 skel = uprobe_syscall_executed__open_and_load(); 712 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load")) 713 return; 714 715 skel->bss->pid = getpid(); 716 717 link = bpf_program__attach_usdt(skel->progs.test_usdt, 718 -1 /* all PIDs */, "/proc/self/exe", 719 "optimized_uprobe", "usdt", NULL); 720 if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) 721 goto cleanup; 722 723 tramp = check(skel, link, usdt_test, addr, 2); 724 725 /* reattach and detach without triggering optimization */ 726 link = bpf_program__attach_usdt(skel->progs.test_usdt, 727 -1 /* all PIDs */, "/proc/self/exe", 728 "optimized_uprobe", "usdt", NULL); 729 if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) 730 goto cleanup; 731 732 check_attach_notrigger(skel, addr, 2); 733 734 bpf_link__destroy(link); 735 if (!check_detach(addr, tramp)) 736 goto cleanup; 737 738 usdt_test(); 739 ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); 740 741 /* reattach with triggering optimization */ 742 link = bpf_program__attach_usdt(skel->progs.test_usdt, 743 -1 /* all PIDs */, "/proc/self/exe", 744 "optimized_uprobe", "usdt", NULL); 745 if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) 746 goto cleanup; 747 748 check(skel, link, usdt_test, addr, 4); 749 750 cleanup: 751 uprobe_syscall_executed__destroy(skel); 752 } 753 754 /* 755 * Borrowed from tools/testing/selftests/x86/test_shadow_stack.c. 756 * 757 * For use in inline enablement of shadow stack. 758 * 759 * The program can't return from the point where shadow stack gets enabled 760 * because there will be no address on the shadow stack. So it can't use 761 * syscall() for enablement, since it is a function. 762 * 763 * Based on code from nolibc.h. Keep a copy here because this can't pull 764 * in all of nolibc.h. 765 */ 766 #define ARCH_PRCTL(arg1, arg2) \ 767 ({ \ 768 long _ret; \ 769 register long _num asm("eax") = __NR_arch_prctl; \ 770 register long _arg1 asm("rdi") = (long)(arg1); \ 771 register long _arg2 asm("rsi") = (long)(arg2); \ 772 \ 773 asm volatile ( \ 774 "syscall\n" \ 775 : "=a"(_ret) \ 776 : "r"(_arg1), "r"(_arg2), \ 777 "0"(_num) \ 778 : "rcx", "r11", "memory", "cc" \ 779 ); \ 780 _ret; \ 781 }) 782 783 #ifndef ARCH_SHSTK_ENABLE 784 #define ARCH_SHSTK_ENABLE 0x5001 785 #define ARCH_SHSTK_DISABLE 0x5002 786 #define ARCH_SHSTK_SHSTK (1ULL << 0) 787 #endif 788 789 static void test_uretprobe_shadow_stack(void) 790 { 791 if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) { 792 test__skip(); 793 return; 794 } 795 796 /* Run all the tests with shadow stack in place. */ 797 798 test_uprobe_regs_equal(false); 799 test_uprobe_regs_equal(true); 800 test_uretprobe_syscall_call(); 801 802 test_uprobe_legacy(); 803 test_uprobe_multi(); 804 test_uprobe_session(); 805 test_uprobe_usdt(); 806 807 test_regs_change(); 808 809 ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK); 810 } 811 812 static volatile bool race_stop; 813 814 static USDT_DEFINE_SEMA(race); 815 816 static void *worker_trigger(void *arg) 817 { 818 unsigned long rounds = 0; 819 820 while (!race_stop) { 821 uprobe_test(); 822 rounds++; 823 } 824 825 printf("tid %ld trigger rounds: %lu\n", sys_gettid(), rounds); 826 return NULL; 827 } 828 829 static void *worker_attach(void *arg) 830 { 831 LIBBPF_OPTS(bpf_uprobe_opts, opts); 832 struct uprobe_syscall_executed *skel; 833 unsigned long rounds = 0, offset; 834 const char *sema[2] = { 835 __stringify(USDT_SEMA(race)), 836 NULL, 837 }; 838 unsigned long *ref; 839 int err; 840 841 offset = get_uprobe_offset(&uprobe_test); 842 if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) 843 return NULL; 844 845 err = elf_resolve_syms_offsets("/proc/self/exe", 1, (const char **) &sema, &ref, STT_OBJECT); 846 if (!ASSERT_OK(err, "elf_resolve_syms_offsets_sema")) 847 return NULL; 848 849 opts.ref_ctr_offset = *ref; 850 851 skel = uprobe_syscall_executed__open_and_load(); 852 if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load")) 853 return NULL; 854 855 skel->bss->pid = getpid(); 856 857 while (!race_stop) { 858 skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, 859 0, "/proc/self/exe", offset, &opts); 860 if (!ASSERT_OK_PTR(skel->links.test_uprobe, "bpf_program__attach_uprobe_opts")) 861 break; 862 863 bpf_link__destroy(skel->links.test_uprobe); 864 skel->links.test_uprobe = NULL; 865 rounds++; 866 } 867 868 printf("tid %ld attach rounds: %lu hits: %d\n", sys_gettid(), rounds, skel->bss->executed); 869 uprobe_syscall_executed__destroy(skel); 870 free(ref); 871 return NULL; 872 } 873 874 static useconds_t race_msec(void) 875 { 876 char *env; 877 878 env = getenv("BPF_SELFTESTS_UPROBE_SYSCALL_RACE_MSEC"); 879 if (env) 880 return atoi(env); 881 882 /* default duration is 500ms */ 883 return 500; 884 } 885 886 static void test_uprobe_race(void) 887 { 888 int err, i, nr_threads; 889 pthread_t *threads; 890 891 nr_threads = libbpf_num_possible_cpus(); 892 if (!ASSERT_GT(nr_threads, 0, "libbpf_num_possible_cpus")) 893 return; 894 nr_threads = max(2, nr_threads); 895 896 threads = alloca(sizeof(*threads) * nr_threads); 897 if (!ASSERT_OK_PTR(threads, "malloc")) 898 return; 899 900 for (i = 0; i < nr_threads; i++) { 901 err = pthread_create(&threads[i], NULL, i % 2 ? worker_trigger : worker_attach, 902 NULL); 903 if (!ASSERT_OK(err, "pthread_create")) 904 goto cleanup; 905 } 906 907 usleep(race_msec() * 1000); 908 909 cleanup: 910 race_stop = true; 911 for (nr_threads = i, i = 0; i < nr_threads; i++) 912 pthread_join(threads[i], NULL); 913 914 ASSERT_FALSE(USDT_SEMA_IS_ACTIVE(race), "race_semaphore"); 915 } 916 917 #ifndef __NR_uprobe 918 #define __NR_uprobe 336 919 #endif 920 921 static void test_uprobe_red_zone(void) 922 { 923 struct uprobe_syscall_executed *skel; 924 struct bpf_link *link; 925 void *nop10_addr; 926 size_t offset; 927 int i; 928 929 nop10_addr = find_nop10(uprobe_red_zone_test); 930 if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10")) 931 return; 932 933 skel = uprobe_syscall_executed__open_and_load(); 934 if (!ASSERT_OK_PTR(skel, "open_and_load")) 935 return; 936 937 offset = get_uprobe_offset(nop10_addr); 938 link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, 939 0, "/proc/self/exe", offset, NULL); 940 if (!ASSERT_OK_PTR(link, "attach_uprobe")) 941 goto cleanup; 942 943 for (i = 0; i < 10; i++) 944 ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact"); 945 946 bpf_link__destroy(link); 947 948 cleanup: 949 uprobe_syscall_executed__destroy(skel); 950 } 951 952 static void test_uprobe_error(void) 953 { 954 long err = syscall(__NR_uprobe); 955 956 ASSERT_EQ(err, -1, "error"); 957 ASSERT_EQ(errno, EPROTO, "errno"); 958 } 959 960 __attribute__((aligned(16))) 961 __nocf_check __weak __naked void uprobe_fork_test(void) 962 { 963 asm volatile ( 964 ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ 965 "ret\n" 966 ); 967 } 968 969 static noreturn int child_func(void *arg) 970 { 971 struct uprobe_syscall_executed *skel = arg; 972 973 /* Make sure the child's probe is still there and optimized.. */ 974 if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp))) 975 _exit(1); 976 977 skel->bss->pid = getpid(); 978 979 /* .. and it executes properly. */ 980 uprobe_fork_test(); 981 982 if (skel->bss->executed != 3) 983 _exit(2); 984 985 _exit(0); 986 } 987 988 static void test_uprobe_fork_optimized(bool clone_vm) 989 { 990 struct uprobe_syscall_executed *skel = NULL; 991 unsigned long offset; 992 int pid, status, err; 993 char stack[65535]; 994 995 offset = get_uprobe_offset(&uprobe_fork_test); 996 if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) 997 return; 998 999 skel = uprobe_syscall_executed__open_and_load(); 1000 if (!ASSERT_OK_PTR(skel, "open_and_load")) 1001 goto cleanup; 1002 1003 skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, 1004 -1, "/proc/self/exe", offset, NULL); 1005 if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe")) 1006 goto cleanup; 1007 1008 skel->bss->pid = getpid(); 1009 1010 /* Trigger optimization of uprobe in uprobe_fork_test. */ 1011 uprobe_fork_test(); 1012 uprobe_fork_test(); 1013 1014 /* Make sure it got optimied. */ 1015 if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized")) 1016 goto cleanup; 1017 1018 if (clone_vm) { 1019 pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel); 1020 if (!ASSERT_GT(pid, 0, "clone")) 1021 goto cleanup; 1022 } else { 1023 pid = fork(); 1024 if (!ASSERT_GE(pid, 0, "fork")) 1025 goto cleanup; 1026 if (pid == 0) 1027 child_func(skel); 1028 } 1029 1030 /* Wait for the child and verify it exited properly with 0. */ 1031 err = waitpid(pid, &status, 0); 1032 if (ASSERT_EQ(err, pid, "waitpid")) { 1033 ASSERT_EQ(WIFEXITED(status), 1, "child_exited"); 1034 ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code"); 1035 } 1036 1037 cleanup: 1038 uprobe_syscall_executed__destroy(skel); 1039 } 1040 1041 static void __test_uprobe_syscall(void) 1042 { 1043 if (test__start_subtest("uretprobe_regs_equal")) 1044 test_uprobe_regs_equal(true); 1045 if (test__start_subtest("uretprobe_syscall_call")) 1046 test_uretprobe_syscall_call(); 1047 if (test__start_subtest("uretprobe_shadow_stack")) 1048 test_uretprobe_shadow_stack(); 1049 if (test__start_subtest("uprobe_legacy")) 1050 test_uprobe_legacy(); 1051 if (test__start_subtest("uprobe_multi")) 1052 test_uprobe_multi(); 1053 if (test__start_subtest("uprobe_session")) 1054 test_uprobe_session(); 1055 if (test__start_subtest("uprobe_usdt")) 1056 test_uprobe_usdt(); 1057 if (test__start_subtest("uprobe_race")) 1058 test_uprobe_race(); 1059 if (test__start_subtest("uprobe_red_zone")) 1060 test_uprobe_red_zone(); 1061 if (test__start_subtest("uprobe_optimized_fork")) 1062 test_uprobe_fork_optimized(false); 1063 if (test__start_subtest("uprobe_optimized_clone_vm")) 1064 test_uprobe_fork_optimized(true); 1065 if (test__start_subtest("uprobe_error")) 1066 test_uprobe_error(); 1067 if (test__start_subtest("uprobe_regs_equal")) 1068 test_uprobe_regs_equal(false); 1069 if (test__start_subtest("regs_change")) 1070 test_regs_change(); 1071 } 1072 #else 1073 static void __test_uprobe_syscall(void) 1074 { 1075 test__skip(); 1076 } 1077 #endif 1078 1079 void test_uprobe_syscall(void) 1080 { 1081 __test_uprobe_syscall(); 1082 } 1083