1 // SPDX-License-Identifier: GPL-2.0 2 #include <trace/syscall.h> 3 #include <trace/events/syscalls.h> 4 #include <linux/kernel_stat.h> 5 #include <linux/syscalls.h> 6 #include <linux/slab.h> 7 #include <linux/kernel.h> 8 #include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */ 9 #include <linux/ftrace.h> 10 #include <linux/perf_event.h> 11 #include <linux/xarray.h> 12 #include <linux/btf_ids.h> 13 #include <asm/syscall.h> 14 15 #include "trace_output.h" 16 #include "trace.h" 17 18 static DEFINE_MUTEX(syscall_trace_lock); 19 20 static int syscall_enter_register(struct trace_event_call *event, 21 enum trace_reg type, void *data); 22 static int syscall_exit_register(struct trace_event_call *event, 23 enum trace_reg type, void *data); 24 25 static struct list_head * 26 syscall_get_enter_fields(struct trace_event_call *call) 27 { 28 struct syscall_metadata *entry = call->data; 29 30 return &entry->enter_fields; 31 } 32 33 extern struct syscall_metadata *__start_syscalls_metadata[]; 34 extern struct syscall_metadata *__stop_syscalls_metadata[]; 35 36 static DEFINE_XARRAY(syscalls_metadata_sparse); 37 static struct syscall_metadata **syscalls_metadata; 38 39 #ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME 40 static inline bool arch_syscall_match_sym_name(const char *sym, const char *name) 41 { 42 /* 43 * Only compare after the "sys" prefix. Archs that use 44 * syscall wrappers may have syscalls symbols aliases prefixed 45 * with ".SyS" or ".sys" instead of "sys", leading to an unwanted 46 * mismatch. 47 */ 48 return !strcmp(sym + 3, name + 3); 49 } 50 #endif 51 52 #ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS 53 /* 54 * Some architectures that allow for 32bit applications 55 * to run on a 64bit kernel, do not map the syscalls for 56 * the 32bit tasks the same as they do for 64bit tasks. 57 * 58 * *cough*x86*cough* 59 * 60 * In such a case, instead of reporting the wrong syscalls, 61 * simply ignore them. 62 * 63 * For an arch to ignore the compat syscalls it needs to 64 * define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as 65 * define the function arch_trace_is_compat_syscall() to let 66 * the tracing system know that it should ignore it. 67 */ 68 static int 69 trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs) 70 { 71 if (unlikely(arch_trace_is_compat_syscall(regs))) 72 return -1; 73 74 return syscall_get_nr(task, regs); 75 } 76 #else 77 static inline int 78 trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs) 79 { 80 return syscall_get_nr(task, regs); 81 } 82 #endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */ 83 84 static __init struct syscall_metadata * 85 find_syscall_meta(unsigned long syscall) 86 { 87 struct syscall_metadata **start; 88 struct syscall_metadata **stop; 89 char str[KSYM_SYMBOL_LEN]; 90 91 92 start = __start_syscalls_metadata; 93 stop = __stop_syscalls_metadata; 94 kallsyms_lookup(syscall, NULL, NULL, NULL, str); 95 96 if (arch_syscall_match_sym_name(str, "sys_ni_syscall")) 97 return NULL; 98 99 for ( ; start < stop; start++) { 100 if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name)) 101 return *start; 102 } 103 return NULL; 104 } 105 106 static struct syscall_metadata *syscall_nr_to_meta(int nr) 107 { 108 if (IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) 109 return xa_load(&syscalls_metadata_sparse, (unsigned long)nr); 110 111 if (!syscalls_metadata || nr >= NR_syscalls || nr < 0) 112 return NULL; 113 114 return syscalls_metadata[nr]; 115 } 116 117 const char *get_syscall_name(int syscall) 118 { 119 struct syscall_metadata *entry; 120 121 entry = syscall_nr_to_meta(syscall); 122 if (!entry) 123 return NULL; 124 125 return entry->name; 126 } 127 128 /* Added to user strings or arrays when max limit is reached */ 129 #define EXTRA "..." 130 131 static void get_dynamic_len_ptr(struct syscall_trace_enter *trace, 132 struct syscall_metadata *entry, 133 int *offset_p, int *len_p, unsigned char **ptr_p) 134 { 135 unsigned char *ptr; 136 int offset = *offset_p; 137 int val; 138 139 /* This arg points to a user space string */ 140 ptr = (void *)trace->args + sizeof(long) * entry->nb_args + offset; 141 val = *(int *)ptr; 142 143 /* The value is a dynamic string (len << 16 | offset) */ 144 ptr = (void *)trace + (val & 0xffff); 145 *len_p = val >> 16; 146 offset += 4; 147 148 *ptr_p = ptr; 149 *offset_p = offset; 150 } 151 152 static enum print_line_t 153 sys_enter_openat_print(struct syscall_trace_enter *trace, struct syscall_metadata *entry, 154 struct trace_seq *s, struct trace_event *event) 155 { 156 unsigned char *ptr; 157 int offset = 0; 158 int bits, len; 159 bool done = false; 160 static const struct trace_print_flags __flags[] = 161 { 162 { O_TMPFILE, "O_TMPFILE" }, 163 { O_WRONLY, "O_WRONLY" }, 164 { O_RDWR, "O_RDWR" }, 165 { O_CREAT, "O_CREAT" }, 166 { O_EXCL, "O_EXCL" }, 167 { O_NOCTTY, "O_NOCTTY" }, 168 { O_TRUNC, "O_TRUNC" }, 169 { O_APPEND, "O_APPEND" }, 170 { O_NONBLOCK, "O_NONBLOCK" }, 171 { O_DSYNC, "O_DSYNC" }, 172 { O_DIRECT, "O_DIRECT" }, 173 { O_LARGEFILE, "O_LARGEFILE" }, 174 { O_DIRECTORY, "O_DIRECTORY" }, 175 { O_NOFOLLOW, "O_NOFOLLOW" }, 176 { O_NOATIME, "O_NOATIME" }, 177 { O_CLOEXEC, "O_CLOEXEC" }, 178 }; 179 180 trace_seq_printf(s, "%s(", entry->name); 181 182 for (int i = 0; !done && i < entry->nb_args; i++) { 183 184 if (trace_seq_has_overflowed(s)) 185 goto end; 186 187 if (i) 188 trace_seq_puts(s, ", "); 189 190 switch (i) { 191 case 2: 192 bits = trace->args[2]; 193 194 trace_seq_puts(s, "flags: "); 195 196 /* No need to show mode when not creating the file */ 197 if (!(bits & (O_CREAT|O_TMPFILE))) 198 done = true; 199 200 if (!(bits & O_ACCMODE)) { 201 if (!bits) { 202 trace_seq_puts(s, "O_RDONLY"); 203 continue; 204 } 205 trace_seq_puts(s, "O_RDONLY|"); 206 } 207 208 trace_print_flags_seq(s, "|", bits, __flags, ARRAY_SIZE(__flags)); 209 /* 210 * trace_print_flags_seq() adds a '\0' to the 211 * buffer, but this needs to append more to the seq. 212 */ 213 if (!trace_seq_has_overflowed(s)) 214 trace_seq_pop(s); 215 216 continue; 217 case 3: 218 trace_seq_printf(s, "%s: 0%03o", entry->args[i], 219 (unsigned int)trace->args[i]); 220 continue; 221 } 222 223 trace_seq_printf(s, "%s: %lu", entry->args[i], 224 trace->args[i]); 225 226 if (!(BIT(i) & entry->user_mask)) 227 continue; 228 229 get_dynamic_len_ptr(trace, entry, &offset, &len, &ptr); 230 trace_seq_printf(s, " \"%.*s\"", len, ptr); 231 } 232 233 trace_seq_putc(s, ')'); 234 end: 235 trace_seq_putc(s, '\n'); 236 237 return trace_handle_return(s); 238 } 239 240 static enum print_line_t 241 print_syscall_enter(struct trace_iterator *iter, int flags, 242 struct trace_event *event) 243 { 244 struct trace_array *tr = iter->tr; 245 struct trace_seq *s = &iter->seq; 246 struct trace_entry *ent = iter->ent; 247 struct syscall_trace_enter *trace; 248 struct syscall_metadata *entry; 249 int i, syscall, val, len; 250 unsigned char *ptr; 251 int offset = 0; 252 253 trace = (typeof(trace))ent; 254 syscall = trace->nr; 255 entry = syscall_nr_to_meta(syscall); 256 257 if (!entry) 258 goto end; 259 260 if (entry->enter_event->event.type != ent->type) { 261 WARN_ON_ONCE(1); 262 goto end; 263 } 264 265 switch (entry->syscall_nr) { 266 case __NR_openat: 267 if (!tr || !(tr->trace_flags & TRACE_ITER(VERBOSE))) 268 return sys_enter_openat_print(trace, entry, s, event); 269 break; 270 default: 271 break; 272 } 273 274 trace_seq_printf(s, "%s(", entry->name); 275 276 for (i = 0; i < entry->nb_args; i++) { 277 bool printable = false; 278 char *str; 279 280 if (trace_seq_has_overflowed(s)) 281 goto end; 282 283 if (i) 284 trace_seq_puts(s, ", "); 285 286 /* parameter types */ 287 if (tr && tr->trace_flags & TRACE_ITER(VERBOSE)) 288 trace_seq_printf(s, "%s ", entry->types[i]); 289 290 /* parameter values */ 291 if (trace->args[i] < 10) 292 trace_seq_printf(s, "%s: %lu", entry->args[i], 293 trace->args[i]); 294 else 295 trace_seq_printf(s, "%s: 0x%lx", entry->args[i], 296 trace->args[i]); 297 298 if (!(BIT(i) & entry->user_mask)) 299 continue; 300 301 get_dynamic_len_ptr(trace, entry, &offset, &len, &ptr); 302 303 if (entry->user_arg_size < 0 || entry->user_arg_is_str) { 304 trace_seq_printf(s, " \"%.*s\"", len, ptr); 305 continue; 306 } 307 308 val = trace->args[entry->user_arg_size]; 309 310 str = ptr; 311 trace_seq_puts(s, " ("); 312 for (int x = 0; x < len; x++, ptr++) { 313 if (isascii(*ptr) && isprint(*ptr)) 314 printable = true; 315 if (x) 316 trace_seq_putc(s, ':'); 317 trace_seq_printf(s, "%02x", *ptr); 318 } 319 if (len < val) 320 trace_seq_printf(s, ", %s", EXTRA); 321 322 trace_seq_putc(s, ')'); 323 324 /* If nothing is printable, don't bother printing anything */ 325 if (!printable) 326 continue; 327 328 trace_seq_puts(s, " \""); 329 for (int x = 0; x < len; x++) { 330 if (isascii(str[x]) && isprint(str[x])) 331 trace_seq_putc(s, str[x]); 332 else 333 trace_seq_putc(s, '.'); 334 } 335 if (len < val) 336 trace_seq_printf(s, "\"%s", EXTRA); 337 else 338 trace_seq_putc(s, '"'); 339 } 340 341 trace_seq_putc(s, ')'); 342 end: 343 trace_seq_putc(s, '\n'); 344 345 return trace_handle_return(s); 346 } 347 348 static enum print_line_t 349 print_syscall_exit(struct trace_iterator *iter, int flags, 350 struct trace_event *event) 351 { 352 struct trace_seq *s = &iter->seq; 353 struct trace_entry *ent = iter->ent; 354 struct syscall_trace_exit *trace; 355 int syscall; 356 struct syscall_metadata *entry; 357 358 trace = (typeof(trace))ent; 359 syscall = trace->nr; 360 entry = syscall_nr_to_meta(syscall); 361 362 if (!entry) { 363 trace_seq_putc(s, '\n'); 364 goto out; 365 } 366 367 if (entry->exit_event->event.type != ent->type) { 368 WARN_ON_ONCE(1); 369 return TRACE_TYPE_UNHANDLED; 370 } 371 372 trace_seq_printf(s, "%s -> 0x%lx\n", entry->name, 373 trace->ret); 374 375 out: 376 return trace_handle_return(s); 377 } 378 379 #define SYSCALL_FIELD(_type, _name) { \ 380 .type = #_type, .name = #_name, \ 381 .size = sizeof(_type), .align = __alignof__(_type), \ 382 .is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER } 383 384 /* When len=0, we just calculate the needed length */ 385 #define LEN_OR_ZERO (len ? len - pos : 0) 386 387 static int __init 388 sys_enter_openat_print_fmt(struct syscall_metadata *entry, char *buf, int len) 389 { 390 int pos = 0; 391 392 pos += snprintf(buf + pos, LEN_OR_ZERO, 393 "\"dfd: 0x%%08lx, filename: 0x%%08lx \\\"%%s\\\", flags: %%s%%s, mode: 0%%03o\","); 394 pos += snprintf(buf + pos, LEN_OR_ZERO, 395 " ((unsigned long)(REC->dfd)),"); 396 pos += snprintf(buf + pos, LEN_OR_ZERO, 397 " ((unsigned long)(REC->filename)),"); 398 pos += snprintf(buf + pos, LEN_OR_ZERO, 399 " __get_str(__filename_val),"); 400 pos += snprintf(buf + pos, LEN_OR_ZERO, 401 " (REC->flags & ~3) && !(REC->flags & 3) ? \"O_RDONLY|\" : \"\", "); 402 pos += snprintf(buf + pos, LEN_OR_ZERO, 403 " REC->flags ? __print_flags(REC->flags, \"|\", "); 404 pos += snprintf(buf + pos, LEN_OR_ZERO, 405 "{ 0x%x, \"O_WRONLY\" }, ", O_WRONLY); 406 pos += snprintf(buf + pos, LEN_OR_ZERO, 407 "{ 0x%x, \"O_RDWR\" }, ", O_RDWR); 408 pos += snprintf(buf + pos, LEN_OR_ZERO, 409 "{ 0x%x, \"O_CREAT\" }, ", O_CREAT); 410 pos += snprintf(buf + pos, LEN_OR_ZERO, 411 "{ 0x%x, \"O_EXCL\" }, ", O_EXCL); 412 pos += snprintf(buf + pos, LEN_OR_ZERO, 413 "{ 0x%x, \"O_NOCTTY\" }, ", O_NOCTTY); 414 pos += snprintf(buf + pos, LEN_OR_ZERO, 415 "{ 0x%x, \"O_TRUNC\" }, ", O_TRUNC); 416 pos += snprintf(buf + pos, LEN_OR_ZERO, 417 "{ 0x%x, \"O_APPEND\" }, ", O_APPEND); 418 pos += snprintf(buf + pos, LEN_OR_ZERO, 419 "{ 0x%x, \"O_NONBLOCK\" }, ", O_NONBLOCK); 420 pos += snprintf(buf + pos, LEN_OR_ZERO, 421 "{ 0x%x, \"O_DSYNC\" }, ", O_DSYNC); 422 pos += snprintf(buf + pos, LEN_OR_ZERO, 423 "{ 0x%x, \"O_DIRECT\" }, ", O_DIRECT); 424 pos += snprintf(buf + pos, LEN_OR_ZERO, 425 "{ 0x%x, \"O_LARGEFILE\" }, ", O_LARGEFILE); 426 pos += snprintf(buf + pos, LEN_OR_ZERO, 427 "{ 0x%x, \"O_DIRECTORY\" }, ", O_DIRECTORY); 428 pos += snprintf(buf + pos, LEN_OR_ZERO, 429 "{ 0x%x, \"O_NOFOLLOW\" }, ", O_NOFOLLOW); 430 pos += snprintf(buf + pos, LEN_OR_ZERO, 431 "{ 0x%x, \"O_NOATIME\" }, ", O_NOATIME); 432 pos += snprintf(buf + pos, LEN_OR_ZERO, 433 "{ 0x%x, \"O_CLOEXEC\" }) : \"O_RDONLY\", ", O_CLOEXEC); 434 435 pos += snprintf(buf + pos, LEN_OR_ZERO, 436 " ((unsigned long)(REC->mode))"); 437 return pos; 438 } 439 440 static int __init 441 __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len) 442 { 443 bool is_string = entry->user_arg_is_str; 444 int i; 445 int pos = 0; 446 447 switch (entry->syscall_nr) { 448 case __NR_openat: 449 return sys_enter_openat_print_fmt(entry, buf, len); 450 default: 451 break; 452 } 453 454 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 455 for (i = 0; i < entry->nb_args; i++) { 456 if (i) 457 pos += snprintf(buf + pos, LEN_OR_ZERO, ", "); 458 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx", 459 entry->args[i], sizeof(unsigned long)); 460 461 if (!(BIT(i) & entry->user_mask)) 462 continue; 463 464 /* Add the format for the user space string or array */ 465 if (entry->user_arg_size < 0 || is_string) 466 pos += snprintf(buf + pos, LEN_OR_ZERO, " \\\"%%s\\\""); 467 else 468 pos += snprintf(buf + pos, LEN_OR_ZERO, " (%%s)"); 469 } 470 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 471 472 for (i = 0; i < entry->nb_args; i++) { 473 pos += snprintf(buf + pos, LEN_OR_ZERO, 474 ", ((unsigned long)(REC->%s))", entry->args[i]); 475 if (!(BIT(i) & entry->user_mask)) 476 continue; 477 /* The user space data for arg has name __<arg>_val */ 478 if (entry->user_arg_size < 0 || is_string) { 479 pos += snprintf(buf + pos, LEN_OR_ZERO, ", __get_str(__%s_val)", 480 entry->args[i]); 481 } else { 482 pos += snprintf(buf + pos, LEN_OR_ZERO, ", __print_dynamic_array(__%s_val, 1)", 483 entry->args[i]); 484 } 485 } 486 487 #undef LEN_OR_ZERO 488 489 /* return the length of print_fmt */ 490 return pos; 491 } 492 493 static int __init set_syscall_print_fmt(struct trace_event_call *call) 494 { 495 char *print_fmt; 496 int len; 497 struct syscall_metadata *entry = call->data; 498 499 if (entry->enter_event != call) { 500 call->print_fmt = "\"0x%lx\", REC->ret"; 501 return 0; 502 } 503 504 /* First: called with 0 length to calculate the needed length */ 505 len = __set_enter_print_fmt(entry, NULL, 0); 506 507 print_fmt = kmalloc(len + 1, GFP_KERNEL); 508 if (!print_fmt) 509 return -ENOMEM; 510 511 /* Second: actually write the @print_fmt */ 512 __set_enter_print_fmt(entry, print_fmt, len + 1); 513 call->print_fmt = print_fmt; 514 515 return 0; 516 } 517 518 static void __init free_syscall_print_fmt(struct trace_event_call *call) 519 { 520 struct syscall_metadata *entry = call->data; 521 522 if (entry->enter_event == call) 523 kfree(call->print_fmt); 524 } 525 526 static int __init syscall_enter_define_fields(struct trace_event_call *call) 527 { 528 struct syscall_trace_enter trace; 529 struct syscall_metadata *meta = call->data; 530 unsigned long mask; 531 char *arg; 532 int offset = offsetof(typeof(trace), args); 533 int ret = 0; 534 int len; 535 int i; 536 537 for (i = 0; i < meta->nb_args; i++) { 538 ret = trace_define_field(call, meta->types[i], 539 meta->args[i], offset, 540 sizeof(unsigned long), 0, 541 FILTER_OTHER); 542 if (ret) 543 break; 544 offset += sizeof(unsigned long); 545 } 546 547 if (ret || !meta->user_mask) 548 return ret; 549 550 mask = meta->user_mask; 551 552 while (mask) { 553 int idx = ffs(mask) - 1; 554 mask &= ~BIT(idx); 555 556 /* 557 * User space data is faulted into a temporary buffer and then 558 * added as a dynamic string or array to the end of the event. 559 * The user space data name for the arg pointer is 560 * "__<arg>_val". 561 */ 562 len = strlen(meta->args[idx]) + sizeof("___val"); 563 arg = kmalloc(len, GFP_KERNEL); 564 if (WARN_ON_ONCE(!arg)) { 565 meta->user_mask = 0; 566 return -ENOMEM; 567 } 568 569 snprintf(arg, len, "__%s_val", meta->args[idx]); 570 571 ret = trace_define_field(call, "__data_loc char[]", 572 arg, offset, sizeof(int), 0, 573 FILTER_OTHER); 574 if (ret) { 575 kfree(arg); 576 break; 577 } 578 offset += 4; 579 } 580 return ret; 581 } 582 583 /* 584 * Create a per CPU temporary buffer to copy user space pointers into. 585 * 586 * SYSCALL_FAULT_USER_MAX is the amount to copy from user space. 587 * (defined in kernel/trace/trace.h) 588 589 * SYSCALL_FAULT_ARG_SZ is the amount to copy from user space plus the 590 * nul terminating byte and possibly appended EXTRA (4 bytes). 591 * 592 * SYSCALL_FAULT_BUF_SZ holds the size of the per CPU buffer to use 593 * to copy memory from user space addresses into that will hold 594 * 3 args as only 3 args are allowed to be copied from system calls. 595 */ 596 #define SYSCALL_FAULT_ARG_SZ (SYSCALL_FAULT_USER_MAX + 1 + 4) 597 #define SYSCALL_FAULT_MAX_CNT 3 598 #define SYSCALL_FAULT_BUF_SZ (SYSCALL_FAULT_ARG_SZ * SYSCALL_FAULT_MAX_CNT) 599 600 /* Use the tracing per CPU buffer infrastructure to copy from user space */ 601 struct syscall_user_buffer { 602 struct trace_user_buf_info buf; 603 struct rcu_head rcu; 604 }; 605 606 static struct syscall_user_buffer *syscall_buffer; 607 608 static int syscall_fault_buffer_enable(void) 609 { 610 struct syscall_user_buffer *sbuf; 611 int ret; 612 613 lockdep_assert_held(&syscall_trace_lock); 614 615 if (syscall_buffer) { 616 trace_user_fault_get(&syscall_buffer->buf); 617 return 0; 618 } 619 620 sbuf = kmalloc_obj(*sbuf); 621 if (!sbuf) 622 return -ENOMEM; 623 624 ret = trace_user_fault_init(&sbuf->buf, SYSCALL_FAULT_BUF_SZ); 625 if (ret < 0) { 626 kfree(sbuf); 627 return ret; 628 } 629 630 WRITE_ONCE(syscall_buffer, sbuf); 631 632 return 0; 633 } 634 635 static void rcu_free_syscall_buffer(struct rcu_head *rcu) 636 { 637 struct syscall_user_buffer *sbuf = 638 container_of(rcu, struct syscall_user_buffer, rcu); 639 640 trace_user_fault_destroy(&sbuf->buf); 641 kfree(sbuf); 642 } 643 644 645 static void syscall_fault_buffer_disable(void) 646 { 647 struct syscall_user_buffer *sbuf = syscall_buffer; 648 649 lockdep_assert_held(&syscall_trace_lock); 650 651 if (trace_user_fault_put(&sbuf->buf)) 652 return; 653 654 WRITE_ONCE(syscall_buffer, NULL); 655 call_rcu_tasks_trace(&sbuf->rcu, rcu_free_syscall_buffer); 656 } 657 658 struct syscall_args { 659 char *ptr_array[SYSCALL_FAULT_MAX_CNT]; 660 int read[SYSCALL_FAULT_MAX_CNT]; 661 int uargs; 662 }; 663 664 static int syscall_copy_user(char *buf, const char __user *ptr, 665 size_t size, void *data) 666 { 667 struct syscall_args *args = data; 668 int ret; 669 670 for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) { 671 ptr = (char __user *)args->ptr_array[i]; 672 ret = strncpy_from_user(buf, ptr, size); 673 args->read[i] = ret; 674 } 675 return 0; 676 } 677 678 static int syscall_copy_user_array(char *buf, const char __user *ptr, 679 size_t size, void *data) 680 { 681 struct syscall_args *args = data; 682 int ret; 683 684 for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) { 685 ptr = (char __user *)args->ptr_array[i]; 686 ret = __copy_from_user(buf, ptr, size); 687 args->read[i] = ret ? -1 : size; 688 } 689 return 0; 690 } 691 692 static char *sys_fault_user(unsigned int buf_size, 693 struct syscall_metadata *sys_data, 694 struct syscall_user_buffer *sbuf, 695 unsigned long *args, 696 unsigned int data_size[SYSCALL_FAULT_MAX_CNT]) 697 { 698 trace_user_buf_copy syscall_copy = syscall_copy_user; 699 unsigned long mask = sys_data->user_mask; 700 unsigned long size = SYSCALL_FAULT_ARG_SZ - 1; 701 struct syscall_args sargs; 702 bool array = false; 703 char *buffer; 704 char *buf; 705 int ret; 706 int i = 0; 707 708 /* The extra is appended to the user data in the buffer */ 709 BUILD_BUG_ON(SYSCALL_FAULT_USER_MAX + sizeof(EXTRA) >= 710 SYSCALL_FAULT_ARG_SZ); 711 712 /* 713 * If this system call event has a size argument, use 714 * it to define how much of user space memory to read, 715 * and read it as an array and not a string. 716 */ 717 if (sys_data->user_arg_size >= 0) { 718 array = true; 719 size = args[sys_data->user_arg_size]; 720 if (size > SYSCALL_FAULT_ARG_SZ - 1) 721 size = SYSCALL_FAULT_ARG_SZ - 1; 722 syscall_copy = syscall_copy_user_array; 723 } 724 725 while (mask) { 726 int idx = ffs(mask) - 1; 727 mask &= ~BIT(idx); 728 729 if (WARN_ON_ONCE(i == SYSCALL_FAULT_MAX_CNT)) 730 break; 731 732 /* Get the pointer to user space memory to read */ 733 sargs.ptr_array[i++] = (char *)args[idx]; 734 } 735 736 sargs.uargs = i; 737 738 /* Clear the values that are not used */ 739 for (; i < SYSCALL_FAULT_MAX_CNT; i++) { 740 data_size[i] = -1; /* Denotes no pointer */ 741 } 742 743 /* A zero size means do not even try */ 744 if (!buf_size) 745 return NULL; 746 747 buffer = trace_user_fault_read(&sbuf->buf, NULL, size, 748 syscall_copy, &sargs); 749 if (!buffer) 750 return NULL; 751 752 buf = buffer; 753 for (i = 0; i < sargs.uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) { 754 755 ret = sargs.read[i]; 756 if (ret < 0) 757 continue; 758 buf[ret] = '\0'; 759 760 /* For strings, replace any non-printable characters with '.' */ 761 if (!array) { 762 for (int x = 0; x < ret; x++) { 763 if (!isprint(buf[x])) 764 buf[x] = '.'; 765 } 766 767 size = min(buf_size, SYSCALL_FAULT_USER_MAX); 768 769 /* 770 * If the text was truncated due to our max limit, 771 * add "..." to the string. 772 */ 773 if (ret > size) { 774 strscpy(buf + size, EXTRA, sizeof(EXTRA)); 775 ret = size + sizeof(EXTRA); 776 } else { 777 buf[ret++] = '\0'; 778 } 779 } else { 780 ret = min((unsigned int)ret, buf_size); 781 } 782 data_size[i] = ret; 783 } 784 785 return buffer; 786 } 787 788 static int 789 syscall_get_data(struct syscall_metadata *sys_data, unsigned long *args, 790 char **buffer, int *size, int *user_sizes, int *uargs, 791 int buf_size) 792 { 793 struct syscall_user_buffer *sbuf; 794 int i; 795 796 /* If the syscall_buffer is NULL, tracing is being shutdown */ 797 sbuf = READ_ONCE(syscall_buffer); 798 if (!sbuf) 799 return -1; 800 801 *buffer = sys_fault_user(buf_size, sys_data, sbuf, args, user_sizes); 802 /* 803 * user_size is the amount of data to append. 804 * Need to add 4 for the meta field that points to 805 * the user memory at the end of the event and also 806 * stores its size. 807 */ 808 for (i = 0; i < SYSCALL_FAULT_MAX_CNT; i++) { 809 if (user_sizes[i] < 0) 810 break; 811 *size += user_sizes[i] + 4; 812 } 813 /* Save the number of user read arguments of this syscall */ 814 *uargs = i; 815 return 0; 816 } 817 818 static void syscall_put_data(struct syscall_metadata *sys_data, 819 struct syscall_trace_enter *entry, 820 char *buffer, int size, int *user_sizes, int uargs) 821 { 822 char *buf = buffer; 823 void *ptr; 824 int val; 825 826 /* 827 * Set the pointer to point to the meta data of the event 828 * that has information about the stored user space memory. 829 */ 830 ptr = (void *)entry->args + sizeof(unsigned long) * sys_data->nb_args; 831 832 /* 833 * The meta data will store the offset of the user data from 834 * the beginning of the event. That is after the static arguments 835 * and the meta data fields. 836 */ 837 val = (ptr - (void *)entry) + 4 * uargs; 838 839 for (int i = 0; i < uargs; i++) { 840 841 if (i) 842 val += user_sizes[i - 1]; 843 844 /* Store the offset and the size into the meta data */ 845 *(int *)ptr = val | (user_sizes[i] << 16); 846 847 /* Skip the meta data */ 848 ptr += 4; 849 } 850 851 for (int i = 0; i < uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) { 852 /* Nothing to do if the user space was empty or faulted */ 853 if (!user_sizes[i]) 854 continue; 855 856 memcpy(ptr, buf, user_sizes[i]); 857 ptr += user_sizes[i]; 858 } 859 } 860 861 static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id) 862 { 863 struct trace_array *tr = data; 864 struct trace_event_file *trace_file; 865 struct syscall_trace_enter *entry; 866 struct syscall_metadata *sys_data; 867 struct trace_event_buffer fbuffer; 868 unsigned long args[6]; 869 char *user_ptr; 870 int user_sizes[SYSCALL_FAULT_MAX_CNT] = {}; 871 int syscall_nr; 872 int size = 0; 873 int uargs = 0; 874 bool mayfault; 875 876 /* 877 * Syscall probe called with preemption enabled, but the ring 878 * buffer and per-cpu data require preemption to be disabled. 879 */ 880 might_fault(); 881 882 syscall_nr = trace_get_syscall_nr(current, regs); 883 if (syscall_nr < 0 || syscall_nr >= NR_syscalls) 884 return; 885 886 trace_file = READ_ONCE(tr->enter_syscall_files[syscall_nr]); 887 if (!trace_file) 888 return; 889 890 if (trace_trigger_soft_disabled(trace_file)) 891 return; 892 893 sys_data = syscall_nr_to_meta(syscall_nr); 894 if (!sys_data) 895 return; 896 897 /* Check if this syscall event faults in user space memory */ 898 mayfault = sys_data->user_mask != 0; 899 900 guard(preempt_notrace)(); 901 902 syscall_get_arguments(current, regs, args); 903 904 if (mayfault) { 905 if (syscall_get_data(sys_data, args, &user_ptr, 906 &size, user_sizes, &uargs, tr->syscall_buf_sz) < 0) 907 return; 908 } 909 910 size += sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args; 911 912 entry = trace_event_buffer_reserve(&fbuffer, trace_file, size); 913 if (!entry) 914 return; 915 916 entry = ring_buffer_event_data(fbuffer.event); 917 entry->nr = syscall_nr; 918 919 memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args); 920 921 if (mayfault) 922 syscall_put_data(sys_data, entry, user_ptr, size, user_sizes, uargs); 923 924 trace_event_buffer_commit(&fbuffer); 925 } 926 927 static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) 928 { 929 struct trace_array *tr = data; 930 struct trace_event_file *trace_file; 931 struct syscall_trace_exit *entry; 932 struct syscall_metadata *sys_data; 933 struct trace_event_buffer fbuffer; 934 int syscall_nr; 935 936 /* 937 * Syscall probe called with preemption enabled, but the ring 938 * buffer and per-cpu data require preemption to be disabled. 939 */ 940 might_fault(); 941 guard(preempt_notrace)(); 942 943 syscall_nr = trace_get_syscall_nr(current, regs); 944 if (syscall_nr < 0 || syscall_nr >= NR_syscalls) 945 return; 946 947 trace_file = READ_ONCE(tr->exit_syscall_files[syscall_nr]); 948 if (!trace_file) 949 return; 950 951 if (trace_trigger_soft_disabled(trace_file)) 952 return; 953 954 sys_data = syscall_nr_to_meta(syscall_nr); 955 if (!sys_data) 956 return; 957 958 entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry)); 959 if (!entry) 960 return; 961 962 entry = ring_buffer_event_data(fbuffer.event); 963 entry->nr = syscall_nr; 964 entry->ret = syscall_get_return_value(current, regs); 965 966 trace_event_buffer_commit(&fbuffer); 967 } 968 969 static int reg_event_syscall_enter(struct trace_event_file *file, 970 struct trace_event_call *call) 971 { 972 struct syscall_metadata *sys_data = call->data; 973 struct trace_array *tr = file->tr; 974 int ret = 0; 975 int num; 976 977 num = sys_data->syscall_nr; 978 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls)) 979 return -ENOSYS; 980 guard(mutex)(&syscall_trace_lock); 981 if (sys_data->user_mask) { 982 ret = syscall_fault_buffer_enable(); 983 if (ret < 0) 984 return ret; 985 } 986 if (!tr->sys_refcount_enter) { 987 ret = register_trace_sys_enter(ftrace_syscall_enter, tr); 988 if (ret < 0) { 989 if (sys_data->user_mask) 990 syscall_fault_buffer_disable(); 991 return ret; 992 } 993 } 994 WRITE_ONCE(tr->enter_syscall_files[num], file); 995 tr->sys_refcount_enter++; 996 return 0; 997 } 998 999 static void unreg_event_syscall_enter(struct trace_event_file *file, 1000 struct trace_event_call *call) 1001 { 1002 struct syscall_metadata *sys_data = call->data; 1003 struct trace_array *tr = file->tr; 1004 int num; 1005 1006 num = sys_data->syscall_nr; 1007 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls)) 1008 return; 1009 guard(mutex)(&syscall_trace_lock); 1010 tr->sys_refcount_enter--; 1011 WRITE_ONCE(tr->enter_syscall_files[num], NULL); 1012 if (!tr->sys_refcount_enter) 1013 unregister_trace_sys_enter(ftrace_syscall_enter, tr); 1014 if (sys_data->user_mask) 1015 syscall_fault_buffer_disable(); 1016 } 1017 1018 static int reg_event_syscall_exit(struct trace_event_file *file, 1019 struct trace_event_call *call) 1020 { 1021 struct trace_array *tr = file->tr; 1022 int ret = 0; 1023 int num; 1024 1025 num = ((struct syscall_metadata *)call->data)->syscall_nr; 1026 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls)) 1027 return -ENOSYS; 1028 mutex_lock(&syscall_trace_lock); 1029 if (!tr->sys_refcount_exit) 1030 ret = register_trace_sys_exit(ftrace_syscall_exit, tr); 1031 if (!ret) { 1032 WRITE_ONCE(tr->exit_syscall_files[num], file); 1033 tr->sys_refcount_exit++; 1034 } 1035 mutex_unlock(&syscall_trace_lock); 1036 return ret; 1037 } 1038 1039 static void unreg_event_syscall_exit(struct trace_event_file *file, 1040 struct trace_event_call *call) 1041 { 1042 struct trace_array *tr = file->tr; 1043 int num; 1044 1045 num = ((struct syscall_metadata *)call->data)->syscall_nr; 1046 if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls)) 1047 return; 1048 mutex_lock(&syscall_trace_lock); 1049 tr->sys_refcount_exit--; 1050 WRITE_ONCE(tr->exit_syscall_files[num], NULL); 1051 if (!tr->sys_refcount_exit) 1052 unregister_trace_sys_exit(ftrace_syscall_exit, tr); 1053 mutex_unlock(&syscall_trace_lock); 1054 } 1055 1056 /* 1057 * For system calls that reference user space memory that can 1058 * be recorded into the event, set the system call meta data's user_mask 1059 * to the "args" index that points to the user space memory to retrieve. 1060 */ 1061 static void check_faultable_syscall(struct trace_event_call *call, int nr) 1062 { 1063 struct syscall_metadata *sys_data = call->data; 1064 unsigned long mask; 1065 1066 /* Only work on entry */ 1067 if (sys_data->enter_event != call) 1068 return; 1069 1070 sys_data->user_arg_size = -1; 1071 1072 switch (nr) { 1073 /* user arg 1 with size arg at 2 */ 1074 case __NR_write: 1075 #ifdef __NR_mq_timedsend 1076 case __NR_mq_timedsend: 1077 #endif 1078 case __NR_pwrite64: 1079 sys_data->user_mask = BIT(1); 1080 sys_data->user_arg_size = 2; 1081 break; 1082 /* user arg 0 with size arg at 1 as string */ 1083 case __NR_setdomainname: 1084 case __NR_sethostname: 1085 sys_data->user_mask = BIT(0); 1086 sys_data->user_arg_size = 1; 1087 sys_data->user_arg_is_str = 1; 1088 break; 1089 #ifdef __NR_kexec_file_load 1090 /* user arg 4 with size arg at 3 as string */ 1091 case __NR_kexec_file_load: 1092 sys_data->user_mask = BIT(4); 1093 sys_data->user_arg_size = 3; 1094 sys_data->user_arg_is_str = 1; 1095 break; 1096 #endif 1097 /* user arg at position 0 */ 1098 #ifdef __NR_access 1099 case __NR_access: 1100 #endif 1101 case __NR_acct: 1102 case __NR_chdir: 1103 #ifdef __NR_chown 1104 case __NR_chown: 1105 #endif 1106 #ifdef __NR_chmod 1107 case __NR_chmod: 1108 #endif 1109 case __NR_chroot: 1110 #ifdef __NR_creat 1111 case __NR_creat: 1112 #endif 1113 case __NR_delete_module: 1114 case __NR_execve: 1115 case __NR_fsopen: 1116 #ifdef __NR_lchown 1117 case __NR_lchown: 1118 #endif 1119 #ifdef __NR_open 1120 case __NR_open: 1121 #endif 1122 case __NR_memfd_create: 1123 #ifdef __NR_mkdir 1124 case __NR_mkdir: 1125 #endif 1126 #ifdef __NR_mknod 1127 case __NR_mknod: 1128 #endif 1129 case __NR_mq_open: 1130 case __NR_mq_unlink: 1131 #ifdef __NR_readlink 1132 case __NR_readlink: 1133 #endif 1134 #ifdef __NR_rmdir 1135 case __NR_rmdir: 1136 #endif 1137 case __NR_shmdt: 1138 #ifdef __NR_statfs 1139 case __NR_statfs: 1140 #endif 1141 case __NR_swapon: 1142 case __NR_swapoff: 1143 #ifdef __NR_truncate 1144 case __NR_truncate: 1145 #endif 1146 #ifdef __NR_unlink 1147 case __NR_unlink: 1148 #endif 1149 case __NR_umount2: 1150 #ifdef __NR_utime 1151 case __NR_utime: 1152 #endif 1153 #ifdef __NR_utimes 1154 case __NR_utimes: 1155 #endif 1156 sys_data->user_mask = BIT(0); 1157 break; 1158 /* user arg at position 1 */ 1159 case __NR_execveat: 1160 case __NR_faccessat: 1161 case __NR_faccessat2: 1162 case __NR_finit_module: 1163 case __NR_fchmodat: 1164 case __NR_fchmodat2: 1165 case __NR_fchownat: 1166 case __NR_fgetxattr: 1167 case __NR_flistxattr: 1168 case __NR_fsetxattr: 1169 case __NR_fspick: 1170 case __NR_fremovexattr: 1171 #ifdef __NR_futimesat 1172 case __NR_futimesat: 1173 #endif 1174 case __NR_inotify_add_watch: 1175 case __NR_mkdirat: 1176 case __NR_mknodat: 1177 case __NR_mount_setattr: 1178 case __NR_name_to_handle_at: 1179 #ifdef __NR_newfstatat 1180 case __NR_newfstatat: 1181 #endif 1182 case __NR_openat: 1183 case __NR_openat2: 1184 case __NR_open_tree: 1185 case __NR_open_tree_attr: 1186 case __NR_readlinkat: 1187 case __NR_quotactl: 1188 case __NR_syslog: 1189 case __NR_statx: 1190 case __NR_unlinkat: 1191 #ifdef __NR_utimensat 1192 case __NR_utimensat: 1193 #endif 1194 sys_data->user_mask = BIT(1); 1195 break; 1196 /* user arg at position 2 */ 1197 case __NR_init_module: 1198 case __NR_fsconfig: 1199 sys_data->user_mask = BIT(2); 1200 break; 1201 /* user arg at position 4 */ 1202 case __NR_fanotify_mark: 1203 sys_data->user_mask = BIT(4); 1204 break; 1205 /* 2 user args, 0 and 1 */ 1206 case __NR_add_key: 1207 case __NR_getxattr: 1208 case __NR_lgetxattr: 1209 case __NR_lremovexattr: 1210 #ifdef __NR_link 1211 case __NR_link: 1212 #endif 1213 case __NR_listxattr: 1214 case __NR_llistxattr: 1215 case __NR_lsetxattr: 1216 case __NR_pivot_root: 1217 case __NR_removexattr: 1218 #ifdef __NR_rename 1219 case __NR_rename: 1220 #endif 1221 case __NR_request_key: 1222 case __NR_setxattr: 1223 #ifdef __NR_symlink 1224 case __NR_symlink: 1225 #endif 1226 sys_data->user_mask = BIT(0) | BIT(1); 1227 break; 1228 /* 2 user args, 0 and 2 */ 1229 case __NR_symlinkat: 1230 sys_data->user_mask = BIT(0) | BIT(2); 1231 break; 1232 /* 2 user args, 1 and 3 */ 1233 case __NR_getxattrat: 1234 case __NR_linkat: 1235 case __NR_listxattrat: 1236 case __NR_move_mount: 1237 #ifdef __NR_renameat 1238 case __NR_renameat: 1239 #endif 1240 case __NR_renameat2: 1241 case __NR_removexattrat: 1242 case __NR_setxattrat: 1243 sys_data->user_mask = BIT(1) | BIT(3); 1244 break; 1245 case __NR_mount: /* Just dev_name and dir_name, TODO add type */ 1246 sys_data->user_mask = BIT(0) | BIT(1) | BIT(2); 1247 break; 1248 default: 1249 sys_data->user_mask = 0; 1250 return; 1251 } 1252 1253 if (sys_data->user_arg_size < 0) 1254 return; 1255 1256 /* 1257 * The user_arg_size can only be used when the system call 1258 * is reading only a single address from user space. 1259 */ 1260 mask = sys_data->user_mask; 1261 if (WARN_ON(mask & (mask - 1))) 1262 sys_data->user_arg_size = -1; 1263 } 1264 1265 static int __init init_syscall_trace(struct trace_event_call *call) 1266 { 1267 int id; 1268 int num; 1269 1270 num = ((struct syscall_metadata *)call->data)->syscall_nr; 1271 if (num < 0 || num >= NR_syscalls) { 1272 pr_debug("syscall %s metadata not mapped, disabling ftrace event\n", 1273 ((struct syscall_metadata *)call->data)->name); 1274 return -ENOSYS; 1275 } 1276 1277 check_faultable_syscall(call, num); 1278 1279 if (set_syscall_print_fmt(call) < 0) 1280 return -ENOMEM; 1281 1282 id = trace_event_raw_init(call); 1283 1284 if (id < 0) { 1285 free_syscall_print_fmt(call); 1286 return id; 1287 } 1288 1289 return id; 1290 } 1291 1292 static struct trace_event_fields __refdata syscall_enter_fields_array[] = { 1293 SYSCALL_FIELD(int, __syscall_nr), 1294 { .type = TRACE_FUNCTION_TYPE, 1295 .define_fields = syscall_enter_define_fields }, 1296 {} 1297 }; 1298 1299 struct trace_event_functions enter_syscall_print_funcs = { 1300 .trace = print_syscall_enter, 1301 }; 1302 1303 struct trace_event_functions exit_syscall_print_funcs = { 1304 .trace = print_syscall_exit, 1305 }; 1306 1307 #if defined(CONFIG_BPF_EVENTS) && defined(CONFIG_DEBUG_INFO_BTF) 1308 /* BTF id lists for the shared sys_enter/sys_exit dispatcher tracepoints. */ 1309 BTF_ID_LIST(syscall_enter_btf_ids) 1310 BTF_ID(func, __bpf_trace_sys_enter) 1311 BTF_ID(struct, trace_event_raw_sys_enter) 1312 1313 BTF_ID_LIST(syscall_exit_btf_ids) 1314 BTF_ID(func, __bpf_trace_sys_exit) 1315 BTF_ID(struct, trace_event_raw_sys_exit) 1316 #endif 1317 1318 struct trace_event_class __refdata event_class_syscall_enter = { 1319 .system = "syscalls", 1320 .reg = syscall_enter_register, 1321 .fields_array = syscall_enter_fields_array, 1322 .get_fields = syscall_get_enter_fields, 1323 .raw_init = init_syscall_trace, 1324 #if defined(CONFIG_BPF_EVENTS) && defined(CONFIG_DEBUG_INFO_BTF) 1325 .btf_ids = syscall_enter_btf_ids, 1326 #endif 1327 }; 1328 1329 struct trace_event_class __refdata event_class_syscall_exit = { 1330 .system = "syscalls", 1331 .reg = syscall_exit_register, 1332 .fields_array = (struct trace_event_fields[]){ 1333 SYSCALL_FIELD(int, __syscall_nr), 1334 SYSCALL_FIELD(long, ret), 1335 {} 1336 }, 1337 .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields), 1338 .raw_init = init_syscall_trace, 1339 #if defined(CONFIG_BPF_EVENTS) && defined(CONFIG_DEBUG_INFO_BTF) 1340 .btf_ids = syscall_exit_btf_ids, 1341 #endif 1342 }; 1343 1344 unsigned long __init __weak arch_syscall_addr(int nr) 1345 { 1346 return (unsigned long)sys_call_table[nr]; 1347 } 1348 1349 void __init init_ftrace_syscalls(void) 1350 { 1351 struct syscall_metadata *meta; 1352 unsigned long addr; 1353 int i; 1354 void *ret; 1355 1356 if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) { 1357 syscalls_metadata = kzalloc_objs(*syscalls_metadata, 1358 NR_syscalls); 1359 if (!syscalls_metadata) { 1360 WARN_ON(1); 1361 return; 1362 } 1363 } 1364 1365 for (i = 0; i < NR_syscalls; i++) { 1366 addr = arch_syscall_addr(i); 1367 meta = find_syscall_meta(addr); 1368 if (!meta) 1369 continue; 1370 1371 meta->syscall_nr = i; 1372 1373 if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) { 1374 syscalls_metadata[i] = meta; 1375 } else { 1376 ret = xa_store(&syscalls_metadata_sparse, i, meta, 1377 GFP_KERNEL); 1378 WARN(xa_is_err(ret), 1379 "Syscall memory allocation failed\n"); 1380 } 1381 1382 } 1383 } 1384 1385 #ifdef CONFIG_PERF_EVENTS 1386 1387 static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls); 1388 static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); 1389 static int sys_perf_refcount_enter; 1390 static int sys_perf_refcount_exit; 1391 1392 static int perf_call_bpf_enter(struct trace_event_call *call, 1393 struct syscall_metadata *sys_data, 1394 int syscall_nr, unsigned long *args) 1395 { 1396 struct syscall_tp_t { 1397 struct trace_entry ent; 1398 int syscall_nr; 1399 unsigned long args[SYSCALL_DEFINE_MAXARGS]; 1400 } __aligned(8) param; 1401 struct pt_regs regs = {}; 1402 int i; 1403 1404 BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *)); 1405 1406 /* bpf prog requires 'regs' to be the first member in the ctx */ 1407 perf_fetch_caller_regs(®s); 1408 *(struct pt_regs **)¶m = ®s; 1409 param.syscall_nr = syscall_nr; 1410 for (i = 0; i < sys_data->nb_args; i++) 1411 param.args[i] = args[i]; 1412 return trace_call_bpf_faultable(call, ¶m); 1413 } 1414 1415 static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) 1416 { 1417 struct syscall_metadata *sys_data; 1418 struct syscall_trace_enter *rec; 1419 struct hlist_head *head; 1420 unsigned long args[6]; 1421 bool valid_prog_array; 1422 bool mayfault; 1423 char *user_ptr; 1424 int user_sizes[SYSCALL_FAULT_MAX_CNT] = {}; 1425 int buf_size = CONFIG_TRACE_SYSCALL_BUF_SIZE_DEFAULT; 1426 int syscall_nr; 1427 int rctx; 1428 int size = 0; 1429 int uargs = 0; 1430 1431 might_fault(); 1432 1433 syscall_nr = trace_get_syscall_nr(current, regs); 1434 if (syscall_nr < 0 || syscall_nr >= NR_syscalls) 1435 return; 1436 if (!test_bit(syscall_nr, enabled_perf_enter_syscalls)) 1437 return; 1438 1439 sys_data = syscall_nr_to_meta(syscall_nr); 1440 if (!sys_data) 1441 return; 1442 1443 syscall_get_arguments(current, regs, args); 1444 1445 /* 1446 * Run BPF program in faultable context before per-cpu buffer 1447 * allocation, allowing sleepable BPF programs to execute. 1448 */ 1449 valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); 1450 if (valid_prog_array && 1451 !perf_call_bpf_enter(sys_data->enter_event, sys_data, 1452 syscall_nr, args)) 1453 return; 1454 1455 /* 1456 * Per-cpu ring buffer and perf event list operations require 1457 * preemption to be disabled. 1458 */ 1459 guard(preempt_notrace)(); 1460 1461 head = this_cpu_ptr(sys_data->enter_event->perf_events); 1462 if (hlist_empty(head)) 1463 return; 1464 1465 /* Check if this syscall event faults in user space memory */ 1466 mayfault = sys_data->user_mask != 0; 1467 1468 if (mayfault) { 1469 if (syscall_get_data(sys_data, args, &user_ptr, 1470 &size, user_sizes, &uargs, buf_size) < 0) 1471 return; 1472 1473 /* The above may have caused a migration */ 1474 head = this_cpu_ptr(sys_data->enter_event->perf_events); 1475 if (hlist_empty(head)) 1476 return; 1477 } 1478 1479 /* get the size after alignment with the u32 buffer size field */ 1480 size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec); 1481 size = ALIGN(size + sizeof(u32), sizeof(u64)); 1482 size -= sizeof(u32); 1483 1484 rec = perf_trace_buf_alloc(size, NULL, &rctx); 1485 if (!rec) 1486 return; 1487 1488 rec->nr = syscall_nr; 1489 memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args); 1490 1491 if (mayfault) 1492 syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs); 1493 1494 perf_trace_buf_submit(rec, size, rctx, 1495 sys_data->enter_event->event.type, 1, regs, 1496 head, NULL); 1497 } 1498 1499 static int perf_sysenter_enable(struct trace_event_call *call) 1500 { 1501 struct syscall_metadata *sys_data = call->data; 1502 int num; 1503 int ret; 1504 1505 num = sys_data->syscall_nr; 1506 1507 guard(mutex)(&syscall_trace_lock); 1508 if (sys_data->user_mask) { 1509 ret = syscall_fault_buffer_enable(); 1510 if (ret < 0) 1511 return ret; 1512 } 1513 if (!sys_perf_refcount_enter) { 1514 ret = register_trace_sys_enter(perf_syscall_enter, NULL); 1515 if (ret) { 1516 pr_info("event trace: Could not activate syscall entry trace point"); 1517 if (sys_data->user_mask) 1518 syscall_fault_buffer_disable(); 1519 return ret; 1520 } 1521 } 1522 set_bit(num, enabled_perf_enter_syscalls); 1523 sys_perf_refcount_enter++; 1524 return 0; 1525 } 1526 1527 static void perf_sysenter_disable(struct trace_event_call *call) 1528 { 1529 struct syscall_metadata *sys_data = call->data; 1530 int num; 1531 1532 num = sys_data->syscall_nr; 1533 1534 guard(mutex)(&syscall_trace_lock); 1535 sys_perf_refcount_enter--; 1536 clear_bit(num, enabled_perf_enter_syscalls); 1537 if (!sys_perf_refcount_enter) 1538 unregister_trace_sys_enter(perf_syscall_enter, NULL); 1539 if (sys_data->user_mask) 1540 syscall_fault_buffer_disable(); 1541 } 1542 1543 static int perf_call_bpf_exit(struct trace_event_call *call, 1544 int syscall_nr, long ret_val) 1545 { 1546 struct syscall_tp_t { 1547 struct trace_entry ent; 1548 int syscall_nr; 1549 unsigned long ret; 1550 } __aligned(8) param; 1551 struct pt_regs regs = {}; 1552 1553 /* bpf prog requires 'regs' to be the first member in the ctx */ 1554 perf_fetch_caller_regs(®s); 1555 *(struct pt_regs **)¶m = ®s; 1556 param.syscall_nr = syscall_nr; 1557 param.ret = ret_val; 1558 return trace_call_bpf_faultable(call, ¶m); 1559 } 1560 1561 static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) 1562 { 1563 struct syscall_metadata *sys_data; 1564 struct syscall_trace_exit *rec; 1565 struct hlist_head *head; 1566 bool valid_prog_array; 1567 int syscall_nr; 1568 int rctx; 1569 int size; 1570 1571 might_fault(); 1572 1573 syscall_nr = trace_get_syscall_nr(current, regs); 1574 if (syscall_nr < 0 || syscall_nr >= NR_syscalls) 1575 return; 1576 if (!test_bit(syscall_nr, enabled_perf_exit_syscalls)) 1577 return; 1578 1579 sys_data = syscall_nr_to_meta(syscall_nr); 1580 if (!sys_data) 1581 return; 1582 1583 /* 1584 * Run BPF program in faultable context before per-cpu buffer 1585 * allocation, allowing sleepable BPF programs to execute. 1586 */ 1587 valid_prog_array = bpf_prog_array_valid(sys_data->exit_event); 1588 if (valid_prog_array && 1589 !perf_call_bpf_exit(sys_data->exit_event, syscall_nr, 1590 syscall_get_return_value(current, regs))) 1591 return; 1592 1593 /* 1594 * Per-cpu ring buffer and perf event list operations require 1595 * preemption to be disabled. 1596 */ 1597 guard(preempt_notrace)(); 1598 1599 head = this_cpu_ptr(sys_data->exit_event->perf_events); 1600 if (hlist_empty(head)) 1601 return; 1602 1603 /* We can probably do that at build time */ 1604 size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64)); 1605 size -= sizeof(u32); 1606 1607 rec = perf_trace_buf_alloc(size, NULL, &rctx); 1608 if (!rec) 1609 return; 1610 1611 rec->nr = syscall_nr; 1612 rec->ret = syscall_get_return_value(current, regs); 1613 1614 perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type, 1615 1, regs, head, NULL); 1616 } 1617 1618 static int perf_sysexit_enable(struct trace_event_call *call) 1619 { 1620 int num; 1621 1622 num = ((struct syscall_metadata *)call->data)->syscall_nr; 1623 1624 guard(mutex)(&syscall_trace_lock); 1625 if (!sys_perf_refcount_exit) { 1626 int ret = register_trace_sys_exit(perf_syscall_exit, NULL); 1627 if (ret) { 1628 pr_info("event trace: Could not activate syscall exit trace point"); 1629 return ret; 1630 } 1631 } 1632 set_bit(num, enabled_perf_exit_syscalls); 1633 sys_perf_refcount_exit++; 1634 return 0; 1635 } 1636 1637 static void perf_sysexit_disable(struct trace_event_call *call) 1638 { 1639 int num; 1640 1641 num = ((struct syscall_metadata *)call->data)->syscall_nr; 1642 1643 guard(mutex)(&syscall_trace_lock); 1644 sys_perf_refcount_exit--; 1645 clear_bit(num, enabled_perf_exit_syscalls); 1646 if (!sys_perf_refcount_exit) 1647 unregister_trace_sys_exit(perf_syscall_exit, NULL); 1648 } 1649 1650 #endif /* CONFIG_PERF_EVENTS */ 1651 1652 static int syscall_enter_register(struct trace_event_call *event, 1653 enum trace_reg type, void *data) 1654 { 1655 struct trace_event_file *file = data; 1656 1657 switch (type) { 1658 case TRACE_REG_REGISTER: 1659 return reg_event_syscall_enter(file, event); 1660 case TRACE_REG_UNREGISTER: 1661 unreg_event_syscall_enter(file, event); 1662 return 0; 1663 1664 #ifdef CONFIG_PERF_EVENTS 1665 case TRACE_REG_PERF_REGISTER: 1666 return perf_sysenter_enable(event); 1667 case TRACE_REG_PERF_UNREGISTER: 1668 perf_sysenter_disable(event); 1669 return 0; 1670 case TRACE_REG_PERF_OPEN: 1671 case TRACE_REG_PERF_CLOSE: 1672 case TRACE_REG_PERF_ADD: 1673 case TRACE_REG_PERF_DEL: 1674 return 0; 1675 #endif 1676 } 1677 return 0; 1678 } 1679 1680 static int syscall_exit_register(struct trace_event_call *event, 1681 enum trace_reg type, void *data) 1682 { 1683 struct trace_event_file *file = data; 1684 1685 switch (type) { 1686 case TRACE_REG_REGISTER: 1687 return reg_event_syscall_exit(file, event); 1688 case TRACE_REG_UNREGISTER: 1689 unreg_event_syscall_exit(file, event); 1690 return 0; 1691 1692 #ifdef CONFIG_PERF_EVENTS 1693 case TRACE_REG_PERF_REGISTER: 1694 return perf_sysexit_enable(event); 1695 case TRACE_REG_PERF_UNREGISTER: 1696 perf_sysexit_disable(event); 1697 return 0; 1698 case TRACE_REG_PERF_OPEN: 1699 case TRACE_REG_PERF_CLOSE: 1700 case TRACE_REG_PERF_ADD: 1701 case TRACE_REG_PERF_DEL: 1702 return 0; 1703 #endif 1704 } 1705 return 0; 1706 } 1707