1 /* 2 * Kprobes-based tracing events 3 * 4 * Created by Masami Hiramatsu <mhiramat@redhat.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 20 #include <linux/module.h> 21 #include <linux/uaccess.h> 22 #include <linux/kprobes.h> 23 #include <linux/seq_file.h> 24 #include <linux/slab.h> 25 #include <linux/smp.h> 26 #include <linux/debugfs.h> 27 #include <linux/types.h> 28 #include <linux/string.h> 29 #include <linux/ctype.h> 30 #include <linux/ptrace.h> 31 #include <linux/perf_event.h> 32 #include <linux/stringify.h> 33 #include <asm/bitsperlong.h> 34 35 #include "trace.h" 36 #include "trace_output.h" 37 38 #define MAX_TRACE_ARGS 128 39 #define MAX_ARGSTR_LEN 63 40 #define MAX_EVENT_NAME_LEN 64 41 #define KPROBE_EVENT_SYSTEM "kprobes" 42 43 /* Reserved field names */ 44 #define FIELD_STRING_IP "__probe_ip" 45 #define FIELD_STRING_RETIP "__probe_ret_ip" 46 #define FIELD_STRING_FUNC "__probe_func" 47 48 const char *reserved_field_names[] = { 49 "common_type", 50 "common_flags", 51 "common_preempt_count", 52 "common_pid", 53 "common_tgid", 54 "common_lock_depth", 55 FIELD_STRING_IP, 56 FIELD_STRING_RETIP, 57 FIELD_STRING_FUNC, 58 }; 59 60 /* Printing function type */ 61 typedef int (*print_type_func_t)(struct trace_seq *, const char *, void *); 62 #define PRINT_TYPE_FUNC_NAME(type) print_type_##type 63 #define PRINT_TYPE_FMT_NAME(type) print_type_format_##type 64 65 /* Printing in basic type function template */ 66 #define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \ 67 static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \ 68 const char *name, void *data)\ 69 { \ 70 return trace_seq_printf(s, " %s=" fmt, name, (cast)*(type *)data);\ 71 } \ 72 static const char PRINT_TYPE_FMT_NAME(type)[] = fmt; 73 74 DEFINE_BASIC_PRINT_TYPE_FUNC(u8, "%x", unsigned int) 75 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "%x", unsigned int) 76 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, "%lx", unsigned long) 77 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, "%llx", unsigned long long) 78 DEFINE_BASIC_PRINT_TYPE_FUNC(s8, "%d", int) 79 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, "%d", int) 80 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%ld", long) 81 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%lld", long long) 82 83 /* Data fetch function type */ 84 typedef void (*fetch_func_t)(struct pt_regs *, void *, void *); 85 86 struct fetch_param { 87 fetch_func_t fn; 88 void *data; 89 }; 90 91 static __kprobes void call_fetch(struct fetch_param *fprm, 92 struct pt_regs *regs, void *dest) 93 { 94 return fprm->fn(regs, fprm->data, dest); 95 } 96 97 #define FETCH_FUNC_NAME(kind, type) fetch_##kind##_##type 98 /* 99 * Define macro for basic types - we don't need to define s* types, because 100 * we have to care only about bitwidth at recording time. 101 */ 102 #define DEFINE_BASIC_FETCH_FUNCS(kind) \ 103 DEFINE_FETCH_##kind(u8) \ 104 DEFINE_FETCH_##kind(u16) \ 105 DEFINE_FETCH_##kind(u32) \ 106 DEFINE_FETCH_##kind(u64) 107 108 #define CHECK_BASIC_FETCH_FUNCS(kind, fn) \ 109 ((FETCH_FUNC_NAME(kind, u8) == fn) || \ 110 (FETCH_FUNC_NAME(kind, u16) == fn) || \ 111 (FETCH_FUNC_NAME(kind, u32) == fn) || \ 112 (FETCH_FUNC_NAME(kind, u64) == fn)) 113 114 /* Data fetch function templates */ 115 #define DEFINE_FETCH_reg(type) \ 116 static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \ 117 void *offset, void *dest) \ 118 { \ 119 *(type *)dest = (type)regs_get_register(regs, \ 120 (unsigned int)((unsigned long)offset)); \ 121 } 122 DEFINE_BASIC_FETCH_FUNCS(reg) 123 124 #define DEFINE_FETCH_stack(type) \ 125 static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\ 126 void *offset, void *dest) \ 127 { \ 128 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \ 129 (unsigned int)((unsigned long)offset)); \ 130 } 131 DEFINE_BASIC_FETCH_FUNCS(stack) 132 133 #define DEFINE_FETCH_retval(type) \ 134 static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\ 135 void *dummy, void *dest) \ 136 { \ 137 *(type *)dest = (type)regs_return_value(regs); \ 138 } 139 DEFINE_BASIC_FETCH_FUNCS(retval) 140 141 #define DEFINE_FETCH_memory(type) \ 142 static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\ 143 void *addr, void *dest) \ 144 { \ 145 type retval; \ 146 if (probe_kernel_address(addr, retval)) \ 147 *(type *)dest = 0; \ 148 else \ 149 *(type *)dest = retval; \ 150 } 151 DEFINE_BASIC_FETCH_FUNCS(memory) 152 153 /* Memory fetching by symbol */ 154 struct symbol_cache { 155 char *symbol; 156 long offset; 157 unsigned long addr; 158 }; 159 160 static unsigned long update_symbol_cache(struct symbol_cache *sc) 161 { 162 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol); 163 if (sc->addr) 164 sc->addr += sc->offset; 165 return sc->addr; 166 } 167 168 static void free_symbol_cache(struct symbol_cache *sc) 169 { 170 kfree(sc->symbol); 171 kfree(sc); 172 } 173 174 static struct symbol_cache *alloc_symbol_cache(const char *sym, long offset) 175 { 176 struct symbol_cache *sc; 177 178 if (!sym || strlen(sym) == 0) 179 return NULL; 180 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL); 181 if (!sc) 182 return NULL; 183 184 sc->symbol = kstrdup(sym, GFP_KERNEL); 185 if (!sc->symbol) { 186 kfree(sc); 187 return NULL; 188 } 189 sc->offset = offset; 190 191 update_symbol_cache(sc); 192 return sc; 193 } 194 195 #define DEFINE_FETCH_symbol(type) \ 196 static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\ 197 void *data, void *dest) \ 198 { \ 199 struct symbol_cache *sc = data; \ 200 if (sc->addr) \ 201 fetch_memory_##type(regs, (void *)sc->addr, dest); \ 202 else \ 203 *(type *)dest = 0; \ 204 } 205 DEFINE_BASIC_FETCH_FUNCS(symbol) 206 207 /* Dereference memory access function */ 208 struct deref_fetch_param { 209 struct fetch_param orig; 210 long offset; 211 }; 212 213 #define DEFINE_FETCH_deref(type) \ 214 static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\ 215 void *data, void *dest) \ 216 { \ 217 struct deref_fetch_param *dprm = data; \ 218 unsigned long addr; \ 219 call_fetch(&dprm->orig, regs, &addr); \ 220 if (addr) { \ 221 addr += dprm->offset; \ 222 fetch_memory_##type(regs, (void *)addr, dest); \ 223 } else \ 224 *(type *)dest = 0; \ 225 } 226 DEFINE_BASIC_FETCH_FUNCS(deref) 227 228 static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data) 229 { 230 if (CHECK_BASIC_FETCH_FUNCS(deref, data->orig.fn)) 231 free_deref_fetch_param(data->orig.data); 232 else if (CHECK_BASIC_FETCH_FUNCS(symbol, data->orig.fn)) 233 free_symbol_cache(data->orig.data); 234 kfree(data); 235 } 236 237 /* Default (unsigned long) fetch type */ 238 #define __DEFAULT_FETCH_TYPE(t) u##t 239 #define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t) 240 #define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG) 241 #define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE) 242 243 #define ASSIGN_FETCH_FUNC(kind, type) \ 244 .kind = FETCH_FUNC_NAME(kind, type) 245 246 #define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \ 247 {.name = #ptype, \ 248 .size = sizeof(ftype), \ 249 .is_signed = sign, \ 250 .print = PRINT_TYPE_FUNC_NAME(ptype), \ 251 .fmt = PRINT_TYPE_FMT_NAME(ptype), \ 252 ASSIGN_FETCH_FUNC(reg, ftype), \ 253 ASSIGN_FETCH_FUNC(stack, ftype), \ 254 ASSIGN_FETCH_FUNC(retval, ftype), \ 255 ASSIGN_FETCH_FUNC(memory, ftype), \ 256 ASSIGN_FETCH_FUNC(symbol, ftype), \ 257 ASSIGN_FETCH_FUNC(deref, ftype), \ 258 } 259 260 /* Fetch type information table */ 261 static const struct fetch_type { 262 const char *name; /* Name of type */ 263 size_t size; /* Byte size of type */ 264 int is_signed; /* Signed flag */ 265 print_type_func_t print; /* Print functions */ 266 const char *fmt; /* Fromat string */ 267 /* Fetch functions */ 268 fetch_func_t reg; 269 fetch_func_t stack; 270 fetch_func_t retval; 271 fetch_func_t memory; 272 fetch_func_t symbol; 273 fetch_func_t deref; 274 } fetch_type_table[] = { 275 ASSIGN_FETCH_TYPE(u8, u8, 0), 276 ASSIGN_FETCH_TYPE(u16, u16, 0), 277 ASSIGN_FETCH_TYPE(u32, u32, 0), 278 ASSIGN_FETCH_TYPE(u64, u64, 0), 279 ASSIGN_FETCH_TYPE(s8, u8, 1), 280 ASSIGN_FETCH_TYPE(s16, u16, 1), 281 ASSIGN_FETCH_TYPE(s32, u32, 1), 282 ASSIGN_FETCH_TYPE(s64, u64, 1), 283 }; 284 285 static const struct fetch_type *find_fetch_type(const char *type) 286 { 287 int i; 288 289 if (!type) 290 type = DEFAULT_FETCH_TYPE_STR; 291 292 for (i = 0; i < ARRAY_SIZE(fetch_type_table); i++) 293 if (strcmp(type, fetch_type_table[i].name) == 0) 294 return &fetch_type_table[i]; 295 return NULL; 296 } 297 298 /* Special function : only accept unsigned long */ 299 static __kprobes void fetch_stack_address(struct pt_regs *regs, 300 void *dummy, void *dest) 301 { 302 *(unsigned long *)dest = kernel_stack_pointer(regs); 303 } 304 305 /** 306 * Kprobe event core functions 307 */ 308 309 struct probe_arg { 310 struct fetch_param fetch; 311 unsigned int offset; /* Offset from argument entry */ 312 const char *name; /* Name of this argument */ 313 const char *comm; /* Command of this argument */ 314 const struct fetch_type *type; /* Type of this argument */ 315 }; 316 317 /* Flags for trace_probe */ 318 #define TP_FLAG_TRACE 1 319 #define TP_FLAG_PROFILE 2 320 321 struct trace_probe { 322 struct list_head list; 323 struct kretprobe rp; /* Use rp.kp for kprobe use */ 324 unsigned long nhit; 325 unsigned int flags; /* For TP_FLAG_* */ 326 const char *symbol; /* symbol name */ 327 struct ftrace_event_call call; 328 struct trace_event event; 329 ssize_t size; /* trace entry size */ 330 unsigned int nr_args; 331 struct probe_arg args[]; 332 }; 333 334 #define SIZEOF_TRACE_PROBE(n) \ 335 (offsetof(struct trace_probe, args) + \ 336 (sizeof(struct probe_arg) * (n))) 337 338 339 static __kprobes int probe_is_return(struct trace_probe *tp) 340 { 341 return tp->rp.handler != NULL; 342 } 343 344 static __kprobes const char *probe_symbol(struct trace_probe *tp) 345 { 346 return tp->symbol ? tp->symbol : "unknown"; 347 } 348 349 static int register_probe_event(struct trace_probe *tp); 350 static void unregister_probe_event(struct trace_probe *tp); 351 352 static DEFINE_MUTEX(probe_lock); 353 static LIST_HEAD(probe_list); 354 355 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 356 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 357 struct pt_regs *regs); 358 359 /* Check the name is good for event/group */ 360 static int check_event_name(const char *name) 361 { 362 if (!isalpha(*name) && *name != '_') 363 return 0; 364 while (*++name != '\0') { 365 if (!isalpha(*name) && !isdigit(*name) && *name != '_') 366 return 0; 367 } 368 return 1; 369 } 370 371 /* 372 * Allocate new trace_probe and initialize it (including kprobes). 373 */ 374 static struct trace_probe *alloc_trace_probe(const char *group, 375 const char *event, 376 void *addr, 377 const char *symbol, 378 unsigned long offs, 379 int nargs, int is_return) 380 { 381 struct trace_probe *tp; 382 int ret = -ENOMEM; 383 384 tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL); 385 if (!tp) 386 return ERR_PTR(ret); 387 388 if (symbol) { 389 tp->symbol = kstrdup(symbol, GFP_KERNEL); 390 if (!tp->symbol) 391 goto error; 392 tp->rp.kp.symbol_name = tp->symbol; 393 tp->rp.kp.offset = offs; 394 } else 395 tp->rp.kp.addr = addr; 396 397 if (is_return) 398 tp->rp.handler = kretprobe_dispatcher; 399 else 400 tp->rp.kp.pre_handler = kprobe_dispatcher; 401 402 if (!event || !check_event_name(event)) { 403 ret = -EINVAL; 404 goto error; 405 } 406 407 tp->call.name = kstrdup(event, GFP_KERNEL); 408 if (!tp->call.name) 409 goto error; 410 411 if (!group || !check_event_name(group)) { 412 ret = -EINVAL; 413 goto error; 414 } 415 416 tp->call.system = kstrdup(group, GFP_KERNEL); 417 if (!tp->call.system) 418 goto error; 419 420 INIT_LIST_HEAD(&tp->list); 421 return tp; 422 error: 423 kfree(tp->call.name); 424 kfree(tp->symbol); 425 kfree(tp); 426 return ERR_PTR(ret); 427 } 428 429 static void free_probe_arg(struct probe_arg *arg) 430 { 431 if (CHECK_BASIC_FETCH_FUNCS(deref, arg->fetch.fn)) 432 free_deref_fetch_param(arg->fetch.data); 433 else if (CHECK_BASIC_FETCH_FUNCS(symbol, arg->fetch.fn)) 434 free_symbol_cache(arg->fetch.data); 435 kfree(arg->name); 436 kfree(arg->comm); 437 } 438 439 static void free_trace_probe(struct trace_probe *tp) 440 { 441 int i; 442 443 for (i = 0; i < tp->nr_args; i++) 444 free_probe_arg(&tp->args[i]); 445 446 kfree(tp->call.system); 447 kfree(tp->call.name); 448 kfree(tp->symbol); 449 kfree(tp); 450 } 451 452 static struct trace_probe *find_probe_event(const char *event, 453 const char *group) 454 { 455 struct trace_probe *tp; 456 457 list_for_each_entry(tp, &probe_list, list) 458 if (strcmp(tp->call.name, event) == 0 && 459 strcmp(tp->call.system, group) == 0) 460 return tp; 461 return NULL; 462 } 463 464 /* Unregister a trace_probe and probe_event: call with locking probe_lock */ 465 static void unregister_trace_probe(struct trace_probe *tp) 466 { 467 if (probe_is_return(tp)) 468 unregister_kretprobe(&tp->rp); 469 else 470 unregister_kprobe(&tp->rp.kp); 471 list_del(&tp->list); 472 unregister_probe_event(tp); 473 } 474 475 /* Register a trace_probe and probe_event */ 476 static int register_trace_probe(struct trace_probe *tp) 477 { 478 struct trace_probe *old_tp; 479 int ret; 480 481 mutex_lock(&probe_lock); 482 483 /* register as an event */ 484 old_tp = find_probe_event(tp->call.name, tp->call.system); 485 if (old_tp) { 486 /* delete old event */ 487 unregister_trace_probe(old_tp); 488 free_trace_probe(old_tp); 489 } 490 ret = register_probe_event(tp); 491 if (ret) { 492 pr_warning("Faild to register probe event(%d)\n", ret); 493 goto end; 494 } 495 496 tp->rp.kp.flags |= KPROBE_FLAG_DISABLED; 497 if (probe_is_return(tp)) 498 ret = register_kretprobe(&tp->rp); 499 else 500 ret = register_kprobe(&tp->rp.kp); 501 502 if (ret) { 503 pr_warning("Could not insert probe(%d)\n", ret); 504 if (ret == -EILSEQ) { 505 pr_warning("Probing address(0x%p) is not an " 506 "instruction boundary.\n", 507 tp->rp.kp.addr); 508 ret = -EINVAL; 509 } 510 unregister_probe_event(tp); 511 } else 512 list_add_tail(&tp->list, &probe_list); 513 end: 514 mutex_unlock(&probe_lock); 515 return ret; 516 } 517 518 /* Split symbol and offset. */ 519 static int split_symbol_offset(char *symbol, unsigned long *offset) 520 { 521 char *tmp; 522 int ret; 523 524 if (!offset) 525 return -EINVAL; 526 527 tmp = strchr(symbol, '+'); 528 if (tmp) { 529 /* skip sign because strict_strtol doesn't accept '+' */ 530 ret = strict_strtoul(tmp + 1, 0, offset); 531 if (ret) 532 return ret; 533 *tmp = '\0'; 534 } else 535 *offset = 0; 536 return 0; 537 } 538 539 #define PARAM_MAX_ARGS 16 540 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 541 542 static int parse_probe_vars(char *arg, const struct fetch_type *t, 543 struct fetch_param *f, int is_return) 544 { 545 int ret = 0; 546 unsigned long param; 547 548 if (strcmp(arg, "retval") == 0) { 549 if (is_return) 550 f->fn = t->retval; 551 else 552 ret = -EINVAL; 553 } else if (strncmp(arg, "stack", 5) == 0) { 554 if (arg[5] == '\0') { 555 if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR) == 0) 556 f->fn = fetch_stack_address; 557 else 558 ret = -EINVAL; 559 } else if (isdigit(arg[5])) { 560 ret = strict_strtoul(arg + 5, 10, ¶m); 561 if (ret || param > PARAM_MAX_STACK) 562 ret = -EINVAL; 563 else { 564 f->fn = t->stack; 565 f->data = (void *)param; 566 } 567 } else 568 ret = -EINVAL; 569 } else 570 ret = -EINVAL; 571 return ret; 572 } 573 574 /* Recursive argument parser */ 575 static int __parse_probe_arg(char *arg, const struct fetch_type *t, 576 struct fetch_param *f, int is_return) 577 { 578 int ret = 0; 579 unsigned long param; 580 long offset; 581 char *tmp; 582 583 switch (arg[0]) { 584 case '$': 585 ret = parse_probe_vars(arg + 1, t, f, is_return); 586 break; 587 case '%': /* named register */ 588 ret = regs_query_register_offset(arg + 1); 589 if (ret >= 0) { 590 f->fn = t->reg; 591 f->data = (void *)(unsigned long)ret; 592 ret = 0; 593 } 594 break; 595 case '@': /* memory or symbol */ 596 if (isdigit(arg[1])) { 597 ret = strict_strtoul(arg + 1, 0, ¶m); 598 if (ret) 599 break; 600 f->fn = t->memory; 601 f->data = (void *)param; 602 } else { 603 ret = split_symbol_offset(arg + 1, &offset); 604 if (ret) 605 break; 606 f->data = alloc_symbol_cache(arg + 1, offset); 607 if (f->data) 608 f->fn = t->symbol; 609 } 610 break; 611 case '+': /* deref memory */ 612 case '-': 613 tmp = strchr(arg, '('); 614 if (!tmp) 615 break; 616 *tmp = '\0'; 617 ret = strict_strtol(arg + 1, 0, &offset); 618 if (ret) 619 break; 620 if (arg[0] == '-') 621 offset = -offset; 622 arg = tmp + 1; 623 tmp = strrchr(arg, ')'); 624 if (tmp) { 625 struct deref_fetch_param *dprm; 626 const struct fetch_type *t2 = find_fetch_type(NULL); 627 *tmp = '\0'; 628 dprm = kzalloc(sizeof(struct deref_fetch_param), 629 GFP_KERNEL); 630 if (!dprm) 631 return -ENOMEM; 632 dprm->offset = offset; 633 ret = __parse_probe_arg(arg, t2, &dprm->orig, 634 is_return); 635 if (ret) 636 kfree(dprm); 637 else { 638 f->fn = t->deref; 639 f->data = (void *)dprm; 640 } 641 } 642 break; 643 } 644 if (!ret && !f->fn) 645 ret = -EINVAL; 646 return ret; 647 } 648 649 /* String length checking wrapper */ 650 static int parse_probe_arg(char *arg, struct trace_probe *tp, 651 struct probe_arg *parg, int is_return) 652 { 653 const char *t; 654 655 if (strlen(arg) > MAX_ARGSTR_LEN) { 656 pr_info("Argument is too long.: %s\n", arg); 657 return -ENOSPC; 658 } 659 parg->comm = kstrdup(arg, GFP_KERNEL); 660 if (!parg->comm) { 661 pr_info("Failed to allocate memory for command '%s'.\n", arg); 662 return -ENOMEM; 663 } 664 t = strchr(parg->comm, ':'); 665 if (t) { 666 arg[t - parg->comm] = '\0'; 667 t++; 668 } 669 parg->type = find_fetch_type(t); 670 if (!parg->type) { 671 pr_info("Unsupported type: %s\n", t); 672 return -EINVAL; 673 } 674 parg->offset = tp->size; 675 tp->size += parg->type->size; 676 return __parse_probe_arg(arg, parg->type, &parg->fetch, is_return); 677 } 678 679 /* Return 1 if name is reserved or already used by another argument */ 680 static int conflict_field_name(const char *name, 681 struct probe_arg *args, int narg) 682 { 683 int i; 684 for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++) 685 if (strcmp(reserved_field_names[i], name) == 0) 686 return 1; 687 for (i = 0; i < narg; i++) 688 if (strcmp(args[i].name, name) == 0) 689 return 1; 690 return 0; 691 } 692 693 static int create_trace_probe(int argc, char **argv) 694 { 695 /* 696 * Argument syntax: 697 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS] 698 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS] 699 * Fetch args: 700 * $retval : fetch return value 701 * $stack : fetch stack address 702 * $stackN : fetch Nth of stack (N:0-) 703 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 704 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 705 * %REG : fetch register REG 706 * Dereferencing memory fetch: 707 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 708 * Alias name of args: 709 * NAME=FETCHARG : set NAME as alias of FETCHARG. 710 * Type of args: 711 * FETCHARG:TYPE : use TYPE instead of unsigned long. 712 */ 713 struct trace_probe *tp; 714 int i, ret = 0; 715 int is_return = 0, is_delete = 0; 716 char *symbol = NULL, *event = NULL, *group = NULL; 717 char *arg, *tmp; 718 unsigned long offset = 0; 719 void *addr = NULL; 720 char buf[MAX_EVENT_NAME_LEN]; 721 722 /* argc must be >= 1 */ 723 if (argv[0][0] == 'p') 724 is_return = 0; 725 else if (argv[0][0] == 'r') 726 is_return = 1; 727 else if (argv[0][0] == '-') 728 is_delete = 1; 729 else { 730 pr_info("Probe definition must be started with 'p', 'r' or" 731 " '-'.\n"); 732 return -EINVAL; 733 } 734 735 if (argv[0][1] == ':') { 736 event = &argv[0][2]; 737 if (strchr(event, '/')) { 738 group = event; 739 event = strchr(group, '/') + 1; 740 event[-1] = '\0'; 741 if (strlen(group) == 0) { 742 pr_info("Group name is not specified\n"); 743 return -EINVAL; 744 } 745 } 746 if (strlen(event) == 0) { 747 pr_info("Event name is not specified\n"); 748 return -EINVAL; 749 } 750 } 751 if (!group) 752 group = KPROBE_EVENT_SYSTEM; 753 754 if (is_delete) { 755 if (!event) { 756 pr_info("Delete command needs an event name.\n"); 757 return -EINVAL; 758 } 759 tp = find_probe_event(event, group); 760 if (!tp) { 761 pr_info("Event %s/%s doesn't exist.\n", group, event); 762 return -ENOENT; 763 } 764 /* delete an event */ 765 unregister_trace_probe(tp); 766 free_trace_probe(tp); 767 return 0; 768 } 769 770 if (argc < 2) { 771 pr_info("Probe point is not specified.\n"); 772 return -EINVAL; 773 } 774 if (isdigit(argv[1][0])) { 775 if (is_return) { 776 pr_info("Return probe point must be a symbol.\n"); 777 return -EINVAL; 778 } 779 /* an address specified */ 780 ret = strict_strtoul(&argv[1][0], 0, (unsigned long *)&addr); 781 if (ret) { 782 pr_info("Failed to parse address.\n"); 783 return ret; 784 } 785 } else { 786 /* a symbol specified */ 787 symbol = argv[1]; 788 /* TODO: support .init module functions */ 789 ret = split_symbol_offset(symbol, &offset); 790 if (ret) { 791 pr_info("Failed to parse symbol.\n"); 792 return ret; 793 } 794 if (offset && is_return) { 795 pr_info("Return probe must be used without offset.\n"); 796 return -EINVAL; 797 } 798 } 799 argc -= 2; argv += 2; 800 801 /* setup a probe */ 802 if (!event) { 803 /* Make a new event name */ 804 if (symbol) 805 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 806 is_return ? 'r' : 'p', symbol, offset); 807 else 808 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p", 809 is_return ? 'r' : 'p', addr); 810 event = buf; 811 } 812 tp = alloc_trace_probe(group, event, addr, symbol, offset, argc, 813 is_return); 814 if (IS_ERR(tp)) { 815 pr_info("Failed to allocate trace_probe.(%d)\n", 816 (int)PTR_ERR(tp)); 817 return PTR_ERR(tp); 818 } 819 820 /* parse arguments */ 821 ret = 0; 822 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 823 /* Parse argument name */ 824 arg = strchr(argv[i], '='); 825 if (arg) 826 *arg++ = '\0'; 827 else 828 arg = argv[i]; 829 830 tp->args[i].name = kstrdup(argv[i], GFP_KERNEL); 831 if (!tp->args[i].name) { 832 pr_info("Failed to allocate argument%d name '%s'.\n", 833 i, argv[i]); 834 ret = -ENOMEM; 835 goto error; 836 } 837 tmp = strchr(tp->args[i].name, ':'); 838 if (tmp) 839 *tmp = '_'; /* convert : to _ */ 840 841 if (conflict_field_name(tp->args[i].name, tp->args, i)) { 842 pr_info("Argument%d name '%s' conflicts with " 843 "another field.\n", i, argv[i]); 844 ret = -EINVAL; 845 goto error; 846 } 847 848 /* Parse fetch argument */ 849 ret = parse_probe_arg(arg, tp, &tp->args[i], is_return); 850 if (ret) { 851 pr_info("Parse error at argument%d. (%d)\n", i, ret); 852 kfree(tp->args[i].name); 853 goto error; 854 } 855 856 tp->nr_args++; 857 } 858 859 ret = register_trace_probe(tp); 860 if (ret) 861 goto error; 862 return 0; 863 864 error: 865 free_trace_probe(tp); 866 return ret; 867 } 868 869 static void cleanup_all_probes(void) 870 { 871 struct trace_probe *tp; 872 873 mutex_lock(&probe_lock); 874 /* TODO: Use batch unregistration */ 875 while (!list_empty(&probe_list)) { 876 tp = list_entry(probe_list.next, struct trace_probe, list); 877 unregister_trace_probe(tp); 878 free_trace_probe(tp); 879 } 880 mutex_unlock(&probe_lock); 881 } 882 883 884 /* Probes listing interfaces */ 885 static void *probes_seq_start(struct seq_file *m, loff_t *pos) 886 { 887 mutex_lock(&probe_lock); 888 return seq_list_start(&probe_list, *pos); 889 } 890 891 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) 892 { 893 return seq_list_next(v, &probe_list, pos); 894 } 895 896 static void probes_seq_stop(struct seq_file *m, void *v) 897 { 898 mutex_unlock(&probe_lock); 899 } 900 901 static int probes_seq_show(struct seq_file *m, void *v) 902 { 903 struct trace_probe *tp = v; 904 int i; 905 906 seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p'); 907 seq_printf(m, ":%s/%s", tp->call.system, tp->call.name); 908 909 if (!tp->symbol) 910 seq_printf(m, " 0x%p", tp->rp.kp.addr); 911 else if (tp->rp.kp.offset) 912 seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset); 913 else 914 seq_printf(m, " %s", probe_symbol(tp)); 915 916 for (i = 0; i < tp->nr_args; i++) 917 seq_printf(m, " %s=%s", tp->args[i].name, tp->args[i].comm); 918 seq_printf(m, "\n"); 919 920 return 0; 921 } 922 923 static const struct seq_operations probes_seq_op = { 924 .start = probes_seq_start, 925 .next = probes_seq_next, 926 .stop = probes_seq_stop, 927 .show = probes_seq_show 928 }; 929 930 static int probes_open(struct inode *inode, struct file *file) 931 { 932 if ((file->f_mode & FMODE_WRITE) && 933 (file->f_flags & O_TRUNC)) 934 cleanup_all_probes(); 935 936 return seq_open(file, &probes_seq_op); 937 } 938 939 static int command_trace_probe(const char *buf) 940 { 941 char **argv; 942 int argc = 0, ret = 0; 943 944 argv = argv_split(GFP_KERNEL, buf, &argc); 945 if (!argv) 946 return -ENOMEM; 947 948 if (argc) 949 ret = create_trace_probe(argc, argv); 950 951 argv_free(argv); 952 return ret; 953 } 954 955 #define WRITE_BUFSIZE 128 956 957 static ssize_t probes_write(struct file *file, const char __user *buffer, 958 size_t count, loff_t *ppos) 959 { 960 char *kbuf, *tmp; 961 int ret; 962 size_t done; 963 size_t size; 964 965 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); 966 if (!kbuf) 967 return -ENOMEM; 968 969 ret = done = 0; 970 while (done < count) { 971 size = count - done; 972 if (size >= WRITE_BUFSIZE) 973 size = WRITE_BUFSIZE - 1; 974 if (copy_from_user(kbuf, buffer + done, size)) { 975 ret = -EFAULT; 976 goto out; 977 } 978 kbuf[size] = '\0'; 979 tmp = strchr(kbuf, '\n'); 980 if (tmp) { 981 *tmp = '\0'; 982 size = tmp - kbuf + 1; 983 } else if (done + size < count) { 984 pr_warning("Line length is too long: " 985 "Should be less than %d.", WRITE_BUFSIZE); 986 ret = -EINVAL; 987 goto out; 988 } 989 done += size; 990 /* Remove comments */ 991 tmp = strchr(kbuf, '#'); 992 if (tmp) 993 *tmp = '\0'; 994 995 ret = command_trace_probe(kbuf); 996 if (ret) 997 goto out; 998 } 999 ret = done; 1000 out: 1001 kfree(kbuf); 1002 return ret; 1003 } 1004 1005 static const struct file_operations kprobe_events_ops = { 1006 .owner = THIS_MODULE, 1007 .open = probes_open, 1008 .read = seq_read, 1009 .llseek = seq_lseek, 1010 .release = seq_release, 1011 .write = probes_write, 1012 }; 1013 1014 /* Probes profiling interfaces */ 1015 static int probes_profile_seq_show(struct seq_file *m, void *v) 1016 { 1017 struct trace_probe *tp = v; 1018 1019 seq_printf(m, " %-44s %15lu %15lu\n", tp->call.name, tp->nhit, 1020 tp->rp.kp.nmissed); 1021 1022 return 0; 1023 } 1024 1025 static const struct seq_operations profile_seq_op = { 1026 .start = probes_seq_start, 1027 .next = probes_seq_next, 1028 .stop = probes_seq_stop, 1029 .show = probes_profile_seq_show 1030 }; 1031 1032 static int profile_open(struct inode *inode, struct file *file) 1033 { 1034 return seq_open(file, &profile_seq_op); 1035 } 1036 1037 static const struct file_operations kprobe_profile_ops = { 1038 .owner = THIS_MODULE, 1039 .open = profile_open, 1040 .read = seq_read, 1041 .llseek = seq_lseek, 1042 .release = seq_release, 1043 }; 1044 1045 /* Kprobe handler */ 1046 static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) 1047 { 1048 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); 1049 struct kprobe_trace_entry_head *entry; 1050 struct ring_buffer_event *event; 1051 struct ring_buffer *buffer; 1052 u8 *data; 1053 int size, i, pc; 1054 unsigned long irq_flags; 1055 struct ftrace_event_call *call = &tp->call; 1056 1057 tp->nhit++; 1058 1059 local_save_flags(irq_flags); 1060 pc = preempt_count(); 1061 1062 size = sizeof(*entry) + tp->size; 1063 1064 event = trace_current_buffer_lock_reserve(&buffer, call->id, size, 1065 irq_flags, pc); 1066 if (!event) 1067 return; 1068 1069 entry = ring_buffer_event_data(event); 1070 entry->ip = (unsigned long)kp->addr; 1071 data = (u8 *)&entry[1]; 1072 for (i = 0; i < tp->nr_args; i++) 1073 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); 1074 1075 if (!filter_current_check_discard(buffer, call, entry, event)) 1076 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); 1077 } 1078 1079 /* Kretprobe handler */ 1080 static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri, 1081 struct pt_regs *regs) 1082 { 1083 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); 1084 struct kretprobe_trace_entry_head *entry; 1085 struct ring_buffer_event *event; 1086 struct ring_buffer *buffer; 1087 u8 *data; 1088 int size, i, pc; 1089 unsigned long irq_flags; 1090 struct ftrace_event_call *call = &tp->call; 1091 1092 local_save_flags(irq_flags); 1093 pc = preempt_count(); 1094 1095 size = sizeof(*entry) + tp->size; 1096 1097 event = trace_current_buffer_lock_reserve(&buffer, call->id, size, 1098 irq_flags, pc); 1099 if (!event) 1100 return; 1101 1102 entry = ring_buffer_event_data(event); 1103 entry->func = (unsigned long)tp->rp.kp.addr; 1104 entry->ret_ip = (unsigned long)ri->ret_addr; 1105 data = (u8 *)&entry[1]; 1106 for (i = 0; i < tp->nr_args; i++) 1107 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); 1108 1109 if (!filter_current_check_discard(buffer, call, entry, event)) 1110 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); 1111 } 1112 1113 /* Event entry printers */ 1114 enum print_line_t 1115 print_kprobe_event(struct trace_iterator *iter, int flags) 1116 { 1117 struct kprobe_trace_entry_head *field; 1118 struct trace_seq *s = &iter->seq; 1119 struct trace_event *event; 1120 struct trace_probe *tp; 1121 u8 *data; 1122 int i; 1123 1124 field = (struct kprobe_trace_entry_head *)iter->ent; 1125 event = ftrace_find_event(field->ent.type); 1126 tp = container_of(event, struct trace_probe, event); 1127 1128 if (!trace_seq_printf(s, "%s: (", tp->call.name)) 1129 goto partial; 1130 1131 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 1132 goto partial; 1133 1134 if (!trace_seq_puts(s, ")")) 1135 goto partial; 1136 1137 data = (u8 *)&field[1]; 1138 for (i = 0; i < tp->nr_args; i++) 1139 if (!tp->args[i].type->print(s, tp->args[i].name, 1140 data + tp->args[i].offset)) 1141 goto partial; 1142 1143 if (!trace_seq_puts(s, "\n")) 1144 goto partial; 1145 1146 return TRACE_TYPE_HANDLED; 1147 partial: 1148 return TRACE_TYPE_PARTIAL_LINE; 1149 } 1150 1151 enum print_line_t 1152 print_kretprobe_event(struct trace_iterator *iter, int flags) 1153 { 1154 struct kretprobe_trace_entry_head *field; 1155 struct trace_seq *s = &iter->seq; 1156 struct trace_event *event; 1157 struct trace_probe *tp; 1158 u8 *data; 1159 int i; 1160 1161 field = (struct kretprobe_trace_entry_head *)iter->ent; 1162 event = ftrace_find_event(field->ent.type); 1163 tp = container_of(event, struct trace_probe, event); 1164 1165 if (!trace_seq_printf(s, "%s: (", tp->call.name)) 1166 goto partial; 1167 1168 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 1169 goto partial; 1170 1171 if (!trace_seq_puts(s, " <- ")) 1172 goto partial; 1173 1174 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 1175 goto partial; 1176 1177 if (!trace_seq_puts(s, ")")) 1178 goto partial; 1179 1180 data = (u8 *)&field[1]; 1181 for (i = 0; i < tp->nr_args; i++) 1182 if (!tp->args[i].type->print(s, tp->args[i].name, 1183 data + tp->args[i].offset)) 1184 goto partial; 1185 1186 if (!trace_seq_puts(s, "\n")) 1187 goto partial; 1188 1189 return TRACE_TYPE_HANDLED; 1190 partial: 1191 return TRACE_TYPE_PARTIAL_LINE; 1192 } 1193 1194 static int probe_event_enable(struct ftrace_event_call *call) 1195 { 1196 struct trace_probe *tp = (struct trace_probe *)call->data; 1197 1198 tp->flags |= TP_FLAG_TRACE; 1199 if (probe_is_return(tp)) 1200 return enable_kretprobe(&tp->rp); 1201 else 1202 return enable_kprobe(&tp->rp.kp); 1203 } 1204 1205 static void probe_event_disable(struct ftrace_event_call *call) 1206 { 1207 struct trace_probe *tp = (struct trace_probe *)call->data; 1208 1209 tp->flags &= ~TP_FLAG_TRACE; 1210 if (!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE))) { 1211 if (probe_is_return(tp)) 1212 disable_kretprobe(&tp->rp); 1213 else 1214 disable_kprobe(&tp->rp.kp); 1215 } 1216 } 1217 1218 static int probe_event_raw_init(struct ftrace_event_call *event_call) 1219 { 1220 INIT_LIST_HEAD(&event_call->fields); 1221 1222 return 0; 1223 } 1224 1225 #undef DEFINE_FIELD 1226 #define DEFINE_FIELD(type, item, name, is_signed) \ 1227 do { \ 1228 ret = trace_define_field(event_call, #type, name, \ 1229 offsetof(typeof(field), item), \ 1230 sizeof(field.item), is_signed, \ 1231 FILTER_OTHER); \ 1232 if (ret) \ 1233 return ret; \ 1234 } while (0) 1235 1236 static int kprobe_event_define_fields(struct ftrace_event_call *event_call) 1237 { 1238 int ret, i; 1239 struct kprobe_trace_entry_head field; 1240 struct trace_probe *tp = (struct trace_probe *)event_call->data; 1241 1242 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1243 /* Set argument names as fields */ 1244 for (i = 0; i < tp->nr_args; i++) { 1245 ret = trace_define_field(event_call, tp->args[i].type->name, 1246 tp->args[i].name, 1247 sizeof(field) + tp->args[i].offset, 1248 tp->args[i].type->size, 1249 tp->args[i].type->is_signed, 1250 FILTER_OTHER); 1251 if (ret) 1252 return ret; 1253 } 1254 return 0; 1255 } 1256 1257 static int kretprobe_event_define_fields(struct ftrace_event_call *event_call) 1258 { 1259 int ret, i; 1260 struct kretprobe_trace_entry_head field; 1261 struct trace_probe *tp = (struct trace_probe *)event_call->data; 1262 1263 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1264 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1265 /* Set argument names as fields */ 1266 for (i = 0; i < tp->nr_args; i++) { 1267 ret = trace_define_field(event_call, tp->args[i].type->name, 1268 tp->args[i].name, 1269 sizeof(field) + tp->args[i].offset, 1270 tp->args[i].type->size, 1271 tp->args[i].type->is_signed, 1272 FILTER_OTHER); 1273 if (ret) 1274 return ret; 1275 } 1276 return 0; 1277 } 1278 1279 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len) 1280 { 1281 int i; 1282 int pos = 0; 1283 1284 const char *fmt, *arg; 1285 1286 if (!probe_is_return(tp)) { 1287 fmt = "(%lx)"; 1288 arg = "REC->" FIELD_STRING_IP; 1289 } else { 1290 fmt = "(%lx <- %lx)"; 1291 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP; 1292 } 1293 1294 /* When len=0, we just calculate the needed length */ 1295 #define LEN_OR_ZERO (len ? len - pos : 0) 1296 1297 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt); 1298 1299 for (i = 0; i < tp->nr_args; i++) { 1300 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s", 1301 tp->args[i].name, tp->args[i].type->fmt); 1302 } 1303 1304 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg); 1305 1306 for (i = 0; i < tp->nr_args; i++) { 1307 pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s", 1308 tp->args[i].name); 1309 } 1310 1311 #undef LEN_OR_ZERO 1312 1313 /* return the length of print_fmt */ 1314 return pos; 1315 } 1316 1317 static int set_print_fmt(struct trace_probe *tp) 1318 { 1319 int len; 1320 char *print_fmt; 1321 1322 /* First: called with 0 length to calculate the needed length */ 1323 len = __set_print_fmt(tp, NULL, 0); 1324 print_fmt = kmalloc(len + 1, GFP_KERNEL); 1325 if (!print_fmt) 1326 return -ENOMEM; 1327 1328 /* Second: actually write the @print_fmt */ 1329 __set_print_fmt(tp, print_fmt, len + 1); 1330 tp->call.print_fmt = print_fmt; 1331 1332 return 0; 1333 } 1334 1335 #ifdef CONFIG_PERF_EVENTS 1336 1337 /* Kprobe profile handler */ 1338 static __kprobes void kprobe_perf_func(struct kprobe *kp, 1339 struct pt_regs *regs) 1340 { 1341 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); 1342 struct ftrace_event_call *call = &tp->call; 1343 struct kprobe_trace_entry_head *entry; 1344 u8 *data; 1345 int size, __size, i; 1346 unsigned long irq_flags; 1347 int rctx; 1348 1349 __size = sizeof(*entry) + tp->size; 1350 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1351 size -= sizeof(u32); 1352 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, 1353 "profile buffer not large enough")) 1354 return; 1355 1356 entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); 1357 if (!entry) 1358 return; 1359 1360 entry->ip = (unsigned long)kp->addr; 1361 data = (u8 *)&entry[1]; 1362 for (i = 0; i < tp->nr_args; i++) 1363 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); 1364 1365 perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, irq_flags, regs); 1366 } 1367 1368 /* Kretprobe profile handler */ 1369 static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, 1370 struct pt_regs *regs) 1371 { 1372 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); 1373 struct ftrace_event_call *call = &tp->call; 1374 struct kretprobe_trace_entry_head *entry; 1375 u8 *data; 1376 int size, __size, i; 1377 unsigned long irq_flags; 1378 int rctx; 1379 1380 __size = sizeof(*entry) + tp->size; 1381 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1382 size -= sizeof(u32); 1383 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, 1384 "profile buffer not large enough")) 1385 return; 1386 1387 entry = perf_trace_buf_prepare(size, call->id, &rctx, &irq_flags); 1388 if (!entry) 1389 return; 1390 1391 entry->func = (unsigned long)tp->rp.kp.addr; 1392 entry->ret_ip = (unsigned long)ri->ret_addr; 1393 data = (u8 *)&entry[1]; 1394 for (i = 0; i < tp->nr_args; i++) 1395 call_fetch(&tp->args[i].fetch, regs, data + tp->args[i].offset); 1396 1397 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, 1398 irq_flags, regs); 1399 } 1400 1401 static int probe_perf_enable(struct ftrace_event_call *call) 1402 { 1403 struct trace_probe *tp = (struct trace_probe *)call->data; 1404 1405 tp->flags |= TP_FLAG_PROFILE; 1406 1407 if (probe_is_return(tp)) 1408 return enable_kretprobe(&tp->rp); 1409 else 1410 return enable_kprobe(&tp->rp.kp); 1411 } 1412 1413 static void probe_perf_disable(struct ftrace_event_call *call) 1414 { 1415 struct trace_probe *tp = (struct trace_probe *)call->data; 1416 1417 tp->flags &= ~TP_FLAG_PROFILE; 1418 1419 if (!(tp->flags & TP_FLAG_TRACE)) { 1420 if (probe_is_return(tp)) 1421 disable_kretprobe(&tp->rp); 1422 else 1423 disable_kprobe(&tp->rp.kp); 1424 } 1425 } 1426 #endif /* CONFIG_PERF_EVENTS */ 1427 1428 1429 static __kprobes 1430 int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1431 { 1432 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); 1433 1434 if (tp->flags & TP_FLAG_TRACE) 1435 kprobe_trace_func(kp, regs); 1436 #ifdef CONFIG_PERF_EVENTS 1437 if (tp->flags & TP_FLAG_PROFILE) 1438 kprobe_perf_func(kp, regs); 1439 #endif 1440 return 0; /* We don't tweek kernel, so just return 0 */ 1441 } 1442 1443 static __kprobes 1444 int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1445 { 1446 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); 1447 1448 if (tp->flags & TP_FLAG_TRACE) 1449 kretprobe_trace_func(ri, regs); 1450 #ifdef CONFIG_PERF_EVENTS 1451 if (tp->flags & TP_FLAG_PROFILE) 1452 kretprobe_perf_func(ri, regs); 1453 #endif 1454 return 0; /* We don't tweek kernel, so just return 0 */ 1455 } 1456 1457 static int register_probe_event(struct trace_probe *tp) 1458 { 1459 struct ftrace_event_call *call = &tp->call; 1460 int ret; 1461 1462 /* Initialize ftrace_event_call */ 1463 if (probe_is_return(tp)) { 1464 tp->event.trace = print_kretprobe_event; 1465 call->raw_init = probe_event_raw_init; 1466 call->define_fields = kretprobe_event_define_fields; 1467 } else { 1468 tp->event.trace = print_kprobe_event; 1469 call->raw_init = probe_event_raw_init; 1470 call->define_fields = kprobe_event_define_fields; 1471 } 1472 if (set_print_fmt(tp) < 0) 1473 return -ENOMEM; 1474 call->event = &tp->event; 1475 call->id = register_ftrace_event(&tp->event); 1476 if (!call->id) { 1477 kfree(call->print_fmt); 1478 return -ENODEV; 1479 } 1480 call->enabled = 0; 1481 call->regfunc = probe_event_enable; 1482 call->unregfunc = probe_event_disable; 1483 1484 #ifdef CONFIG_PERF_EVENTS 1485 call->perf_event_enable = probe_perf_enable; 1486 call->perf_event_disable = probe_perf_disable; 1487 #endif 1488 call->data = tp; 1489 ret = trace_add_event_call(call); 1490 if (ret) { 1491 pr_info("Failed to register kprobe event: %s\n", call->name); 1492 kfree(call->print_fmt); 1493 unregister_ftrace_event(&tp->event); 1494 } 1495 return ret; 1496 } 1497 1498 static void unregister_probe_event(struct trace_probe *tp) 1499 { 1500 /* tp->event is unregistered in trace_remove_event_call() */ 1501 trace_remove_event_call(&tp->call); 1502 kfree(tp->call.print_fmt); 1503 } 1504 1505 /* Make a debugfs interface for controling probe points */ 1506 static __init int init_kprobe_trace(void) 1507 { 1508 struct dentry *d_tracer; 1509 struct dentry *entry; 1510 1511 d_tracer = tracing_init_dentry(); 1512 if (!d_tracer) 1513 return 0; 1514 1515 entry = debugfs_create_file("kprobe_events", 0644, d_tracer, 1516 NULL, &kprobe_events_ops); 1517 1518 /* Event list interface */ 1519 if (!entry) 1520 pr_warning("Could not create debugfs " 1521 "'kprobe_events' entry\n"); 1522 1523 /* Profile interface */ 1524 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer, 1525 NULL, &kprobe_profile_ops); 1526 1527 if (!entry) 1528 pr_warning("Could not create debugfs " 1529 "'kprobe_profile' entry\n"); 1530 return 0; 1531 } 1532 fs_initcall(init_kprobe_trace); 1533 1534 1535 #ifdef CONFIG_FTRACE_STARTUP_TEST 1536 1537 static int kprobe_trace_selftest_target(int a1, int a2, int a3, 1538 int a4, int a5, int a6) 1539 { 1540 return a1 + a2 + a3 + a4 + a5 + a6; 1541 } 1542 1543 static __init int kprobe_trace_self_tests_init(void) 1544 { 1545 int ret, warn = 0; 1546 int (*target)(int, int, int, int, int, int); 1547 struct trace_probe *tp; 1548 1549 target = kprobe_trace_selftest_target; 1550 1551 pr_info("Testing kprobe tracing: "); 1552 1553 ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target " 1554 "$stack $stack0 +0($stack)"); 1555 if (WARN_ON_ONCE(ret)) { 1556 pr_warning("error on probing function entry.\n"); 1557 warn++; 1558 } else { 1559 /* Enable trace point */ 1560 tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM); 1561 if (WARN_ON_ONCE(tp == NULL)) { 1562 pr_warning("error on getting new probe.\n"); 1563 warn++; 1564 } else 1565 probe_event_enable(&tp->call); 1566 } 1567 1568 ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target " 1569 "$retval"); 1570 if (WARN_ON_ONCE(ret)) { 1571 pr_warning("error on probing function return.\n"); 1572 warn++; 1573 } else { 1574 /* Enable trace point */ 1575 tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM); 1576 if (WARN_ON_ONCE(tp == NULL)) { 1577 pr_warning("error on getting new probe.\n"); 1578 warn++; 1579 } else 1580 probe_event_enable(&tp->call); 1581 } 1582 1583 if (warn) 1584 goto end; 1585 1586 ret = target(1, 2, 3, 4, 5, 6); 1587 1588 ret = command_trace_probe("-:testprobe"); 1589 if (WARN_ON_ONCE(ret)) { 1590 pr_warning("error on deleting a probe.\n"); 1591 warn++; 1592 } 1593 1594 ret = command_trace_probe("-:testprobe2"); 1595 if (WARN_ON_ONCE(ret)) { 1596 pr_warning("error on deleting a probe.\n"); 1597 warn++; 1598 } 1599 1600 end: 1601 cleanup_all_probes(); 1602 if (warn) 1603 pr_cont("NG: Some tests are failed. Please check them.\n"); 1604 else 1605 pr_cont("OK\n"); 1606 return 0; 1607 } 1608 1609 late_initcall(kprobe_trace_self_tests_init); 1610 1611 #endif 1612