1 /* 2 * Kprobes-based tracing events 3 * 4 * Created by Masami Hiramatsu <mhiramat@redhat.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 20 #include <linux/module.h> 21 #include <linux/uaccess.h> 22 #include <linux/kprobes.h> 23 #include <linux/seq_file.h> 24 #include <linux/slab.h> 25 #include <linux/smp.h> 26 #include <linux/debugfs.h> 27 #include <linux/types.h> 28 #include <linux/string.h> 29 #include <linux/ctype.h> 30 #include <linux/ptrace.h> 31 #include <linux/perf_event.h> 32 #include <linux/stringify.h> 33 #include <linux/limits.h> 34 #include <asm/bitsperlong.h> 35 36 #include "trace.h" 37 #include "trace_output.h" 38 39 #define MAX_TRACE_ARGS 128 40 #define MAX_ARGSTR_LEN 63 41 #define MAX_EVENT_NAME_LEN 64 42 #define MAX_STRING_SIZE PATH_MAX 43 #define KPROBE_EVENT_SYSTEM "kprobes" 44 45 /* Reserved field names */ 46 #define FIELD_STRING_IP "__probe_ip" 47 #define FIELD_STRING_RETIP "__probe_ret_ip" 48 #define FIELD_STRING_FUNC "__probe_func" 49 50 const char *reserved_field_names[] = { 51 "common_type", 52 "common_flags", 53 "common_preempt_count", 54 "common_pid", 55 "common_tgid", 56 "common_lock_depth", 57 FIELD_STRING_IP, 58 FIELD_STRING_RETIP, 59 FIELD_STRING_FUNC, 60 }; 61 62 /* Printing function type */ 63 typedef int (*print_type_func_t)(struct trace_seq *, const char *, void *, 64 void *); 65 #define PRINT_TYPE_FUNC_NAME(type) print_type_##type 66 #define PRINT_TYPE_FMT_NAME(type) print_type_format_##type 67 68 /* Printing in basic type function template */ 69 #define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \ 70 static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \ 71 const char *name, \ 72 void *data, void *ent)\ 73 { \ 74 return trace_seq_printf(s, " %s=" fmt, name, (cast)*(type *)data);\ 75 } \ 76 static const char PRINT_TYPE_FMT_NAME(type)[] = fmt; 77 78 DEFINE_BASIC_PRINT_TYPE_FUNC(u8, "%x", unsigned int) 79 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, "%x", unsigned int) 80 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, "%lx", unsigned long) 81 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, "%llx", unsigned long long) 82 DEFINE_BASIC_PRINT_TYPE_FUNC(s8, "%d", int) 83 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, "%d", int) 84 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, "%ld", long) 85 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, "%lld", long long) 86 87 /* data_rloc: data relative location, compatible with u32 */ 88 #define make_data_rloc(len, roffs) \ 89 (((u32)(len) << 16) | ((u32)(roffs) & 0xffff)) 90 #define get_rloc_len(dl) ((u32)(dl) >> 16) 91 #define get_rloc_offs(dl) ((u32)(dl) & 0xffff) 92 93 static inline void *get_rloc_data(u32 *dl) 94 { 95 return (u8 *)dl + get_rloc_offs(*dl); 96 } 97 98 /* For data_loc conversion */ 99 static inline void *get_loc_data(u32 *dl, void *ent) 100 { 101 return (u8 *)ent + get_rloc_offs(*dl); 102 } 103 104 /* 105 * Convert data_rloc to data_loc: 106 * data_rloc stores the offset from data_rloc itself, but data_loc 107 * stores the offset from event entry. 108 */ 109 #define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs)) 110 111 /* For defining macros, define string/string_size types */ 112 typedef u32 string; 113 typedef u32 string_size; 114 115 /* Print type function for string type */ 116 static __kprobes int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, 117 const char *name, 118 void *data, void *ent) 119 { 120 int len = *(u32 *)data >> 16; 121 122 if (!len) 123 return trace_seq_printf(s, " %s=(fault)", name); 124 else 125 return trace_seq_printf(s, " %s=\"%s\"", name, 126 (const char *)get_loc_data(data, ent)); 127 } 128 static const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; 129 130 /* Data fetch function type */ 131 typedef void (*fetch_func_t)(struct pt_regs *, void *, void *); 132 133 struct fetch_param { 134 fetch_func_t fn; 135 void *data; 136 }; 137 138 static __kprobes void call_fetch(struct fetch_param *fprm, 139 struct pt_regs *regs, void *dest) 140 { 141 return fprm->fn(regs, fprm->data, dest); 142 } 143 144 #define FETCH_FUNC_NAME(method, type) fetch_##method##_##type 145 /* 146 * Define macro for basic types - we don't need to define s* types, because 147 * we have to care only about bitwidth at recording time. 148 */ 149 #define DEFINE_BASIC_FETCH_FUNCS(method) \ 150 DEFINE_FETCH_##method(u8) \ 151 DEFINE_FETCH_##method(u16) \ 152 DEFINE_FETCH_##method(u32) \ 153 DEFINE_FETCH_##method(u64) 154 155 #define CHECK_FETCH_FUNCS(method, fn) \ 156 (((FETCH_FUNC_NAME(method, u8) == fn) || \ 157 (FETCH_FUNC_NAME(method, u16) == fn) || \ 158 (FETCH_FUNC_NAME(method, u32) == fn) || \ 159 (FETCH_FUNC_NAME(method, u64) == fn) || \ 160 (FETCH_FUNC_NAME(method, string) == fn) || \ 161 (FETCH_FUNC_NAME(method, string_size) == fn)) \ 162 && (fn != NULL)) 163 164 /* Data fetch function templates */ 165 #define DEFINE_FETCH_reg(type) \ 166 static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \ 167 void *offset, void *dest) \ 168 { \ 169 *(type *)dest = (type)regs_get_register(regs, \ 170 (unsigned int)((unsigned long)offset)); \ 171 } 172 DEFINE_BASIC_FETCH_FUNCS(reg) 173 /* No string on the register */ 174 #define fetch_reg_string NULL 175 #define fetch_reg_string_size NULL 176 177 #define DEFINE_FETCH_stack(type) \ 178 static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\ 179 void *offset, void *dest) \ 180 { \ 181 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \ 182 (unsigned int)((unsigned long)offset)); \ 183 } 184 DEFINE_BASIC_FETCH_FUNCS(stack) 185 /* No string on the stack entry */ 186 #define fetch_stack_string NULL 187 #define fetch_stack_string_size NULL 188 189 #define DEFINE_FETCH_retval(type) \ 190 static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\ 191 void *dummy, void *dest) \ 192 { \ 193 *(type *)dest = (type)regs_return_value(regs); \ 194 } 195 DEFINE_BASIC_FETCH_FUNCS(retval) 196 /* No string on the retval */ 197 #define fetch_retval_string NULL 198 #define fetch_retval_string_size NULL 199 200 #define DEFINE_FETCH_memory(type) \ 201 static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\ 202 void *addr, void *dest) \ 203 { \ 204 type retval; \ 205 if (probe_kernel_address(addr, retval)) \ 206 *(type *)dest = 0; \ 207 else \ 208 *(type *)dest = retval; \ 209 } 210 DEFINE_BASIC_FETCH_FUNCS(memory) 211 /* 212 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max 213 * length and relative data location. 214 */ 215 static __kprobes void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, 216 void *addr, void *dest) 217 { 218 long ret; 219 int maxlen = get_rloc_len(*(u32 *)dest); 220 u8 *dst = get_rloc_data(dest); 221 u8 *src = addr; 222 mm_segment_t old_fs = get_fs(); 223 if (!maxlen) 224 return; 225 /* 226 * Try to get string again, since the string can be changed while 227 * probing. 228 */ 229 set_fs(KERNEL_DS); 230 pagefault_disable(); 231 do 232 ret = __copy_from_user_inatomic(dst++, src++, 1); 233 while (dst[-1] && ret == 0 && src - (u8 *)addr < maxlen); 234 dst[-1] = '\0'; 235 pagefault_enable(); 236 set_fs(old_fs); 237 238 if (ret < 0) { /* Failed to fetch string */ 239 ((u8 *)get_rloc_data(dest))[0] = '\0'; 240 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest)); 241 } else 242 *(u32 *)dest = make_data_rloc(src - (u8 *)addr, 243 get_rloc_offs(*(u32 *)dest)); 244 } 245 /* Return the length of string -- including null terminal byte */ 246 static __kprobes void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, 247 void *addr, void *dest) 248 { 249 int ret, len = 0; 250 u8 c; 251 mm_segment_t old_fs = get_fs(); 252 253 set_fs(KERNEL_DS); 254 pagefault_disable(); 255 do { 256 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1); 257 len++; 258 } while (c && ret == 0 && len < MAX_STRING_SIZE); 259 pagefault_enable(); 260 set_fs(old_fs); 261 262 if (ret < 0) /* Failed to check the length */ 263 *(u32 *)dest = 0; 264 else 265 *(u32 *)dest = len; 266 } 267 268 /* Memory fetching by symbol */ 269 struct symbol_cache { 270 char *symbol; 271 long offset; 272 unsigned long addr; 273 }; 274 275 static unsigned long update_symbol_cache(struct symbol_cache *sc) 276 { 277 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol); 278 if (sc->addr) 279 sc->addr += sc->offset; 280 return sc->addr; 281 } 282 283 static void free_symbol_cache(struct symbol_cache *sc) 284 { 285 kfree(sc->symbol); 286 kfree(sc); 287 } 288 289 static struct symbol_cache *alloc_symbol_cache(const char *sym, long offset) 290 { 291 struct symbol_cache *sc; 292 293 if (!sym || strlen(sym) == 0) 294 return NULL; 295 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL); 296 if (!sc) 297 return NULL; 298 299 sc->symbol = kstrdup(sym, GFP_KERNEL); 300 if (!sc->symbol) { 301 kfree(sc); 302 return NULL; 303 } 304 sc->offset = offset; 305 306 update_symbol_cache(sc); 307 return sc; 308 } 309 310 #define DEFINE_FETCH_symbol(type) \ 311 static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\ 312 void *data, void *dest) \ 313 { \ 314 struct symbol_cache *sc = data; \ 315 if (sc->addr) \ 316 fetch_memory_##type(regs, (void *)sc->addr, dest); \ 317 else \ 318 *(type *)dest = 0; \ 319 } 320 DEFINE_BASIC_FETCH_FUNCS(symbol) 321 DEFINE_FETCH_symbol(string) 322 DEFINE_FETCH_symbol(string_size) 323 324 /* Dereference memory access function */ 325 struct deref_fetch_param { 326 struct fetch_param orig; 327 long offset; 328 }; 329 330 #define DEFINE_FETCH_deref(type) \ 331 static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\ 332 void *data, void *dest) \ 333 { \ 334 struct deref_fetch_param *dprm = data; \ 335 unsigned long addr; \ 336 call_fetch(&dprm->orig, regs, &addr); \ 337 if (addr) { \ 338 addr += dprm->offset; \ 339 fetch_memory_##type(regs, (void *)addr, dest); \ 340 } else \ 341 *(type *)dest = 0; \ 342 } 343 DEFINE_BASIC_FETCH_FUNCS(deref) 344 DEFINE_FETCH_deref(string) 345 DEFINE_FETCH_deref(string_size) 346 347 static __kprobes void free_deref_fetch_param(struct deref_fetch_param *data) 348 { 349 if (CHECK_FETCH_FUNCS(deref, data->orig.fn)) 350 free_deref_fetch_param(data->orig.data); 351 else if (CHECK_FETCH_FUNCS(symbol, data->orig.fn)) 352 free_symbol_cache(data->orig.data); 353 kfree(data); 354 } 355 356 /* Default (unsigned long) fetch type */ 357 #define __DEFAULT_FETCH_TYPE(t) u##t 358 #define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t) 359 #define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG) 360 #define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE) 361 362 /* Fetch types */ 363 enum { 364 FETCH_MTD_reg = 0, 365 FETCH_MTD_stack, 366 FETCH_MTD_retval, 367 FETCH_MTD_memory, 368 FETCH_MTD_symbol, 369 FETCH_MTD_deref, 370 FETCH_MTD_END, 371 }; 372 373 #define ASSIGN_FETCH_FUNC(method, type) \ 374 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type) 375 376 #define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \ 377 {.name = _name, \ 378 .size = _size, \ 379 .is_signed = sign, \ 380 .print = PRINT_TYPE_FUNC_NAME(ptype), \ 381 .fmt = PRINT_TYPE_FMT_NAME(ptype), \ 382 .fmttype = _fmttype, \ 383 .fetch = { \ 384 ASSIGN_FETCH_FUNC(reg, ftype), \ 385 ASSIGN_FETCH_FUNC(stack, ftype), \ 386 ASSIGN_FETCH_FUNC(retval, ftype), \ 387 ASSIGN_FETCH_FUNC(memory, ftype), \ 388 ASSIGN_FETCH_FUNC(symbol, ftype), \ 389 ASSIGN_FETCH_FUNC(deref, ftype), \ 390 } \ 391 } 392 393 #define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \ 394 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype) 395 396 #define FETCH_TYPE_STRING 0 397 #define FETCH_TYPE_STRSIZE 1 398 399 /* Fetch type information table */ 400 static const struct fetch_type { 401 const char *name; /* Name of type */ 402 size_t size; /* Byte size of type */ 403 int is_signed; /* Signed flag */ 404 print_type_func_t print; /* Print functions */ 405 const char *fmt; /* Fromat string */ 406 const char *fmttype; /* Name in format file */ 407 /* Fetch functions */ 408 fetch_func_t fetch[FETCH_MTD_END]; 409 } fetch_type_table[] = { 410 /* Special types */ 411 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string, 412 sizeof(u32), 1, "__data_loc char[]"), 413 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32, 414 string_size, sizeof(u32), 0, "u32"), 415 /* Basic types */ 416 ASSIGN_FETCH_TYPE(u8, u8, 0), 417 ASSIGN_FETCH_TYPE(u16, u16, 0), 418 ASSIGN_FETCH_TYPE(u32, u32, 0), 419 ASSIGN_FETCH_TYPE(u64, u64, 0), 420 ASSIGN_FETCH_TYPE(s8, u8, 1), 421 ASSIGN_FETCH_TYPE(s16, u16, 1), 422 ASSIGN_FETCH_TYPE(s32, u32, 1), 423 ASSIGN_FETCH_TYPE(s64, u64, 1), 424 }; 425 426 static const struct fetch_type *find_fetch_type(const char *type) 427 { 428 int i; 429 430 if (!type) 431 type = DEFAULT_FETCH_TYPE_STR; 432 433 for (i = 0; i < ARRAY_SIZE(fetch_type_table); i++) 434 if (strcmp(type, fetch_type_table[i].name) == 0) 435 return &fetch_type_table[i]; 436 return NULL; 437 } 438 439 /* Special function : only accept unsigned long */ 440 static __kprobes void fetch_stack_address(struct pt_regs *regs, 441 void *dummy, void *dest) 442 { 443 *(unsigned long *)dest = kernel_stack_pointer(regs); 444 } 445 446 static fetch_func_t get_fetch_size_function(const struct fetch_type *type, 447 fetch_func_t orig_fn) 448 { 449 int i; 450 451 if (type != &fetch_type_table[FETCH_TYPE_STRING]) 452 return NULL; /* Only string type needs size function */ 453 for (i = 0; i < FETCH_MTD_END; i++) 454 if (type->fetch[i] == orig_fn) 455 return fetch_type_table[FETCH_TYPE_STRSIZE].fetch[i]; 456 457 WARN_ON(1); /* This should not happen */ 458 return NULL; 459 } 460 461 /** 462 * Kprobe event core functions 463 */ 464 465 struct probe_arg { 466 struct fetch_param fetch; 467 struct fetch_param fetch_size; 468 unsigned int offset; /* Offset from argument entry */ 469 const char *name; /* Name of this argument */ 470 const char *comm; /* Command of this argument */ 471 const struct fetch_type *type; /* Type of this argument */ 472 }; 473 474 /* Flags for trace_probe */ 475 #define TP_FLAG_TRACE 1 476 #define TP_FLAG_PROFILE 2 477 478 struct trace_probe { 479 struct list_head list; 480 struct kretprobe rp; /* Use rp.kp for kprobe use */ 481 unsigned long nhit; 482 unsigned int flags; /* For TP_FLAG_* */ 483 const char *symbol; /* symbol name */ 484 struct ftrace_event_class class; 485 struct ftrace_event_call call; 486 ssize_t size; /* trace entry size */ 487 unsigned int nr_args; 488 struct probe_arg args[]; 489 }; 490 491 #define SIZEOF_TRACE_PROBE(n) \ 492 (offsetof(struct trace_probe, args) + \ 493 (sizeof(struct probe_arg) * (n))) 494 495 496 static __kprobes int probe_is_return(struct trace_probe *tp) 497 { 498 return tp->rp.handler != NULL; 499 } 500 501 static __kprobes const char *probe_symbol(struct trace_probe *tp) 502 { 503 return tp->symbol ? tp->symbol : "unknown"; 504 } 505 506 static int register_probe_event(struct trace_probe *tp); 507 static void unregister_probe_event(struct trace_probe *tp); 508 509 static DEFINE_MUTEX(probe_lock); 510 static LIST_HEAD(probe_list); 511 512 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); 513 static int kretprobe_dispatcher(struct kretprobe_instance *ri, 514 struct pt_regs *regs); 515 516 /* Check the name is good for event/group/fields */ 517 static int is_good_name(const char *name) 518 { 519 if (!isalpha(*name) && *name != '_') 520 return 0; 521 while (*++name != '\0') { 522 if (!isalpha(*name) && !isdigit(*name) && *name != '_') 523 return 0; 524 } 525 return 1; 526 } 527 528 /* 529 * Allocate new trace_probe and initialize it (including kprobes). 530 */ 531 static struct trace_probe *alloc_trace_probe(const char *group, 532 const char *event, 533 void *addr, 534 const char *symbol, 535 unsigned long offs, 536 int nargs, int is_return) 537 { 538 struct trace_probe *tp; 539 int ret = -ENOMEM; 540 541 tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL); 542 if (!tp) 543 return ERR_PTR(ret); 544 545 if (symbol) { 546 tp->symbol = kstrdup(symbol, GFP_KERNEL); 547 if (!tp->symbol) 548 goto error; 549 tp->rp.kp.symbol_name = tp->symbol; 550 tp->rp.kp.offset = offs; 551 } else 552 tp->rp.kp.addr = addr; 553 554 if (is_return) 555 tp->rp.handler = kretprobe_dispatcher; 556 else 557 tp->rp.kp.pre_handler = kprobe_dispatcher; 558 559 if (!event || !is_good_name(event)) { 560 ret = -EINVAL; 561 goto error; 562 } 563 564 tp->call.class = &tp->class; 565 tp->call.name = kstrdup(event, GFP_KERNEL); 566 if (!tp->call.name) 567 goto error; 568 569 if (!group || !is_good_name(group)) { 570 ret = -EINVAL; 571 goto error; 572 } 573 574 tp->class.system = kstrdup(group, GFP_KERNEL); 575 if (!tp->class.system) 576 goto error; 577 578 INIT_LIST_HEAD(&tp->list); 579 return tp; 580 error: 581 kfree(tp->call.name); 582 kfree(tp->symbol); 583 kfree(tp); 584 return ERR_PTR(ret); 585 } 586 587 static void free_probe_arg(struct probe_arg *arg) 588 { 589 if (CHECK_FETCH_FUNCS(deref, arg->fetch.fn)) 590 free_deref_fetch_param(arg->fetch.data); 591 else if (CHECK_FETCH_FUNCS(symbol, arg->fetch.fn)) 592 free_symbol_cache(arg->fetch.data); 593 kfree(arg->name); 594 kfree(arg->comm); 595 } 596 597 static void free_trace_probe(struct trace_probe *tp) 598 { 599 int i; 600 601 for (i = 0; i < tp->nr_args; i++) 602 free_probe_arg(&tp->args[i]); 603 604 kfree(tp->call.class->system); 605 kfree(tp->call.name); 606 kfree(tp->symbol); 607 kfree(tp); 608 } 609 610 static struct trace_probe *find_probe_event(const char *event, 611 const char *group) 612 { 613 struct trace_probe *tp; 614 615 list_for_each_entry(tp, &probe_list, list) 616 if (strcmp(tp->call.name, event) == 0 && 617 strcmp(tp->call.class->system, group) == 0) 618 return tp; 619 return NULL; 620 } 621 622 /* Unregister a trace_probe and probe_event: call with locking probe_lock */ 623 static void unregister_trace_probe(struct trace_probe *tp) 624 { 625 if (probe_is_return(tp)) 626 unregister_kretprobe(&tp->rp); 627 else 628 unregister_kprobe(&tp->rp.kp); 629 list_del(&tp->list); 630 unregister_probe_event(tp); 631 } 632 633 /* Register a trace_probe and probe_event */ 634 static int register_trace_probe(struct trace_probe *tp) 635 { 636 struct trace_probe *old_tp; 637 int ret; 638 639 mutex_lock(&probe_lock); 640 641 /* register as an event */ 642 old_tp = find_probe_event(tp->call.name, tp->call.class->system); 643 if (old_tp) { 644 /* delete old event */ 645 unregister_trace_probe(old_tp); 646 free_trace_probe(old_tp); 647 } 648 ret = register_probe_event(tp); 649 if (ret) { 650 pr_warning("Failed to register probe event(%d)\n", ret); 651 goto end; 652 } 653 654 tp->rp.kp.flags |= KPROBE_FLAG_DISABLED; 655 if (probe_is_return(tp)) 656 ret = register_kretprobe(&tp->rp); 657 else 658 ret = register_kprobe(&tp->rp.kp); 659 660 if (ret) { 661 pr_warning("Could not insert probe(%d)\n", ret); 662 if (ret == -EILSEQ) { 663 pr_warning("Probing address(0x%p) is not an " 664 "instruction boundary.\n", 665 tp->rp.kp.addr); 666 ret = -EINVAL; 667 } 668 unregister_probe_event(tp); 669 } else 670 list_add_tail(&tp->list, &probe_list); 671 end: 672 mutex_unlock(&probe_lock); 673 return ret; 674 } 675 676 /* Split symbol and offset. */ 677 static int split_symbol_offset(char *symbol, unsigned long *offset) 678 { 679 char *tmp; 680 int ret; 681 682 if (!offset) 683 return -EINVAL; 684 685 tmp = strchr(symbol, '+'); 686 if (tmp) { 687 /* skip sign because strict_strtol doesn't accept '+' */ 688 ret = strict_strtoul(tmp + 1, 0, offset); 689 if (ret) 690 return ret; 691 *tmp = '\0'; 692 } else 693 *offset = 0; 694 return 0; 695 } 696 697 #define PARAM_MAX_ARGS 16 698 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 699 700 static int parse_probe_vars(char *arg, const struct fetch_type *t, 701 struct fetch_param *f, int is_return) 702 { 703 int ret = 0; 704 unsigned long param; 705 706 if (strcmp(arg, "retval") == 0) { 707 if (is_return) 708 f->fn = t->fetch[FETCH_MTD_retval]; 709 else 710 ret = -EINVAL; 711 } else if (strncmp(arg, "stack", 5) == 0) { 712 if (arg[5] == '\0') { 713 if (strcmp(t->name, DEFAULT_FETCH_TYPE_STR) == 0) 714 f->fn = fetch_stack_address; 715 else 716 ret = -EINVAL; 717 } else if (isdigit(arg[5])) { 718 ret = strict_strtoul(arg + 5, 10, ¶m); 719 if (ret || param > PARAM_MAX_STACK) 720 ret = -EINVAL; 721 else { 722 f->fn = t->fetch[FETCH_MTD_stack]; 723 f->data = (void *)param; 724 } 725 } else 726 ret = -EINVAL; 727 } else 728 ret = -EINVAL; 729 return ret; 730 } 731 732 /* Recursive argument parser */ 733 static int __parse_probe_arg(char *arg, const struct fetch_type *t, 734 struct fetch_param *f, int is_return) 735 { 736 int ret = 0; 737 unsigned long param; 738 long offset; 739 char *tmp; 740 741 switch (arg[0]) { 742 case '$': 743 ret = parse_probe_vars(arg + 1, t, f, is_return); 744 break; 745 case '%': /* named register */ 746 ret = regs_query_register_offset(arg + 1); 747 if (ret >= 0) { 748 f->fn = t->fetch[FETCH_MTD_reg]; 749 f->data = (void *)(unsigned long)ret; 750 ret = 0; 751 } 752 break; 753 case '@': /* memory or symbol */ 754 if (isdigit(arg[1])) { 755 ret = strict_strtoul(arg + 1, 0, ¶m); 756 if (ret) 757 break; 758 f->fn = t->fetch[FETCH_MTD_memory]; 759 f->data = (void *)param; 760 } else { 761 ret = split_symbol_offset(arg + 1, &offset); 762 if (ret) 763 break; 764 f->data = alloc_symbol_cache(arg + 1, offset); 765 if (f->data) 766 f->fn = t->fetch[FETCH_MTD_symbol]; 767 } 768 break; 769 case '+': /* deref memory */ 770 case '-': 771 tmp = strchr(arg, '('); 772 if (!tmp) 773 break; 774 *tmp = '\0'; 775 ret = strict_strtol(arg + 1, 0, &offset); 776 if (ret) 777 break; 778 if (arg[0] == '-') 779 offset = -offset; 780 arg = tmp + 1; 781 tmp = strrchr(arg, ')'); 782 if (tmp) { 783 struct deref_fetch_param *dprm; 784 const struct fetch_type *t2 = find_fetch_type(NULL); 785 *tmp = '\0'; 786 dprm = kzalloc(sizeof(struct deref_fetch_param), 787 GFP_KERNEL); 788 if (!dprm) 789 return -ENOMEM; 790 dprm->offset = offset; 791 ret = __parse_probe_arg(arg, t2, &dprm->orig, 792 is_return); 793 if (ret) 794 kfree(dprm); 795 else { 796 f->fn = t->fetch[FETCH_MTD_deref]; 797 f->data = (void *)dprm; 798 } 799 } 800 break; 801 } 802 if (!ret && !f->fn) { /* Parsed, but do not find fetch method */ 803 pr_info("%s type has no corresponding fetch method.\n", 804 t->name); 805 ret = -EINVAL; 806 } 807 return ret; 808 } 809 810 /* String length checking wrapper */ 811 static int parse_probe_arg(char *arg, struct trace_probe *tp, 812 struct probe_arg *parg, int is_return) 813 { 814 const char *t; 815 int ret; 816 817 if (strlen(arg) > MAX_ARGSTR_LEN) { 818 pr_info("Argument is too long.: %s\n", arg); 819 return -ENOSPC; 820 } 821 parg->comm = kstrdup(arg, GFP_KERNEL); 822 if (!parg->comm) { 823 pr_info("Failed to allocate memory for command '%s'.\n", arg); 824 return -ENOMEM; 825 } 826 t = strchr(parg->comm, ':'); 827 if (t) { 828 arg[t - parg->comm] = '\0'; 829 t++; 830 } 831 parg->type = find_fetch_type(t); 832 if (!parg->type) { 833 pr_info("Unsupported type: %s\n", t); 834 return -EINVAL; 835 } 836 parg->offset = tp->size; 837 tp->size += parg->type->size; 838 ret = __parse_probe_arg(arg, parg->type, &parg->fetch, is_return); 839 if (ret >= 0) { 840 parg->fetch_size.fn = get_fetch_size_function(parg->type, 841 parg->fetch.fn); 842 parg->fetch_size.data = parg->fetch.data; 843 } 844 return ret; 845 } 846 847 /* Return 1 if name is reserved or already used by another argument */ 848 static int conflict_field_name(const char *name, 849 struct probe_arg *args, int narg) 850 { 851 int i; 852 for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++) 853 if (strcmp(reserved_field_names[i], name) == 0) 854 return 1; 855 for (i = 0; i < narg; i++) 856 if (strcmp(args[i].name, name) == 0) 857 return 1; 858 return 0; 859 } 860 861 static int create_trace_probe(int argc, char **argv) 862 { 863 /* 864 * Argument syntax: 865 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS] 866 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS] 867 * Fetch args: 868 * $retval : fetch return value 869 * $stack : fetch stack address 870 * $stackN : fetch Nth of stack (N:0-) 871 * @ADDR : fetch memory at ADDR (ADDR should be in kernel) 872 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) 873 * %REG : fetch register REG 874 * Dereferencing memory fetch: 875 * +|-offs(ARG) : fetch memory at ARG +|- offs address. 876 * Alias name of args: 877 * NAME=FETCHARG : set NAME as alias of FETCHARG. 878 * Type of args: 879 * FETCHARG:TYPE : use TYPE instead of unsigned long. 880 */ 881 struct trace_probe *tp; 882 int i, ret = 0; 883 int is_return = 0, is_delete = 0; 884 char *symbol = NULL, *event = NULL, *group = NULL; 885 char *arg; 886 unsigned long offset = 0; 887 void *addr = NULL; 888 char buf[MAX_EVENT_NAME_LEN]; 889 890 /* argc must be >= 1 */ 891 if (argv[0][0] == 'p') 892 is_return = 0; 893 else if (argv[0][0] == 'r') 894 is_return = 1; 895 else if (argv[0][0] == '-') 896 is_delete = 1; 897 else { 898 pr_info("Probe definition must be started with 'p', 'r' or" 899 " '-'.\n"); 900 return -EINVAL; 901 } 902 903 if (argv[0][1] == ':') { 904 event = &argv[0][2]; 905 if (strchr(event, '/')) { 906 group = event; 907 event = strchr(group, '/') + 1; 908 event[-1] = '\0'; 909 if (strlen(group) == 0) { 910 pr_info("Group name is not specified\n"); 911 return -EINVAL; 912 } 913 } 914 if (strlen(event) == 0) { 915 pr_info("Event name is not specified\n"); 916 return -EINVAL; 917 } 918 } 919 if (!group) 920 group = KPROBE_EVENT_SYSTEM; 921 922 if (is_delete) { 923 if (!event) { 924 pr_info("Delete command needs an event name.\n"); 925 return -EINVAL; 926 } 927 mutex_lock(&probe_lock); 928 tp = find_probe_event(event, group); 929 if (!tp) { 930 mutex_unlock(&probe_lock); 931 pr_info("Event %s/%s doesn't exist.\n", group, event); 932 return -ENOENT; 933 } 934 /* delete an event */ 935 unregister_trace_probe(tp); 936 free_trace_probe(tp); 937 mutex_unlock(&probe_lock); 938 return 0; 939 } 940 941 if (argc < 2) { 942 pr_info("Probe point is not specified.\n"); 943 return -EINVAL; 944 } 945 if (isdigit(argv[1][0])) { 946 if (is_return) { 947 pr_info("Return probe point must be a symbol.\n"); 948 return -EINVAL; 949 } 950 /* an address specified */ 951 ret = strict_strtoul(&argv[1][0], 0, (unsigned long *)&addr); 952 if (ret) { 953 pr_info("Failed to parse address.\n"); 954 return ret; 955 } 956 } else { 957 /* a symbol specified */ 958 symbol = argv[1]; 959 /* TODO: support .init module functions */ 960 ret = split_symbol_offset(symbol, &offset); 961 if (ret) { 962 pr_info("Failed to parse symbol.\n"); 963 return ret; 964 } 965 if (offset && is_return) { 966 pr_info("Return probe must be used without offset.\n"); 967 return -EINVAL; 968 } 969 } 970 argc -= 2; argv += 2; 971 972 /* setup a probe */ 973 if (!event) { 974 /* Make a new event name */ 975 if (symbol) 976 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", 977 is_return ? 'r' : 'p', symbol, offset); 978 else 979 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p", 980 is_return ? 'r' : 'p', addr); 981 event = buf; 982 } 983 tp = alloc_trace_probe(group, event, addr, symbol, offset, argc, 984 is_return); 985 if (IS_ERR(tp)) { 986 pr_info("Failed to allocate trace_probe.(%d)\n", 987 (int)PTR_ERR(tp)); 988 return PTR_ERR(tp); 989 } 990 991 /* parse arguments */ 992 ret = 0; 993 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { 994 /* Increment count for freeing args in error case */ 995 tp->nr_args++; 996 997 /* Parse argument name */ 998 arg = strchr(argv[i], '='); 999 if (arg) { 1000 *arg++ = '\0'; 1001 tp->args[i].name = kstrdup(argv[i], GFP_KERNEL); 1002 } else { 1003 arg = argv[i]; 1004 /* If argument name is omitted, set "argN" */ 1005 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1); 1006 tp->args[i].name = kstrdup(buf, GFP_KERNEL); 1007 } 1008 1009 if (!tp->args[i].name) { 1010 pr_info("Failed to allocate argument[%d] name.\n", i); 1011 ret = -ENOMEM; 1012 goto error; 1013 } 1014 1015 if (!is_good_name(tp->args[i].name)) { 1016 pr_info("Invalid argument[%d] name: %s\n", 1017 i, tp->args[i].name); 1018 ret = -EINVAL; 1019 goto error; 1020 } 1021 1022 if (conflict_field_name(tp->args[i].name, tp->args, i)) { 1023 pr_info("Argument[%d] name '%s' conflicts with " 1024 "another field.\n", i, argv[i]); 1025 ret = -EINVAL; 1026 goto error; 1027 } 1028 1029 /* Parse fetch argument */ 1030 ret = parse_probe_arg(arg, tp, &tp->args[i], is_return); 1031 if (ret) { 1032 pr_info("Parse error at argument[%d]. (%d)\n", i, ret); 1033 goto error; 1034 } 1035 } 1036 1037 ret = register_trace_probe(tp); 1038 if (ret) 1039 goto error; 1040 return 0; 1041 1042 error: 1043 free_trace_probe(tp); 1044 return ret; 1045 } 1046 1047 static void cleanup_all_probes(void) 1048 { 1049 struct trace_probe *tp; 1050 1051 mutex_lock(&probe_lock); 1052 /* TODO: Use batch unregistration */ 1053 while (!list_empty(&probe_list)) { 1054 tp = list_entry(probe_list.next, struct trace_probe, list); 1055 unregister_trace_probe(tp); 1056 free_trace_probe(tp); 1057 } 1058 mutex_unlock(&probe_lock); 1059 } 1060 1061 1062 /* Probes listing interfaces */ 1063 static void *probes_seq_start(struct seq_file *m, loff_t *pos) 1064 { 1065 mutex_lock(&probe_lock); 1066 return seq_list_start(&probe_list, *pos); 1067 } 1068 1069 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) 1070 { 1071 return seq_list_next(v, &probe_list, pos); 1072 } 1073 1074 static void probes_seq_stop(struct seq_file *m, void *v) 1075 { 1076 mutex_unlock(&probe_lock); 1077 } 1078 1079 static int probes_seq_show(struct seq_file *m, void *v) 1080 { 1081 struct trace_probe *tp = v; 1082 int i; 1083 1084 seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p'); 1085 seq_printf(m, ":%s/%s", tp->call.class->system, tp->call.name); 1086 1087 if (!tp->symbol) 1088 seq_printf(m, " 0x%p", tp->rp.kp.addr); 1089 else if (tp->rp.kp.offset) 1090 seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset); 1091 else 1092 seq_printf(m, " %s", probe_symbol(tp)); 1093 1094 for (i = 0; i < tp->nr_args; i++) 1095 seq_printf(m, " %s=%s", tp->args[i].name, tp->args[i].comm); 1096 seq_printf(m, "\n"); 1097 1098 return 0; 1099 } 1100 1101 static const struct seq_operations probes_seq_op = { 1102 .start = probes_seq_start, 1103 .next = probes_seq_next, 1104 .stop = probes_seq_stop, 1105 .show = probes_seq_show 1106 }; 1107 1108 static int probes_open(struct inode *inode, struct file *file) 1109 { 1110 if ((file->f_mode & FMODE_WRITE) && 1111 (file->f_flags & O_TRUNC)) 1112 cleanup_all_probes(); 1113 1114 return seq_open(file, &probes_seq_op); 1115 } 1116 1117 static int command_trace_probe(const char *buf) 1118 { 1119 char **argv; 1120 int argc = 0, ret = 0; 1121 1122 argv = argv_split(GFP_KERNEL, buf, &argc); 1123 if (!argv) 1124 return -ENOMEM; 1125 1126 if (argc) 1127 ret = create_trace_probe(argc, argv); 1128 1129 argv_free(argv); 1130 return ret; 1131 } 1132 1133 #define WRITE_BUFSIZE 128 1134 1135 static ssize_t probes_write(struct file *file, const char __user *buffer, 1136 size_t count, loff_t *ppos) 1137 { 1138 char *kbuf, *tmp; 1139 int ret; 1140 size_t done; 1141 size_t size; 1142 1143 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); 1144 if (!kbuf) 1145 return -ENOMEM; 1146 1147 ret = done = 0; 1148 while (done < count) { 1149 size = count - done; 1150 if (size >= WRITE_BUFSIZE) 1151 size = WRITE_BUFSIZE - 1; 1152 if (copy_from_user(kbuf, buffer + done, size)) { 1153 ret = -EFAULT; 1154 goto out; 1155 } 1156 kbuf[size] = '\0'; 1157 tmp = strchr(kbuf, '\n'); 1158 if (tmp) { 1159 *tmp = '\0'; 1160 size = tmp - kbuf + 1; 1161 } else if (done + size < count) { 1162 pr_warning("Line length is too long: " 1163 "Should be less than %d.", WRITE_BUFSIZE); 1164 ret = -EINVAL; 1165 goto out; 1166 } 1167 done += size; 1168 /* Remove comments */ 1169 tmp = strchr(kbuf, '#'); 1170 if (tmp) 1171 *tmp = '\0'; 1172 1173 ret = command_trace_probe(kbuf); 1174 if (ret) 1175 goto out; 1176 } 1177 ret = done; 1178 out: 1179 kfree(kbuf); 1180 return ret; 1181 } 1182 1183 static const struct file_operations kprobe_events_ops = { 1184 .owner = THIS_MODULE, 1185 .open = probes_open, 1186 .read = seq_read, 1187 .llseek = seq_lseek, 1188 .release = seq_release, 1189 .write = probes_write, 1190 }; 1191 1192 /* Probes profiling interfaces */ 1193 static int probes_profile_seq_show(struct seq_file *m, void *v) 1194 { 1195 struct trace_probe *tp = v; 1196 1197 seq_printf(m, " %-44s %15lu %15lu\n", tp->call.name, tp->nhit, 1198 tp->rp.kp.nmissed); 1199 1200 return 0; 1201 } 1202 1203 static const struct seq_operations profile_seq_op = { 1204 .start = probes_seq_start, 1205 .next = probes_seq_next, 1206 .stop = probes_seq_stop, 1207 .show = probes_profile_seq_show 1208 }; 1209 1210 static int profile_open(struct inode *inode, struct file *file) 1211 { 1212 return seq_open(file, &profile_seq_op); 1213 } 1214 1215 static const struct file_operations kprobe_profile_ops = { 1216 .owner = THIS_MODULE, 1217 .open = profile_open, 1218 .read = seq_read, 1219 .llseek = seq_lseek, 1220 .release = seq_release, 1221 }; 1222 1223 /* Sum up total data length for dynamic arraies (strings) */ 1224 static __kprobes int __get_data_size(struct trace_probe *tp, 1225 struct pt_regs *regs) 1226 { 1227 int i, ret = 0; 1228 u32 len; 1229 1230 for (i = 0; i < tp->nr_args; i++) 1231 if (unlikely(tp->args[i].fetch_size.fn)) { 1232 call_fetch(&tp->args[i].fetch_size, regs, &len); 1233 ret += len; 1234 } 1235 1236 return ret; 1237 } 1238 1239 /* Store the value of each argument */ 1240 static __kprobes void store_trace_args(int ent_size, struct trace_probe *tp, 1241 struct pt_regs *regs, 1242 u8 *data, int maxlen) 1243 { 1244 int i; 1245 u32 end = tp->size; 1246 u32 *dl; /* Data (relative) location */ 1247 1248 for (i = 0; i < tp->nr_args; i++) { 1249 if (unlikely(tp->args[i].fetch_size.fn)) { 1250 /* 1251 * First, we set the relative location and 1252 * maximum data length to *dl 1253 */ 1254 dl = (u32 *)(data + tp->args[i].offset); 1255 *dl = make_data_rloc(maxlen, end - tp->args[i].offset); 1256 /* Then try to fetch string or dynamic array data */ 1257 call_fetch(&tp->args[i].fetch, regs, dl); 1258 /* Reduce maximum length */ 1259 end += get_rloc_len(*dl); 1260 maxlen -= get_rloc_len(*dl); 1261 /* Trick here, convert data_rloc to data_loc */ 1262 *dl = convert_rloc_to_loc(*dl, 1263 ent_size + tp->args[i].offset); 1264 } else 1265 /* Just fetching data normally */ 1266 call_fetch(&tp->args[i].fetch, regs, 1267 data + tp->args[i].offset); 1268 } 1269 } 1270 1271 /* Kprobe handler */ 1272 static __kprobes void kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) 1273 { 1274 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); 1275 struct kprobe_trace_entry_head *entry; 1276 struct ring_buffer_event *event; 1277 struct ring_buffer *buffer; 1278 int size, dsize, pc; 1279 unsigned long irq_flags; 1280 struct ftrace_event_call *call = &tp->call; 1281 1282 tp->nhit++; 1283 1284 local_save_flags(irq_flags); 1285 pc = preempt_count(); 1286 1287 dsize = __get_data_size(tp, regs); 1288 size = sizeof(*entry) + tp->size + dsize; 1289 1290 event = trace_current_buffer_lock_reserve(&buffer, call->event.type, 1291 size, irq_flags, pc); 1292 if (!event) 1293 return; 1294 1295 entry = ring_buffer_event_data(event); 1296 entry->ip = (unsigned long)kp->addr; 1297 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); 1298 1299 if (!filter_current_check_discard(buffer, call, entry, event)) 1300 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); 1301 } 1302 1303 /* Kretprobe handler */ 1304 static __kprobes void kretprobe_trace_func(struct kretprobe_instance *ri, 1305 struct pt_regs *regs) 1306 { 1307 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); 1308 struct kretprobe_trace_entry_head *entry; 1309 struct ring_buffer_event *event; 1310 struct ring_buffer *buffer; 1311 int size, pc, dsize; 1312 unsigned long irq_flags; 1313 struct ftrace_event_call *call = &tp->call; 1314 1315 local_save_flags(irq_flags); 1316 pc = preempt_count(); 1317 1318 dsize = __get_data_size(tp, regs); 1319 size = sizeof(*entry) + tp->size + dsize; 1320 1321 event = trace_current_buffer_lock_reserve(&buffer, call->event.type, 1322 size, irq_flags, pc); 1323 if (!event) 1324 return; 1325 1326 entry = ring_buffer_event_data(event); 1327 entry->func = (unsigned long)tp->rp.kp.addr; 1328 entry->ret_ip = (unsigned long)ri->ret_addr; 1329 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); 1330 1331 if (!filter_current_check_discard(buffer, call, entry, event)) 1332 trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); 1333 } 1334 1335 /* Event entry printers */ 1336 enum print_line_t 1337 print_kprobe_event(struct trace_iterator *iter, int flags, 1338 struct trace_event *event) 1339 { 1340 struct kprobe_trace_entry_head *field; 1341 struct trace_seq *s = &iter->seq; 1342 struct trace_probe *tp; 1343 u8 *data; 1344 int i; 1345 1346 field = (struct kprobe_trace_entry_head *)iter->ent; 1347 tp = container_of(event, struct trace_probe, call.event); 1348 1349 if (!trace_seq_printf(s, "%s: (", tp->call.name)) 1350 goto partial; 1351 1352 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) 1353 goto partial; 1354 1355 if (!trace_seq_puts(s, ")")) 1356 goto partial; 1357 1358 data = (u8 *)&field[1]; 1359 for (i = 0; i < tp->nr_args; i++) 1360 if (!tp->args[i].type->print(s, tp->args[i].name, 1361 data + tp->args[i].offset, field)) 1362 goto partial; 1363 1364 if (!trace_seq_puts(s, "\n")) 1365 goto partial; 1366 1367 return TRACE_TYPE_HANDLED; 1368 partial: 1369 return TRACE_TYPE_PARTIAL_LINE; 1370 } 1371 1372 enum print_line_t 1373 print_kretprobe_event(struct trace_iterator *iter, int flags, 1374 struct trace_event *event) 1375 { 1376 struct kretprobe_trace_entry_head *field; 1377 struct trace_seq *s = &iter->seq; 1378 struct trace_probe *tp; 1379 u8 *data; 1380 int i; 1381 1382 field = (struct kretprobe_trace_entry_head *)iter->ent; 1383 tp = container_of(event, struct trace_probe, call.event); 1384 1385 if (!trace_seq_printf(s, "%s: (", tp->call.name)) 1386 goto partial; 1387 1388 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) 1389 goto partial; 1390 1391 if (!trace_seq_puts(s, " <- ")) 1392 goto partial; 1393 1394 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) 1395 goto partial; 1396 1397 if (!trace_seq_puts(s, ")")) 1398 goto partial; 1399 1400 data = (u8 *)&field[1]; 1401 for (i = 0; i < tp->nr_args; i++) 1402 if (!tp->args[i].type->print(s, tp->args[i].name, 1403 data + tp->args[i].offset, field)) 1404 goto partial; 1405 1406 if (!trace_seq_puts(s, "\n")) 1407 goto partial; 1408 1409 return TRACE_TYPE_HANDLED; 1410 partial: 1411 return TRACE_TYPE_PARTIAL_LINE; 1412 } 1413 1414 static int probe_event_enable(struct ftrace_event_call *call) 1415 { 1416 struct trace_probe *tp = (struct trace_probe *)call->data; 1417 1418 tp->flags |= TP_FLAG_TRACE; 1419 if (probe_is_return(tp)) 1420 return enable_kretprobe(&tp->rp); 1421 else 1422 return enable_kprobe(&tp->rp.kp); 1423 } 1424 1425 static void probe_event_disable(struct ftrace_event_call *call) 1426 { 1427 struct trace_probe *tp = (struct trace_probe *)call->data; 1428 1429 tp->flags &= ~TP_FLAG_TRACE; 1430 if (!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE))) { 1431 if (probe_is_return(tp)) 1432 disable_kretprobe(&tp->rp); 1433 else 1434 disable_kprobe(&tp->rp.kp); 1435 } 1436 } 1437 1438 #undef DEFINE_FIELD 1439 #define DEFINE_FIELD(type, item, name, is_signed) \ 1440 do { \ 1441 ret = trace_define_field(event_call, #type, name, \ 1442 offsetof(typeof(field), item), \ 1443 sizeof(field.item), is_signed, \ 1444 FILTER_OTHER); \ 1445 if (ret) \ 1446 return ret; \ 1447 } while (0) 1448 1449 static int kprobe_event_define_fields(struct ftrace_event_call *event_call) 1450 { 1451 int ret, i; 1452 struct kprobe_trace_entry_head field; 1453 struct trace_probe *tp = (struct trace_probe *)event_call->data; 1454 1455 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); 1456 /* Set argument names as fields */ 1457 for (i = 0; i < tp->nr_args; i++) { 1458 ret = trace_define_field(event_call, tp->args[i].type->fmttype, 1459 tp->args[i].name, 1460 sizeof(field) + tp->args[i].offset, 1461 tp->args[i].type->size, 1462 tp->args[i].type->is_signed, 1463 FILTER_OTHER); 1464 if (ret) 1465 return ret; 1466 } 1467 return 0; 1468 } 1469 1470 static int kretprobe_event_define_fields(struct ftrace_event_call *event_call) 1471 { 1472 int ret, i; 1473 struct kretprobe_trace_entry_head field; 1474 struct trace_probe *tp = (struct trace_probe *)event_call->data; 1475 1476 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); 1477 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); 1478 /* Set argument names as fields */ 1479 for (i = 0; i < tp->nr_args; i++) { 1480 ret = trace_define_field(event_call, tp->args[i].type->fmttype, 1481 tp->args[i].name, 1482 sizeof(field) + tp->args[i].offset, 1483 tp->args[i].type->size, 1484 tp->args[i].type->is_signed, 1485 FILTER_OTHER); 1486 if (ret) 1487 return ret; 1488 } 1489 return 0; 1490 } 1491 1492 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len) 1493 { 1494 int i; 1495 int pos = 0; 1496 1497 const char *fmt, *arg; 1498 1499 if (!probe_is_return(tp)) { 1500 fmt = "(%lx)"; 1501 arg = "REC->" FIELD_STRING_IP; 1502 } else { 1503 fmt = "(%lx <- %lx)"; 1504 arg = "REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP; 1505 } 1506 1507 /* When len=0, we just calculate the needed length */ 1508 #define LEN_OR_ZERO (len ? len - pos : 0) 1509 1510 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt); 1511 1512 for (i = 0; i < tp->nr_args; i++) { 1513 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=%s", 1514 tp->args[i].name, tp->args[i].type->fmt); 1515 } 1516 1517 pos += snprintf(buf + pos, LEN_OR_ZERO, "\", %s", arg); 1518 1519 for (i = 0; i < tp->nr_args; i++) { 1520 if (strcmp(tp->args[i].type->name, "string") == 0) 1521 pos += snprintf(buf + pos, LEN_OR_ZERO, 1522 ", __get_str(%s)", 1523 tp->args[i].name); 1524 else 1525 pos += snprintf(buf + pos, LEN_OR_ZERO, ", REC->%s", 1526 tp->args[i].name); 1527 } 1528 1529 #undef LEN_OR_ZERO 1530 1531 /* return the length of print_fmt */ 1532 return pos; 1533 } 1534 1535 static int set_print_fmt(struct trace_probe *tp) 1536 { 1537 int len; 1538 char *print_fmt; 1539 1540 /* First: called with 0 length to calculate the needed length */ 1541 len = __set_print_fmt(tp, NULL, 0); 1542 print_fmt = kmalloc(len + 1, GFP_KERNEL); 1543 if (!print_fmt) 1544 return -ENOMEM; 1545 1546 /* Second: actually write the @print_fmt */ 1547 __set_print_fmt(tp, print_fmt, len + 1); 1548 tp->call.print_fmt = print_fmt; 1549 1550 return 0; 1551 } 1552 1553 #ifdef CONFIG_PERF_EVENTS 1554 1555 /* Kprobe profile handler */ 1556 static __kprobes void kprobe_perf_func(struct kprobe *kp, 1557 struct pt_regs *regs) 1558 { 1559 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); 1560 struct ftrace_event_call *call = &tp->call; 1561 struct kprobe_trace_entry_head *entry; 1562 struct hlist_head *head; 1563 int size, __size, dsize; 1564 int rctx; 1565 1566 dsize = __get_data_size(tp, regs); 1567 __size = sizeof(*entry) + tp->size + dsize; 1568 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1569 size -= sizeof(u32); 1570 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, 1571 "profile buffer not large enough")) 1572 return; 1573 1574 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx); 1575 if (!entry) 1576 return; 1577 1578 entry->ip = (unsigned long)kp->addr; 1579 memset(&entry[1], 0, dsize); 1580 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); 1581 1582 head = this_cpu_ptr(call->perf_events); 1583 perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head); 1584 } 1585 1586 /* Kretprobe profile handler */ 1587 static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, 1588 struct pt_regs *regs) 1589 { 1590 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); 1591 struct ftrace_event_call *call = &tp->call; 1592 struct kretprobe_trace_entry_head *entry; 1593 struct hlist_head *head; 1594 int size, __size, dsize; 1595 int rctx; 1596 1597 dsize = __get_data_size(tp, regs); 1598 __size = sizeof(*entry) + tp->size + dsize; 1599 size = ALIGN(__size + sizeof(u32), sizeof(u64)); 1600 size -= sizeof(u32); 1601 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, 1602 "profile buffer not large enough")) 1603 return; 1604 1605 entry = perf_trace_buf_prepare(size, call->event.type, regs, &rctx); 1606 if (!entry) 1607 return; 1608 1609 entry->func = (unsigned long)tp->rp.kp.addr; 1610 entry->ret_ip = (unsigned long)ri->ret_addr; 1611 store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); 1612 1613 head = this_cpu_ptr(call->perf_events); 1614 perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head); 1615 } 1616 1617 static int probe_perf_enable(struct ftrace_event_call *call) 1618 { 1619 struct trace_probe *tp = (struct trace_probe *)call->data; 1620 1621 tp->flags |= TP_FLAG_PROFILE; 1622 1623 if (probe_is_return(tp)) 1624 return enable_kretprobe(&tp->rp); 1625 else 1626 return enable_kprobe(&tp->rp.kp); 1627 } 1628 1629 static void probe_perf_disable(struct ftrace_event_call *call) 1630 { 1631 struct trace_probe *tp = (struct trace_probe *)call->data; 1632 1633 tp->flags &= ~TP_FLAG_PROFILE; 1634 1635 if (!(tp->flags & TP_FLAG_TRACE)) { 1636 if (probe_is_return(tp)) 1637 disable_kretprobe(&tp->rp); 1638 else 1639 disable_kprobe(&tp->rp.kp); 1640 } 1641 } 1642 #endif /* CONFIG_PERF_EVENTS */ 1643 1644 static __kprobes 1645 int kprobe_register(struct ftrace_event_call *event, enum trace_reg type) 1646 { 1647 switch (type) { 1648 case TRACE_REG_REGISTER: 1649 return probe_event_enable(event); 1650 case TRACE_REG_UNREGISTER: 1651 probe_event_disable(event); 1652 return 0; 1653 1654 #ifdef CONFIG_PERF_EVENTS 1655 case TRACE_REG_PERF_REGISTER: 1656 return probe_perf_enable(event); 1657 case TRACE_REG_PERF_UNREGISTER: 1658 probe_perf_disable(event); 1659 return 0; 1660 #endif 1661 } 1662 return 0; 1663 } 1664 1665 static __kprobes 1666 int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) 1667 { 1668 struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); 1669 1670 if (tp->flags & TP_FLAG_TRACE) 1671 kprobe_trace_func(kp, regs); 1672 #ifdef CONFIG_PERF_EVENTS 1673 if (tp->flags & TP_FLAG_PROFILE) 1674 kprobe_perf_func(kp, regs); 1675 #endif 1676 return 0; /* We don't tweek kernel, so just return 0 */ 1677 } 1678 1679 static __kprobes 1680 int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) 1681 { 1682 struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); 1683 1684 if (tp->flags & TP_FLAG_TRACE) 1685 kretprobe_trace_func(ri, regs); 1686 #ifdef CONFIG_PERF_EVENTS 1687 if (tp->flags & TP_FLAG_PROFILE) 1688 kretprobe_perf_func(ri, regs); 1689 #endif 1690 return 0; /* We don't tweek kernel, so just return 0 */ 1691 } 1692 1693 static struct trace_event_functions kretprobe_funcs = { 1694 .trace = print_kretprobe_event 1695 }; 1696 1697 static struct trace_event_functions kprobe_funcs = { 1698 .trace = print_kprobe_event 1699 }; 1700 1701 static int register_probe_event(struct trace_probe *tp) 1702 { 1703 struct ftrace_event_call *call = &tp->call; 1704 int ret; 1705 1706 /* Initialize ftrace_event_call */ 1707 INIT_LIST_HEAD(&call->class->fields); 1708 if (probe_is_return(tp)) { 1709 call->event.funcs = &kretprobe_funcs; 1710 call->class->define_fields = kretprobe_event_define_fields; 1711 } else { 1712 call->event.funcs = &kprobe_funcs; 1713 call->class->define_fields = kprobe_event_define_fields; 1714 } 1715 if (set_print_fmt(tp) < 0) 1716 return -ENOMEM; 1717 ret = register_ftrace_event(&call->event); 1718 if (!ret) { 1719 kfree(call->print_fmt); 1720 return -ENODEV; 1721 } 1722 call->flags = 0; 1723 call->class->reg = kprobe_register; 1724 call->data = tp; 1725 ret = trace_add_event_call(call); 1726 if (ret) { 1727 pr_info("Failed to register kprobe event: %s\n", call->name); 1728 kfree(call->print_fmt); 1729 unregister_ftrace_event(&call->event); 1730 } 1731 return ret; 1732 } 1733 1734 static void unregister_probe_event(struct trace_probe *tp) 1735 { 1736 /* tp->event is unregistered in trace_remove_event_call() */ 1737 trace_remove_event_call(&tp->call); 1738 kfree(tp->call.print_fmt); 1739 } 1740 1741 /* Make a debugfs interface for controling probe points */ 1742 static __init int init_kprobe_trace(void) 1743 { 1744 struct dentry *d_tracer; 1745 struct dentry *entry; 1746 1747 d_tracer = tracing_init_dentry(); 1748 if (!d_tracer) 1749 return 0; 1750 1751 entry = debugfs_create_file("kprobe_events", 0644, d_tracer, 1752 NULL, &kprobe_events_ops); 1753 1754 /* Event list interface */ 1755 if (!entry) 1756 pr_warning("Could not create debugfs " 1757 "'kprobe_events' entry\n"); 1758 1759 /* Profile interface */ 1760 entry = debugfs_create_file("kprobe_profile", 0444, d_tracer, 1761 NULL, &kprobe_profile_ops); 1762 1763 if (!entry) 1764 pr_warning("Could not create debugfs " 1765 "'kprobe_profile' entry\n"); 1766 return 0; 1767 } 1768 fs_initcall(init_kprobe_trace); 1769 1770 1771 #ifdef CONFIG_FTRACE_STARTUP_TEST 1772 1773 static int kprobe_trace_selftest_target(int a1, int a2, int a3, 1774 int a4, int a5, int a6) 1775 { 1776 return a1 + a2 + a3 + a4 + a5 + a6; 1777 } 1778 1779 static __init int kprobe_trace_self_tests_init(void) 1780 { 1781 int ret, warn = 0; 1782 int (*target)(int, int, int, int, int, int); 1783 struct trace_probe *tp; 1784 1785 target = kprobe_trace_selftest_target; 1786 1787 pr_info("Testing kprobe tracing: "); 1788 1789 ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target " 1790 "$stack $stack0 +0($stack)"); 1791 if (WARN_ON_ONCE(ret)) { 1792 pr_warning("error on probing function entry.\n"); 1793 warn++; 1794 } else { 1795 /* Enable trace point */ 1796 tp = find_probe_event("testprobe", KPROBE_EVENT_SYSTEM); 1797 if (WARN_ON_ONCE(tp == NULL)) { 1798 pr_warning("error on getting new probe.\n"); 1799 warn++; 1800 } else 1801 probe_event_enable(&tp->call); 1802 } 1803 1804 ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target " 1805 "$retval"); 1806 if (WARN_ON_ONCE(ret)) { 1807 pr_warning("error on probing function return.\n"); 1808 warn++; 1809 } else { 1810 /* Enable trace point */ 1811 tp = find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM); 1812 if (WARN_ON_ONCE(tp == NULL)) { 1813 pr_warning("error on getting new probe.\n"); 1814 warn++; 1815 } else 1816 probe_event_enable(&tp->call); 1817 } 1818 1819 if (warn) 1820 goto end; 1821 1822 ret = target(1, 2, 3, 4, 5, 6); 1823 1824 ret = command_trace_probe("-:testprobe"); 1825 if (WARN_ON_ONCE(ret)) { 1826 pr_warning("error on deleting a probe.\n"); 1827 warn++; 1828 } 1829 1830 ret = command_trace_probe("-:testprobe2"); 1831 if (WARN_ON_ONCE(ret)) { 1832 pr_warning("error on deleting a probe.\n"); 1833 warn++; 1834 } 1835 1836 end: 1837 cleanup_all_probes(); 1838 if (warn) 1839 pr_cont("NG: Some tests are failed. Please check them.\n"); 1840 else 1841 pr_cont("OK\n"); 1842 return 0; 1843 } 1844 1845 late_initcall(kprobe_trace_self_tests_init); 1846 1847 #endif 1848