1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Common code for probe-based Dynamic events. 4 * 5 * This code was copied from kernel/trace/trace_kprobe.c written by 6 * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com> 7 * 8 * Updates to make this generic: 9 * Copyright (C) IBM Corporation, 2010-2011 10 * Author: Srikar Dronamraju 11 */ 12 #define pr_fmt(fmt) "trace_probe: " fmt 13 14 #include <linux/bpf.h> 15 #include <linux/fs.h> 16 17 #include "trace_btf.h" 18 #include "trace_probe.h" 19 20 #undef C 21 #define C(a, b) b 22 23 static const char *trace_probe_err_text[] = { ERRORS }; 24 25 static const char *reserved_field_names[] = { 26 "common_type", 27 "common_flags", 28 "common_preempt_count", 29 "common_pid", 30 "common_tgid", 31 FIELD_STRING_IP, 32 FIELD_STRING_RETIP, 33 FIELD_STRING_FUNC, 34 }; 35 36 /* Printing in basic type function template */ 37 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \ 38 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\ 39 { \ 40 trace_seq_printf(s, fmt, *(type *)data); \ 41 return !trace_seq_has_overflowed(s); \ 42 } \ 43 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt; 44 45 DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u") 46 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u") 47 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u") 48 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu") 49 DEFINE_BASIC_PRINT_TYPE_FUNC(s8, s8, "%d") 50 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d") 51 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d") 52 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld") 53 DEFINE_BASIC_PRINT_TYPE_FUNC(x8, u8, "0x%x") 54 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x") 55 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x") 56 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx") 57 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'") 58 59 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent) 60 { 61 trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data); 62 return !trace_seq_has_overflowed(s); 63 } 64 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS"; 65 66 /* Print type function for string type */ 67 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent) 68 { 69 int len = *(u32 *)data >> 16; 70 71 if (!len) 72 trace_seq_puts(s, FAULT_STRING); 73 else 74 trace_seq_printf(s, "\"%s\"", 75 (const char *)get_loc_data(data, ent)); 76 return !trace_seq_has_overflowed(s); 77 } 78 79 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\""; 80 81 /* Fetch type information table */ 82 static const struct fetch_type probe_fetch_types[] = { 83 /* Special types */ 84 __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1, 85 "__data_loc char[]"), 86 __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1, 87 "__data_loc char[]"), 88 __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1, 89 "__data_loc char[]"), 90 /* Basic types */ 91 ASSIGN_FETCH_TYPE(u8, u8, 0), 92 ASSIGN_FETCH_TYPE(u16, u16, 0), 93 ASSIGN_FETCH_TYPE(u32, u32, 0), 94 ASSIGN_FETCH_TYPE(u64, u64, 0), 95 ASSIGN_FETCH_TYPE(s8, u8, 1), 96 ASSIGN_FETCH_TYPE(s16, u16, 1), 97 ASSIGN_FETCH_TYPE(s32, u32, 1), 98 ASSIGN_FETCH_TYPE(s64, u64, 1), 99 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0), 100 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0), 101 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0), 102 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0), 103 ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8, 0), 104 ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0), 105 106 ASSIGN_FETCH_TYPE_END 107 }; 108 109 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags) 110 { 111 int i; 112 113 /* Reject the symbol/symstr for uprobes */ 114 if (type && (flags & TPARG_FL_USER) && 115 (!strcmp(type, "symbol") || !strcmp(type, "symstr"))) 116 return NULL; 117 118 if (!type) 119 type = DEFAULT_FETCH_TYPE_STR; 120 121 /* Special case: bitfield */ 122 if (*type == 'b') { 123 unsigned long bs; 124 125 type = strchr(type, '/'); 126 if (!type) 127 goto fail; 128 129 type++; 130 if (kstrtoul(type, 0, &bs)) 131 goto fail; 132 133 switch (bs) { 134 case 8: 135 return find_fetch_type("u8", flags); 136 case 16: 137 return find_fetch_type("u16", flags); 138 case 32: 139 return find_fetch_type("u32", flags); 140 case 64: 141 return find_fetch_type("u64", flags); 142 default: 143 goto fail; 144 } 145 } 146 147 for (i = 0; probe_fetch_types[i].name; i++) { 148 if (strcmp(type, probe_fetch_types[i].name) == 0) 149 return &probe_fetch_types[i]; 150 } 151 152 fail: 153 return NULL; 154 } 155 156 static struct trace_probe_log trace_probe_log; 157 extern struct mutex dyn_event_ops_mutex; 158 159 const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv) 160 { 161 lockdep_assert_held(&dyn_event_ops_mutex); 162 163 trace_probe_log.subsystem = subsystem; 164 trace_probe_log.argc = argc; 165 trace_probe_log.argv = argv; 166 trace_probe_log.index = 0; 167 return subsystem; 168 } 169 170 void trace_probe_log_clear(void) 171 { 172 lockdep_assert_held(&dyn_event_ops_mutex); 173 174 memset(&trace_probe_log, 0, sizeof(trace_probe_log)); 175 } 176 177 void trace_probe_log_set_index(int index) 178 { 179 lockdep_assert_held(&dyn_event_ops_mutex); 180 181 trace_probe_log.index = index; 182 } 183 184 void __trace_probe_log_err(int offset, int err_type) 185 { 186 char *command, *p; 187 int i, len = 0, pos = 0; 188 189 lockdep_assert_held(&dyn_event_ops_mutex); 190 191 if (!trace_probe_log.argv || !trace_probe_log.argc) 192 return; 193 194 /* Recalculate the length and allocate buffer */ 195 for (i = 0; i < trace_probe_log.argc; i++) { 196 if (i == trace_probe_log.index) 197 pos = len; 198 len += strlen(trace_probe_log.argv[i]) + 1; 199 } 200 command = kzalloc(len, GFP_KERNEL); 201 if (!command) 202 return; 203 204 if (trace_probe_log.index >= trace_probe_log.argc) { 205 /** 206 * Set the error position is next to the last arg + space. 207 * Note that len includes the terminal null and the cursor 208 * appears at pos + 1. 209 */ 210 pos = len; 211 offset = 0; 212 } 213 214 /* And make a command string from argv array */ 215 p = command; 216 for (i = 0; i < trace_probe_log.argc; i++) { 217 len = strlen(trace_probe_log.argv[i]); 218 memcpy(p, trace_probe_log.argv[i], len); 219 p[len] = ' '; 220 p += len + 1; 221 } 222 *(p - 1) = '\0'; 223 224 tracing_log_err(NULL, trace_probe_log.subsystem, command, 225 trace_probe_err_text, err_type, pos + offset); 226 227 kfree(command); 228 } 229 230 /* Split symbol and offset. */ 231 int traceprobe_split_symbol_offset(char *symbol, long *offset) 232 { 233 char *tmp; 234 int ret; 235 236 if (!offset) 237 return -EINVAL; 238 239 tmp = strpbrk(symbol, "+-"); 240 if (tmp) { 241 ret = kstrtol(tmp, 0, offset); 242 if (ret) 243 return ret; 244 *tmp = '\0'; 245 } else 246 *offset = 0; 247 248 return 0; 249 } 250 251 /** 252 * traceprobe_parse_event_name() - Parse a string into group and event names 253 * @pevent: A pointer to the string to be parsed. 254 * @pgroup: A pointer to the group name. 255 * @buf: A buffer to store the parsed group name. 256 * @offset: The offset of the string in the original user command, for logging. 257 * 258 * This parses a string with the format `[GROUP/][EVENT]` or `[GROUP.][EVENT]` 259 * (either GROUP or EVENT or both must be specified). 260 * Since the parsed group name is stored in @buf, the caller must ensure @buf 261 * is at least MAX_EVENT_NAME_LEN bytes. 262 * 263 * Return: 0 on success, or -EINVAL on failure. 264 * 265 * If success, *@pevent is updated to point to the event name part of the 266 * original string, or NULL if there is no event name. 267 * Also, *@pgroup is updated to point to the parsed group which is stored 268 * in @buf, or NULL if there is no group name. 269 */ 270 int traceprobe_parse_event_name(const char **pevent, const char **pgroup, 271 char *buf, int offset) 272 { 273 const char *slash, *event = *pevent; 274 int len; 275 276 slash = strchr(event, '/'); 277 if (!slash) 278 slash = strchr(event, '.'); 279 280 if (slash) { 281 if (slash == event) { 282 trace_probe_log_err(offset, NO_GROUP_NAME); 283 return -EINVAL; 284 } 285 if (slash - event + 1 > MAX_EVENT_NAME_LEN) { 286 trace_probe_log_err(offset, GROUP_TOO_LONG); 287 return -EINVAL; 288 } 289 strscpy(buf, event, slash - event + 1); 290 if (!is_good_system_name(buf)) { 291 trace_probe_log_err(offset, BAD_GROUP_NAME); 292 return -EINVAL; 293 } 294 *pgroup = buf; 295 *pevent = slash + 1; 296 offset += slash - event + 1; 297 event = *pevent; 298 } 299 len = strlen(event); 300 if (len == 0) { 301 if (slash) { 302 *pevent = NULL; 303 return 0; 304 } 305 trace_probe_log_err(offset, NO_EVENT_NAME); 306 return -EINVAL; 307 } else if (len >= MAX_EVENT_NAME_LEN) { 308 trace_probe_log_err(offset, EVENT_TOO_LONG); 309 return -EINVAL; 310 } 311 if (!is_good_name(event)) { 312 trace_probe_log_err(offset, BAD_EVENT_NAME); 313 return -EINVAL; 314 } 315 return 0; 316 } 317 318 static int parse_trace_event_arg(char *arg, struct fetch_insn *code, 319 struct traceprobe_parse_context *ctx) 320 { 321 struct ftrace_event_field *field; 322 struct list_head *head; 323 324 head = trace_get_fields(ctx->event); 325 list_for_each_entry(field, head, link) { 326 if (!strcmp(arg, field->name)) { 327 code->op = FETCH_OP_TP_ARG; 328 code->data = field; 329 return 0; 330 } 331 } 332 return -ENOENT; 333 } 334 335 static int parse_trace_event(char *arg, struct fetch_insn *code, 336 struct traceprobe_parse_context *ctx) 337 { 338 int ret; 339 340 if (code->data) 341 return -EFAULT; 342 ret = parse_trace_event_arg(arg, code, ctx); 343 if (!ret) 344 return 0; 345 return -EINVAL; 346 } 347 348 /* this_cpu_* parser */ 349 #define THIS_CPU_PTR_PREFIX "this_cpu_ptr(" 350 #define THIS_CPU_READ_PREFIX "this_cpu_read(" 351 #define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1) 352 #define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1) 353 354 static int 355 parse_probe_arg(char *arg, const struct fetch_type *type, 356 struct fetch_insn **pcode, struct fetch_insn *end, 357 struct traceprobe_parse_context *ctx); 358 359 static int parse_this_cpu(char *arg, struct traceprobe_parse_context *ctx) 360 { 361 bool is_read = false; 362 char *tmp; 363 364 /* 365 * This is only for kernel probes, excluding eprobe, because per-cpu 366 * pointer should not be recorded by events. 367 */ 368 if (!(ctx->flags & TPARG_FL_KERNEL) || 369 (ctx->flags & TPARG_FL_TEVENT)) { 370 trace_probe_log_err(ctx->offset, NOSUP_PERCPU); 371 return -EINVAL; 372 } 373 if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) { 374 arg += THIS_CPU_PTR_LEN; 375 ctx->offset += THIS_CPU_PTR_LEN; 376 } else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) { 377 arg += THIS_CPU_READ_LEN; 378 ctx->offset += THIS_CPU_READ_LEN; 379 is_read = true; 380 } else { 381 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG); 382 return -EINVAL; 383 } 384 385 tmp = strrchr(arg, ')'); 386 if (!tmp) { 387 trace_probe_log_err(ctx->offset + strlen(arg), 388 DEREF_OPEN_BRACE); 389 return -EINVAL; 390 } 391 *tmp = '\0'; 392 393 ctx->stack[ctx->depth].type = STATE_DEREF; 394 ctx->stack[ctx->depth].deref.deref = FETCH_OP_CPU_PTR; 395 ctx->stack[ctx->depth].deref.offset = 0; 396 ctx->stack[ctx->depth].deref.cur_offs = ctx->offset; 397 ctx->stack[ctx->depth].deref.inner_arg = arg; 398 ctx->stack[ctx->depth].deref.is_cpu_read = is_read; 399 return 0; 400 } 401 402 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS 403 404 static u32 btf_type_int(const struct btf_type *t) 405 { 406 return *(u32 *)(t + 1); 407 } 408 409 static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type) 410 { 411 const struct btf_type *real_type; 412 u32 intdata; 413 414 if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) 415 return false; 416 417 real_type = btf_type_skip_modifiers(btf, type->type, NULL); 418 if (!real_type) 419 return false; 420 421 if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT) 422 return false; 423 424 intdata = btf_type_int(real_type); 425 return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) 426 && BTF_INT_BITS(intdata) == 8; 427 } 428 429 static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) 430 { 431 const struct btf_type *real_type; 432 const struct btf_array *array; 433 u32 intdata; 434 435 if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY) 436 return false; 437 438 array = (const struct btf_array *)(type + 1); 439 440 real_type = btf_type_skip_modifiers(btf, array->type, NULL); 441 if (!real_type) 442 return false; 443 444 if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT) 445 return false; 446 447 intdata = btf_type_int(real_type); 448 return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) 449 && BTF_INT_BITS(intdata) == 8; 450 } 451 452 static struct btf *ctx_btf(struct traceprobe_parse_context *ctx) 453 { 454 return ctx->struct_btf ? : ctx->btf; 455 } 456 457 static int check_prepare_btf_string_fetch(char *typename, 458 struct fetch_insn **pcode, 459 struct traceprobe_parse_context *ctx) 460 { 461 struct btf *btf = ctx_btf(ctx); 462 463 if (!btf || !ctx->last_type) 464 return 0; 465 466 /* char [] does not need any change. */ 467 if (btf_type_is_char_array(btf, ctx->last_type)) 468 return 0; 469 470 /* char * requires dereference the pointer. */ 471 if (btf_type_is_char_ptr(btf, ctx->last_type)) { 472 struct fetch_insn *code = *pcode + 1; 473 474 if (code->op == FETCH_OP_END) { 475 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 476 return -E2BIG; 477 } 478 if (typename[0] == 'u') 479 code->op = FETCH_OP_UDEREF; 480 else 481 code->op = FETCH_OP_DEREF; 482 code->offset = 0; 483 *pcode = code; 484 return 0; 485 } 486 /* Other types are not available for string */ 487 trace_probe_log_err(ctx->offset, BAD_TYPE4STR); 488 return -EINVAL; 489 } 490 491 static const char *fetch_type_from_btf_type(struct btf *btf, 492 const struct btf_type *type, 493 struct traceprobe_parse_context *ctx) 494 { 495 u32 intdata; 496 497 if (ctx->last_bitsize) { 498 u32 bits = ctx->last_bitsize + ctx->last_bitoffs; 499 500 if (bits <= 8) 501 return "u8"; 502 if (bits <= 16) 503 return "u16"; 504 if (bits <= 32) 505 return "u32"; 506 return "u64"; 507 } 508 509 /* TODO: const char * could be converted as a string */ 510 switch (BTF_INFO_KIND(type->info)) { 511 case BTF_KIND_ENUM: 512 /* enum is "int", so convert to "s32" */ 513 return "s32"; 514 case BTF_KIND_ENUM64: 515 return "s64"; 516 case BTF_KIND_PTR: 517 /* pointer will be converted to "x??" */ 518 return IS_ENABLED(CONFIG_64BIT) ? "x64" : "x32"; 519 case BTF_KIND_INT: 520 intdata = btf_type_int(type); 521 if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) { 522 switch (BTF_INT_BITS(intdata)) { 523 case 8: 524 return "s8"; 525 case 16: 526 return "s16"; 527 case 32: 528 return "s32"; 529 case 64: 530 return "s64"; 531 } 532 } else { /* unsigned */ 533 switch (BTF_INT_BITS(intdata)) { 534 case 8: 535 return "u8"; 536 case 16: 537 return "u16"; 538 case 32: 539 return "u32"; 540 case 64: 541 return "u64"; 542 } 543 } 544 } 545 /* TODO: support other types */ 546 547 return NULL; 548 } 549 550 static int query_btf_context(struct traceprobe_parse_context *ctx) 551 { 552 const struct btf_param *param; 553 const struct btf_type *type; 554 struct btf *btf; 555 s32 nr; 556 557 if (ctx->btf) 558 return 0; 559 560 if (!ctx->funcname) 561 return -EINVAL; 562 563 type = btf_find_func_proto(ctx->funcname, &btf); 564 if (!type) 565 return -ENOENT; 566 567 ctx->btf = btf; 568 ctx->proto = type; 569 570 /* ctx->params is optional, since func(void) will not have params. */ 571 nr = 0; 572 param = btf_get_func_param(type, &nr); 573 if (!IS_ERR_OR_NULL(param)) { 574 /* Hide the first 'data' argument of tracepoint */ 575 if (ctx->flags & TPARG_FL_TPOINT) { 576 nr--; 577 param++; 578 } 579 } 580 581 if (nr > 0) { 582 ctx->nr_params = nr; 583 ctx->params = param; 584 } else { 585 ctx->nr_params = 0; 586 ctx->params = NULL; 587 } 588 589 return 0; 590 } 591 592 static void clear_struct_btf(struct traceprobe_parse_context *ctx) 593 { 594 if (ctx->struct_btf) { 595 btf_put(ctx->struct_btf); 596 ctx->struct_btf = NULL; 597 ctx->last_struct = NULL; 598 } 599 } 600 601 static void clear_btf_context(struct traceprobe_parse_context *ctx) 602 { 603 if (ctx->btf) { 604 btf_put(ctx->btf); 605 ctx->btf = NULL; 606 ctx->proto = NULL; 607 ctx->params = NULL; 608 ctx->nr_params = 0; 609 } 610 } 611 612 /* Return 1 if the field separator is arrow operator ('->') */ 613 static int split_next_field(char *varname, char **next_field, 614 struct traceprobe_parse_context *ctx) 615 { 616 char *field; 617 int ret = 0; 618 619 field = strpbrk(varname, ".-"); 620 if (field) { 621 if (field[0] == '-' && field[1] == '>') { 622 field[0] = '\0'; 623 field += 2; 624 ret = 1; 625 } else if (field[0] == '.') { 626 field[0] = '\0'; 627 field += 1; 628 } else { 629 trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN); 630 return -EINVAL; 631 } 632 *next_field = field; 633 } 634 635 return ret; 636 } 637 638 /* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */ 639 static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype, 640 struct traceprobe_parse_context *ctx) 641 { 642 const struct btf_type *type = *ptype; 643 const struct btf_member *field; 644 const struct btf_type *container_type; 645 struct btf *btf = ctx_btf(ctx); 646 char *fieldname = *pfieldname; 647 int bitoffs = 0; 648 u32 anon_offs; 649 char *next; 650 int is_ptr; 651 652 do { 653 next = NULL; 654 is_ptr = split_next_field(fieldname, &next, ctx); 655 if (is_ptr < 0) 656 return is_ptr; 657 658 anon_offs = 0; 659 field = btf_find_struct_member(btf, type, fieldname, 660 &anon_offs, &container_type); 661 if (IS_ERR(field)) { 662 trace_probe_log_err(ctx->offset, BAD_BTF_TID); 663 return PTR_ERR(field); 664 } 665 if (!field) { 666 trace_probe_log_err(ctx->offset, NO_BTF_FIELD); 667 return -ENOENT; 668 } 669 /* Add anonymous structure/union offset */ 670 bitoffs += anon_offs; 671 672 type = btf_type_skip_modifiers(btf, field->type, NULL); 673 if (!type) { 674 trace_probe_log_err(ctx->offset, BAD_BTF_TID); 675 return -EINVAL; 676 } 677 678 /* Accumulate the bit-offsets of the dot-connected fields */ 679 if (btf_type_kflag(container_type)) { 680 bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); 681 ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); 682 } else { 683 bitoffs += field->offset; 684 ctx->last_bitsize = 0; 685 if (BTF_INFO_KIND(type->info) == BTF_KIND_INT) { 686 u32 intdata = btf_type_int(type); 687 688 bitoffs += BTF_INT_OFFSET(intdata); 689 if ((bitoffs % 8) || 690 BTF_INT_BITS(intdata) != BYTES_TO_BITS(type->size) || 691 !is_power_of_2(type->size)) 692 ctx->last_bitsize = BTF_INT_BITS(intdata); 693 } 694 } 695 696 if (next) 697 ctx->offset += next - fieldname; 698 fieldname = next; 699 } while (!is_ptr && fieldname); 700 701 *pfieldname = fieldname; 702 *ptype = type; 703 704 return bitoffs; 705 } 706 /* 707 * Parse the field of data structure. The @type must be a pointer type 708 * pointing the target data structure type. 709 */ 710 static int parse_btf_field(char *fieldname, const struct btf_type *type, 711 struct fetch_insn **pcode, struct fetch_insn *end, 712 struct traceprobe_parse_context *ctx) 713 { 714 struct fetch_insn *code = *pcode; 715 struct btf *btf = ctx_btf(ctx); 716 bool is_first_field = true; 717 int bitoffs; 718 719 do { 720 /* For the first field of typecast, @type will be the target structure type. */ 721 if (!(is_first_field && ctx->struct_btf)) { 722 /* Outer loop for solving arrow operator ('->') */ 723 if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) { 724 trace_probe_log_err(ctx->offset, NO_PTR_STRCT); 725 return -EINVAL; 726 } 727 728 /* Convert a struct pointer type to a struct type */ 729 type = btf_type_skip_modifiers(btf, type->type, NULL); 730 if (!type) { 731 trace_probe_log_err(ctx->offset, BAD_BTF_TID); 732 return -EINVAL; 733 } 734 } 735 736 bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx); 737 if (bitoffs < 0) 738 return bitoffs; 739 if (++code == end) { 740 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 741 return -EINVAL; 742 } 743 code->op = FETCH_OP_DEREF; /* TODO: user deref support */ 744 code->offset = bitoffs / 8; 745 if (is_first_field && ctx->struct_btf) { 746 /* The first field can be typecasted with field option. */ 747 code->offset -= ctx->prefix_byteoffs; 748 } 749 *pcode = code; 750 751 ctx->last_bitoffs = bitoffs % 8; 752 ctx->last_type = type; 753 is_first_field = false; 754 } while (fieldname); 755 756 return 0; 757 } 758 759 static int __store_entry_arg(struct trace_probe *tp, int argnum); 760 761 static int parse_btf_arg(char *varname, 762 struct fetch_insn **pcode, struct fetch_insn *end, 763 struct traceprobe_parse_context *ctx) 764 { 765 struct fetch_insn *code = *pcode; 766 const struct btf_param *params; 767 const struct btf_type *type; 768 char *field = NULL; 769 int i, is_ptr, ret; 770 u32 tid; 771 772 /* Note: field is not separated at this point, so check prefix. */ 773 if (!str_has_prefix(varname, "$current") && 774 !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT)) 775 return -EINVAL; 776 777 is_ptr = split_next_field(varname, &field, ctx); 778 if (is_ptr < 0) 779 return is_ptr; 780 if (!is_ptr && field) { 781 /* dot-connected field on an argument is not supported. */ 782 trace_probe_log_err(ctx->offset + field - varname, 783 NOSUP_DAT_ARG); 784 return -EOPNOTSUPP; 785 } 786 787 if (!strcmp(varname, "$current")) { 788 code->op = FETCH_OP_CURRENT; 789 /* If no typecast is specified for $current, use task_struct by default */ 790 ret = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT, &ctx->struct_btf); 791 if (ret < 0) { 792 trace_probe_log_err(ctx->offset, NO_BTF_ENTRY); 793 return -ENOENT; 794 } 795 tid = (u32)ret; 796 type = ctx->last_struct = 797 btf_type_skip_modifiers(ctx->struct_btf, tid, NULL); 798 goto found_type; 799 } 800 801 if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) { 802 code->op = FETCH_OP_RETVAL; 803 /* Check whether the function return type is not void, even with typecast. */ 804 if (query_btf_context(ctx) == 0) { 805 if (ctx->proto->type == 0) { 806 trace_probe_log_err(ctx->offset, NO_RETVAL); 807 return -ENOENT; 808 } 809 tid = ctx->proto->type; 810 goto found; 811 } 812 if (field) { 813 trace_probe_log_err(ctx->offset + field - varname, 814 NO_BTF_ENTRY); 815 return -ENOENT; 816 } 817 return 0; 818 } 819 820 if (!ctx->btf) { 821 ret = query_btf_context(ctx); 822 if (ret < 0 || ctx->nr_params == 0) { 823 trace_probe_log_err(ctx->offset, NO_BTF_ENTRY); 824 return -ENOENT; 825 } 826 } 827 params = ctx->params; 828 829 for (i = 0; i < ctx->nr_params; i++) { 830 const char *name = btf_name_by_offset(ctx->btf, params[i].name_off); 831 832 if (name && !strcmp(name, varname)) { 833 if (tparg_is_function_entry(ctx->flags)) { 834 code->op = FETCH_OP_ARG; 835 if (ctx->flags & TPARG_FL_TPOINT) 836 code->param = i + 1; 837 else 838 code->param = i; 839 } else if (tparg_is_function_return(ctx->flags)) { 840 code->op = FETCH_OP_EDATA; 841 ret = __store_entry_arg(ctx->tp, i); 842 if (ret < 0) { 843 /* internal error */ 844 return ret; 845 } 846 code->offset = ret; 847 } 848 tid = params[i].type; 849 goto found; 850 } 851 } 852 trace_probe_log_err(ctx->offset, NO_BTFARG); 853 return -ENOENT; 854 855 found: 856 type = btf_type_skip_modifiers(ctx->btf, tid, NULL); 857 found_type: 858 if (!type) { 859 trace_probe_log_err(ctx->offset, BAD_BTF_TID); 860 return -EINVAL; 861 } 862 /* Initialize the last type information */ 863 ctx->last_type = type; 864 ctx->last_bitoffs = 0; 865 ctx->last_bitsize = 0; 866 if (field) { 867 ctx->offset += field - varname; 868 return parse_btf_field(field, type, pcode, end, ctx); 869 } 870 return 0; 871 } 872 873 static const struct fetch_type *find_fetch_type_from_btf_type( 874 struct traceprobe_parse_context *ctx) 875 { 876 struct btf *btf = ctx_btf(ctx); 877 const char *typestr = NULL; 878 879 if (btf && ctx->last_type) 880 typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx); 881 882 return find_fetch_type(typestr, ctx->flags); 883 } 884 885 static int parse_btf_bitfield(struct fetch_insn **pcode, 886 const struct fetch_type *t, 887 struct traceprobe_parse_context *ctx) 888 { 889 struct fetch_insn *code = *pcode; 890 891 if ((ctx->last_bitsize == 0 || 892 ctx->last_bitsize == BYTES_TO_BITS(t->size)) && 893 ctx->last_bitoffs == 0) 894 return 0; 895 896 if (ctx->last_bitsize == 0 || 897 BYTES_TO_BITS(t->size) < (ctx->last_bitsize + ctx->last_bitoffs)) { 898 trace_probe_log_err(ctx->offset, BAD_BITFIELD); 899 return -EINVAL; 900 } 901 902 code++; 903 if (code->op != FETCH_OP_NOP) { 904 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 905 return -EINVAL; 906 } 907 *pcode = code; 908 909 code->op = FETCH_OP_MOD_BF; 910 #ifdef __BIG_ENDIAN_BITFIELD 911 code->lshift = ctx->last_bitoffs; 912 #else 913 code->lshift = BYTES_TO_BITS(t->size) - (ctx->last_bitsize + ctx->last_bitoffs); 914 #endif 915 code->rshift = BYTES_TO_BITS(t->size) - ctx->last_bitsize; 916 code->basesize = t->size; 917 return 0; 918 } 919 920 static int query_btf_struct(const char *sname, struct traceprobe_parse_context *ctx) 921 { 922 struct btf *btf = NULL; 923 int id; 924 925 /* A struct_btf should only be used by a single argument */ 926 if (WARN_ON_ONCE(ctx->struct_btf)) { 927 btf_put(ctx->struct_btf); 928 ctx->struct_btf = NULL; 929 } 930 931 id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf); 932 if (id < 0) 933 return id; 934 ctx->struct_btf = btf; 935 ctx->last_struct = btf_type_by_id(ctx->struct_btf, id); 936 return 0; 937 } 938 939 static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx) 940 { 941 char *field; 942 int ret; 943 944 /* Field option - evaluated later. */ 945 field = strchr(casttype, ','); 946 if (field) 947 *field++ = '\0'; 948 949 ret = query_btf_struct(casttype, ctx); 950 if (ret < 0) { 951 trace_probe_log_err(ctx->offset, NO_PTR_STRCT); 952 return -EINVAL; 953 } 954 955 if (field) { 956 struct btf_type *type = (struct btf_type *)ctx->last_struct; 957 958 ctx->offset += field - casttype; 959 ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx); 960 if (ret < 0) 961 return ret; 962 if (ret % 8) { 963 trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED); 964 return -EINVAL; 965 } 966 if (field != NULL) { 967 /* this means @field skips an arrow operator ("->"). */ 968 trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW); 969 return -EINVAL; 970 } 971 ctx->prefix_byteoffs = ret / 8; 972 /* Restore the original struct type (overwritten by get_bitoffset_of_field) */ 973 ctx->last_struct = type; 974 } 975 976 return ret; 977 } 978 979 /* Find the matching closing parenthesis for a given opening parenthesis. */ 980 static char *find_matched_close_paren(char *s) 981 { 982 char *p = s; 983 int count = 0; 984 985 while (*p) { 986 if (*p == '(') 987 count++; 988 else if (*p == ')') { 989 if (--count == 0) 990 return p; 991 } 992 p++; 993 } 994 return NULL; 995 } 996 997 static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx) 998 { 999 int orig_offset = ctx->offset; 1000 char *close; 1001 char *tmp; 1002 char *fieldname; 1003 1004 if (!(tparg_is_event_probe(ctx->flags) || 1005 tparg_is_function_entry(ctx->flags) || 1006 tparg_is_function_return(ctx->flags))) { 1007 trace_probe_log_err(ctx->offset, NOSUP_BTFARG); 1008 return -EOPNOTSUPP; 1009 } 1010 1011 /* 1012 * Always consider the token after typecast as a nested call 1013 * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same. 1014 * VAR is solved in the nested call. 1015 */ 1016 tmp = strchr(arg, ')'); 1017 if (!tmp) { 1018 trace_probe_log_err(ctx->offset + strlen(arg), 1019 DEREF_OPEN_BRACE); 1020 return -EINVAL; 1021 } 1022 *tmp++ = '\0'; 1023 1024 ctx->offset += tmp - arg; 1025 if (*tmp == '(') { 1026 close = find_matched_close_paren(tmp); 1027 1028 if (!close) { 1029 trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE); 1030 return -EINVAL; 1031 } 1032 /* We expect a field access for typecast */ 1033 if (close[1] != '-' || close[2] != '>') { 1034 trace_probe_log_err(ctx->offset + close - tmp + 1, 1035 TYPECAST_REQ_FIELD); 1036 return -EINVAL; 1037 } 1038 /* Skip '(' */ 1039 ctx->offset += 1; 1040 tmp++; 1041 } else if (*tmp == '+' || *tmp == '-' || 1042 str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) || 1043 str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) { 1044 /* Dereference can have another field access inside it. */ 1045 char *open = strchr(tmp + 1, '('); 1046 1047 if (!open) { 1048 trace_probe_log_err(ctx->offset, 1049 DEREF_NEED_BRACE); 1050 return -EINVAL; 1051 } 1052 close = find_matched_close_paren(open); 1053 if (!close) { 1054 trace_probe_log_err(ctx->offset + strlen(tmp), 1055 DEREF_OPEN_BRACE); 1056 return -EINVAL; 1057 } 1058 close++; 1059 /* We expect a field access for typecast */ 1060 if (close[0] != '-' || close[1] != '>') { 1061 trace_probe_log_err(ctx->offset + close - tmp, 1062 TYPECAST_REQ_FIELD); 1063 return -EINVAL; 1064 } 1065 } else { 1066 if (tmp[0] == '@') { 1067 /* @sym+offset is not allowed without parenthesized */ 1068 close = strpbrk(tmp, "+-"); 1069 if (close && isdigit(close[1])) { 1070 trace_probe_log_err(ctx->offset, 1071 TYPECAST_SYM_OFFSET); 1072 return -EINVAL; 1073 } 1074 } 1075 /* Inner variable name */ 1076 close = strchr(tmp, '-'); 1077 if (!close || close[1] != '>') { 1078 trace_probe_log_err(ctx->offset + strlen(tmp), 1079 TYPECAST_REQ_FIELD); 1080 return -EINVAL; 1081 } 1082 } 1083 *close = '\0'; 1084 1085 /* Let fieldname point the field name. */ 1086 if (close[1] == '-') 1087 fieldname = close + 3; /* Skip "->" after closing parenthesis */ 1088 else 1089 fieldname = close + 2; /* Skip ">" after inner variable name */ 1090 1091 ctx->stack[ctx->depth].type = STATE_TYPECAST; 1092 ctx->stack[ctx->depth].typecast.casttype = arg + 1; 1093 ctx->stack[ctx->depth].typecast.fieldname = fieldname; 1094 ctx->stack[ctx->depth].typecast.orig_offset = orig_offset; 1095 ctx->stack[ctx->depth].typecast.field_offset_diff = fieldname - arg; 1096 ctx->stack[ctx->depth].typecast.inner_arg = tmp; 1097 return 0; 1098 } 1099 1100 #else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */ 1101 1102 static void clear_struct_btf(struct traceprobe_parse_context *ctx) 1103 { 1104 ctx->struct_btf = NULL; 1105 } 1106 1107 static void clear_btf_context(struct traceprobe_parse_context *ctx) 1108 { 1109 ctx->btf = NULL; 1110 } 1111 1112 static int query_btf_context(struct traceprobe_parse_context *ctx) 1113 { 1114 return -EOPNOTSUPP; 1115 } 1116 1117 static int parse_btf_arg(char *varname, 1118 struct fetch_insn **pcode, struct fetch_insn *end, 1119 struct traceprobe_parse_context *ctx) 1120 { 1121 trace_probe_log_err(ctx->offset, NOSUP_BTFARG); 1122 return -EOPNOTSUPP; 1123 } 1124 1125 static int parse_btf_bitfield(struct fetch_insn **pcode, 1126 const struct fetch_type *t, 1127 struct traceprobe_parse_context *ctx) 1128 { 1129 trace_probe_log_err(ctx->offset, NOSUP_BTFARG); 1130 return -EOPNOTSUPP; 1131 } 1132 1133 #define find_fetch_type_from_btf_type(ctx) \ 1134 find_fetch_type(NULL, ctx->flags) 1135 1136 static int check_prepare_btf_string_fetch(char *typename, 1137 struct fetch_insn **pcode, 1138 struct traceprobe_parse_context *ctx) 1139 { 1140 return 0; 1141 } 1142 1143 static int parse_btf_casttype(char *casttype, 1144 struct traceprobe_parse_context *ctx) 1145 { 1146 return -EOPNOTSUPP; 1147 } 1148 1149 static int parse_btf_field(char *fieldname, const struct btf_type *type, 1150 struct fetch_insn **pcode, struct fetch_insn *end, 1151 struct traceprobe_parse_context *ctx) 1152 { 1153 return -EOPNOTSUPP; 1154 } 1155 1156 static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx) 1157 { 1158 trace_probe_log_err(ctx->offset, NOSUP_BTFARG); 1159 return -EOPNOTSUPP; 1160 } 1161 1162 #endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */ 1163 1164 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1165 1166 static void store_entry_arg_at(struct fetch_insn *code, int argnum, int offset) 1167 { 1168 code[0].op = FETCH_OP_ARG; 1169 code[0].param = argnum; 1170 code[1].op = FETCH_OP_ST_EDATA; 1171 code[1].offset = offset; 1172 } 1173 1174 static int get_entry_arg_max_offset(struct probe_entry_arg *earg) 1175 { 1176 int i, max_offset = 0; 1177 1178 /* 1179 * earg->code[] array has an operation sequence which is run in 1180 * the entry handler. 1181 * The sequence stopped by FETCH_OP_END and each data stored in 1182 * the entry data buffer by FETCH_OP_ST_EDATA. The FETCH_OP_ST_EDATA 1183 * stores the data at the data buffer + its offset, and all data are 1184 * "unsigned long" size. The offset must be increased when a data is 1185 * stored. Thus we need to find the last FETCH_OP_ST_EDATA in the 1186 * code array. 1187 */ 1188 for (i = 0; i < earg->size - 1 && earg->code[i].op != FETCH_OP_END; i++) { 1189 if (earg->code[i].op == FETCH_OP_ST_EDATA) 1190 if (earg->code[i].offset > max_offset) 1191 max_offset = earg->code[i].offset; 1192 } 1193 return max_offset; 1194 } 1195 1196 /* 1197 * Add the entry code to store the 'argnum'th parameter and return the offset 1198 * in the entry data buffer where the data will be stored. 1199 */ 1200 static int __store_entry_arg(struct trace_probe *tp, int argnum) 1201 { 1202 struct probe_entry_arg *earg = tp->entry_arg; 1203 int i, offset, last_offset = 0; 1204 1205 if (!earg) { 1206 earg = kzalloc_flex(*earg, code, 2 * tp->nr_args + 1); 1207 if (!earg) 1208 return -ENOMEM; 1209 earg->size = 2 * tp->nr_args + 1; 1210 /* Fill the code buffer with 'end' to simplify it */ 1211 for (i = 0; i < earg->size; i++) 1212 earg->code[i].op = FETCH_OP_END; 1213 tp->entry_arg = earg; 1214 store_entry_arg_at(earg->code, argnum, 0); 1215 return 0; 1216 } 1217 1218 /* 1219 * NOTE: if anyone change the following rule, please rewrite this. 1220 * The entry code array is filled with the pair of 1221 * 1222 * [FETCH_OP_ARG(argnum)] 1223 * [FETCH_OP_ST_EDATA(offset of entry data buffer)] 1224 * 1225 * and the rest of entries are filled with [FETCH_OP_END]. 1226 * The offset should be incremented, thus the last pair should 1227 * have the largest offset. 1228 */ 1229 1230 /* Search the offset for the sprcified argnum. */ 1231 for (i = 0; i < earg->size - 1 && earg->code[i].op != FETCH_OP_END; i += 2) { 1232 if (WARN_ON_ONCE(earg->code[i].op != FETCH_OP_ARG)) 1233 return -EINVAL; 1234 1235 if (earg->code[i].param != argnum) 1236 continue; 1237 1238 if (WARN_ON_ONCE(earg->code[i + 1].op != FETCH_OP_ST_EDATA)) 1239 return -EINVAL; 1240 1241 return earg->code[i + 1].offset; 1242 } 1243 /* Not found, append new entry if possible. */ 1244 if (i >= earg->size - 1) 1245 return -ENOSPC; 1246 1247 /* The last entry must have the largest offset. */ 1248 if (i != 0) { 1249 if (WARN_ON_ONCE(earg->code[i - 1].op != FETCH_OP_ST_EDATA)) 1250 return -EINVAL; 1251 last_offset = earg->code[i - 1].offset; 1252 } 1253 1254 offset = last_offset + sizeof(unsigned long); 1255 store_entry_arg_at(&earg->code[i], argnum, offset); 1256 return offset; 1257 } 1258 1259 int traceprobe_get_entry_data_size(struct trace_probe *tp) 1260 { 1261 struct probe_entry_arg *earg = tp->entry_arg; 1262 1263 if (!earg) 1264 return 0; 1265 1266 return get_entry_arg_max_offset(earg) + sizeof(unsigned long); 1267 } 1268 1269 void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs) 1270 { 1271 struct probe_entry_arg *earg = tp->entry_arg; 1272 unsigned long val = 0; 1273 int i; 1274 1275 if (!earg) 1276 return; 1277 1278 for (i = 0; i < earg->size; i++) { 1279 struct fetch_insn *code = &earg->code[i]; 1280 1281 switch (code->op) { 1282 case FETCH_OP_ARG: 1283 val = regs_get_kernel_argument(regs, code->param); 1284 break; 1285 case FETCH_OP_ST_EDATA: 1286 *(unsigned long *)((u8 *)edata + code->offset) = val; 1287 break; 1288 case FETCH_OP_END: 1289 goto end; 1290 default: 1291 break; 1292 } 1293 } 1294 end: 1295 return; 1296 } 1297 NOKPROBE_SYMBOL(store_trace_entry_data) 1298 #endif 1299 1300 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) 1301 1302 static int parse_probe_var_retval(char *orig_arg, 1303 struct fetch_insn **pcode, 1304 struct fetch_insn *end, 1305 struct traceprobe_parse_context *ctx) 1306 { 1307 struct fetch_insn *code = *pcode; 1308 1309 if (!(ctx->flags & TPARG_FL_RETURN)) { 1310 trace_probe_log_err(ctx->offset, RETVAL_ON_PROBE); 1311 return -EINVAL; 1312 } 1313 if (!(ctx->flags & TPARG_FL_KERNEL) || 1314 !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) { 1315 code->op = FETCH_OP_RETVAL; 1316 return 0; 1317 } 1318 return parse_btf_arg(orig_arg, pcode, end, ctx); 1319 } 1320 1321 static int parse_probe_var_stack(char *arg, int len, struct fetch_insn *code, 1322 struct traceprobe_parse_context *ctx) 1323 { 1324 unsigned long param; 1325 int ret; 1326 1327 if (arg[len] == '\0') { 1328 code->op = FETCH_OP_STACKP; 1329 return 0; 1330 } 1331 1332 if (isdigit(arg[len])) { 1333 ret = kstrtoul(arg + len, 10, ¶m); 1334 if (ret) { 1335 trace_probe_log_err(ctx->offset, BAD_VAR); 1336 return ret; 1337 } 1338 1339 if ((ctx->flags & TPARG_FL_KERNEL) && 1340 param > PARAM_MAX_STACK) { 1341 trace_probe_log_err(ctx->offset, BAD_STACK_NUM); 1342 return -EINVAL; 1343 } 1344 code->op = FETCH_OP_STACK; 1345 code->param = (unsigned int)param; 1346 return 0; 1347 } 1348 1349 trace_probe_log_err(ctx->offset, BAD_VAR); 1350 return -EINVAL; 1351 } 1352 1353 static int parse_probe_var_current(char *orig_arg, char *arg, 1354 struct fetch_insn **pcode, 1355 struct fetch_insn *end, 1356 struct traceprobe_parse_context *ctx) 1357 { 1358 struct fetch_insn *code = *pcode; 1359 1360 /* $current is only supported by kernel probe. */ 1361 if (!(ctx->flags & TPARG_FL_KERNEL)) { 1362 trace_probe_log_err(ctx->offset, BAD_VAR); 1363 return -EINVAL; 1364 } 1365 arg += strlen("current"); 1366 if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) 1367 return parse_btf_arg(orig_arg, pcode, end, ctx); 1368 1369 if (*arg != '\0') { 1370 trace_probe_log_err(ctx->offset, BAD_VAR); 1371 return -EINVAL; 1372 } 1373 1374 code->op = FETCH_OP_CURRENT; 1375 return 0; 1376 } 1377 1378 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 1379 static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code, 1380 struct traceprobe_parse_context *ctx) 1381 { 1382 unsigned long param; 1383 int ret; 1384 1385 ret = kstrtoul(arg + len, 10, ¶m); 1386 if (ret) { 1387 trace_probe_log_err(ctx->offset, BAD_VAR); 1388 return ret; 1389 } 1390 1391 if (!param || param > PARAM_MAX_STACK) { 1392 trace_probe_log_err(ctx->offset, BAD_ARG_NUM); 1393 return -EINVAL; 1394 } 1395 param--; /* argN starts from 1, but internal arg[N] starts from 0 */ 1396 1397 if (tparg_is_function_entry(ctx->flags)) { 1398 code->op = FETCH_OP_ARG; 1399 code->param = (unsigned int)param; 1400 /* 1401 * The tracepoint probe will probe a stub function, and the 1402 * first parameter of the stub is a dummy and should be ignored. 1403 */ 1404 if (ctx->flags & TPARG_FL_TPOINT) 1405 code->param++; 1406 } else if (tparg_is_function_return(ctx->flags)) { 1407 /* function entry argument access from return probe */ 1408 ret = __store_entry_arg(ctx->tp, param); 1409 if (ret < 0) /* This error should be an internal error */ 1410 return ret; 1411 1412 code->op = FETCH_OP_EDATA; 1413 code->offset = ret; 1414 } else { 1415 trace_probe_log_err(ctx->offset, NOFENTRY_ARGS); 1416 return -EINVAL; 1417 } 1418 return 0; 1419 } 1420 #else 1421 static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code, 1422 struct traceprobe_parse_context *ctx) 1423 { 1424 trace_probe_log_err(ctx->offset, BAD_VAR); 1425 return -EINVAL; 1426 } 1427 #endif 1428 1429 /* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */ 1430 static int parse_probe_vars(char *orig_arg, const struct fetch_type *t, 1431 struct fetch_insn **pcode, 1432 struct fetch_insn *end, 1433 struct traceprobe_parse_context *ctx) 1434 { 1435 struct fetch_insn *code = *pcode; 1436 char *arg = orig_arg + 1; 1437 int len, ret; 1438 1439 if (ctx->flags & TPARG_FL_TEVENT) { 1440 ret = parse_trace_event(arg, code, ctx); 1441 if (!ret) 1442 return 0; 1443 } 1444 1445 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) { 1446 code->op = FETCH_OP_COMM; 1447 return 0; 1448 } 1449 1450 /* eprobe only support event fields or '$comm'. */ 1451 if (ctx->flags & TPARG_FL_TEVENT) 1452 goto inval; 1453 1454 if (str_has_prefix(arg, "retval")) 1455 return parse_probe_var_retval(orig_arg, pcode, end, ctx); 1456 1457 len = str_has_prefix(arg, "stack"); 1458 if (len) 1459 return parse_probe_var_stack(arg, len, code, ctx); 1460 1461 /* $current returns the address of the current task_struct. */ 1462 if (str_has_prefix(arg, "current")) 1463 return parse_probe_var_current(orig_arg, arg, pcode, end, ctx); 1464 1465 len = str_has_prefix(arg, "arg"); 1466 if (len) 1467 return parse_probe_var_arg(arg, len, code, ctx); 1468 1469 inval: 1470 trace_probe_log_err(ctx->offset, BAD_VAR); 1471 return -EINVAL; 1472 } 1473 1474 static int str_to_immediate(char *str, unsigned long *imm) 1475 { 1476 if (isdigit(str[0])) 1477 return kstrtoul(str, 0, imm); 1478 else if (str[0] == '-') 1479 return kstrtol(str, 0, (long *)imm); 1480 else if (str[0] == '+') 1481 return kstrtol(str + 1, 0, (long *)imm); 1482 return -EINVAL; 1483 } 1484 1485 static int __parse_imm_string(char *str, char **pbuf, int offs) 1486 { 1487 size_t len = strlen(str); 1488 1489 if (!len || str[len - 1] != '"') { 1490 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE); 1491 return -EINVAL; 1492 } 1493 *pbuf = kstrndup(str, len - 1, GFP_KERNEL); 1494 if (!*pbuf) 1495 return -ENOMEM; 1496 return 0; 1497 } 1498 1499 static int parse_probe_arg_register(char *arg, struct fetch_insn *code, 1500 struct traceprobe_parse_context *ctx) 1501 { 1502 int ret; 1503 1504 if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) { 1505 /* eprobe and fprobe do not handle registers */ 1506 trace_probe_log_err(ctx->offset, BAD_VAR); 1507 return -EINVAL; 1508 } 1509 ret = regs_query_register_offset(arg + 1); 1510 if (ret >= 0) { 1511 code->op = FETCH_OP_REG; 1512 code->param = (unsigned int)ret; 1513 return 0; 1514 } 1515 trace_probe_log_err(ctx->offset, BAD_REG_NAME); 1516 return -EINVAL; 1517 } 1518 1519 static int parse_probe_arg_mem_symbol(char *arg, struct fetch_insn **pcode, 1520 struct fetch_insn *end, 1521 struct traceprobe_parse_context *ctx) 1522 { 1523 struct fetch_insn *code = *pcode; 1524 unsigned long param; 1525 long offset = 0; 1526 int ret; 1527 1528 if (isdigit(arg[1])) { 1529 ret = kstrtoul(arg + 1, 0, ¶m); 1530 if (ret) { 1531 trace_probe_log_err(ctx->offset, BAD_MEM_ADDR); 1532 return ret; 1533 } 1534 /* load address */ 1535 code->op = FETCH_OP_IMM; 1536 code->immediate = param; 1537 } else if (arg[1] == '+') { 1538 /* Kernel probes do not support file offsets */ 1539 if (ctx->flags & TPARG_FL_KERNEL) { 1540 trace_probe_log_err(ctx->offset, FILE_ON_KPROBE); 1541 return -EINVAL; 1542 } 1543 ret = kstrtol(arg + 2, 0, &offset); 1544 if (ret) { 1545 trace_probe_log_err(ctx->offset, BAD_FILE_OFFS); 1546 return ret; 1547 } 1548 1549 code->op = FETCH_OP_FOFFS; 1550 code->immediate = (unsigned long)offset; 1551 offset = 0; 1552 } else { 1553 /* uprobes don't support symbols */ 1554 if (!(ctx->flags & TPARG_FL_KERNEL)) { 1555 trace_probe_log_err(ctx->offset, SYM_ON_UPROBE); 1556 return -EINVAL; 1557 } 1558 /* Preserve symbol for updating */ 1559 code->op = FETCH_NOP_SYMBOL; 1560 code->data = kstrdup(arg + 1, GFP_KERNEL); 1561 if (!code->data) 1562 return -ENOMEM; 1563 if (++code == end) { 1564 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 1565 return -EINVAL; 1566 } 1567 code->op = FETCH_OP_IMM; 1568 code->immediate = 0; 1569 } 1570 /* These are fetching from memory */ 1571 if (++code == end) { 1572 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 1573 return -EINVAL; 1574 } 1575 *pcode = code; 1576 code->op = FETCH_OP_DEREF; 1577 code->offset = offset; 1578 return 0; 1579 } 1580 1581 static int parse_probe_arg_deref(char *arg, struct traceprobe_parse_context *ctx) 1582 { 1583 int deref = FETCH_OP_DEREF; 1584 long offset = 0; 1585 char *tmp; 1586 int ret; 1587 1588 if (arg[1] == 'u') { 1589 deref = FETCH_OP_UDEREF; 1590 arg[1] = arg[0]; 1591 arg++; 1592 } 1593 if (arg[0] == '+') 1594 arg++; /* Skip '+', because kstrtol() rejects it. */ 1595 tmp = strchr(arg, '('); 1596 if (!tmp) { 1597 trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE); 1598 return -EINVAL; 1599 } 1600 *tmp = '\0'; 1601 ret = kstrtol(arg, 0, &offset); 1602 if (ret) { 1603 trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS); 1604 return ret; 1605 } 1606 ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0); 1607 arg = tmp + 1; 1608 1609 tmp = strrchr(arg, ')'); 1610 if (!tmp) { 1611 trace_probe_log_err(ctx->offset + strlen(arg), 1612 DEREF_OPEN_BRACE); 1613 return -EINVAL; 1614 } 1615 *tmp = '\0'; 1616 1617 ctx->stack[ctx->depth].type = STATE_DEREF; 1618 ctx->stack[ctx->depth].deref.deref = deref; 1619 ctx->stack[ctx->depth].deref.offset = offset; 1620 ctx->stack[ctx->depth].deref.cur_offs = ctx->offset; 1621 ctx->stack[ctx->depth].deref.inner_arg = arg; 1622 ctx->stack[ctx->depth].deref.is_cpu_read = false; 1623 return 0; 1624 } 1625 1626 static int parse_probe_arg_imm(char *arg, struct fetch_insn *code, 1627 struct traceprobe_parse_context *ctx) 1628 { 1629 char *tmp; 1630 int ret; 1631 1632 if (arg[1] == '"') { /* Immediate string */ 1633 ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2); 1634 if (ret) 1635 return ret; 1636 code->op = FETCH_OP_IMMSTR; 1637 code->data = tmp; 1638 } else { 1639 ret = str_to_immediate(arg + 1, &code->immediate); 1640 if (ret) { 1641 trace_probe_log_err(ctx->offset + 1, BAD_IMM); 1642 return ret; 1643 } 1644 code->op = FETCH_OP_IMM; 1645 } 1646 return 0; 1647 } 1648 1649 static int parse_probe_arg_default(char *arg, struct fetch_insn **pcode, 1650 struct fetch_insn *end, 1651 struct traceprobe_parse_context *ctx) 1652 { 1653 int ret; 1654 1655 if (isalpha(arg[0]) || arg[0] == '_') { 1656 /* BTF variable or event field */ 1657 if (ctx->flags & TPARG_FL_TEVENT) { 1658 ret = parse_trace_event(arg, *pcode, ctx); 1659 if (ret < 0) { 1660 trace_probe_log_err(ctx->offset, NO_EVENT_FIELD); 1661 return -EINVAL; 1662 } 1663 return 0; 1664 } 1665 if (!tparg_is_function_entry(ctx->flags) && 1666 !tparg_is_function_return(ctx->flags)) { 1667 trace_probe_log_err(ctx->offset, NOSUP_BTFARG); 1668 return -EINVAL; 1669 } 1670 return parse_btf_arg(arg, pcode, end, ctx); 1671 } 1672 1673 return 0; 1674 } 1675 1676 static int parse_probe_arg_nested(char **parg, struct traceprobe_parse_context *ctx) 1677 { 1678 char *arg = *parg; 1679 int ret; 1680 1681 while (true) { 1682 /* Determine if this is a nested argument */ 1683 if (arg[0] != '+' && arg[0] != '-' && arg[0] != '(' && 1684 !str_has_prefix(arg, THIS_CPU_PTR_PREFIX) && 1685 !str_has_prefix(arg, THIS_CPU_READ_PREFIX)) 1686 break; 1687 1688 /* If nested, check the maximum depth limit */ 1689 if (ctx->depth >= TRACEPROBE_MAX_NESTED_LEVEL) { 1690 trace_probe_log_err(ctx->offset, TOO_MANY_NESTED); 1691 return -E2BIG; 1692 } 1693 1694 /* Perform the actual parsing subroutine calls */ 1695 switch (arg[0]) { 1696 case '+': 1697 case '-': 1698 ret = parse_probe_arg_deref(arg, ctx); 1699 if (ret) 1700 return ret; 1701 arg = ctx->stack[ctx->depth].deref.inner_arg; 1702 break; 1703 case '(': 1704 ret = handle_typecast(arg, ctx); 1705 if (ret) 1706 return ret; 1707 arg = ctx->stack[ctx->depth].typecast.inner_arg; 1708 break; 1709 default: 1710 ret = parse_this_cpu(arg, ctx); 1711 if (ret) 1712 return ret; 1713 arg = ctx->stack[ctx->depth].deref.inner_arg; 1714 break; 1715 } 1716 ctx->depth++; 1717 } 1718 1719 *parg = arg; 1720 return 0; 1721 } 1722 1723 static int parse_probe_arg_leaf(char *arg, const struct fetch_type *type, 1724 struct fetch_insn **pcode, struct fetch_insn *end, 1725 struct traceprobe_parse_context *ctx) 1726 { 1727 struct fetch_insn *code = *pcode; 1728 int ret; 1729 1730 switch (arg[0]) { 1731 case '$': 1732 ret = parse_probe_vars(arg, type, pcode, end, ctx); 1733 break; 1734 case '%': /* named register */ 1735 ret = parse_probe_arg_register(arg, code, ctx); 1736 break; 1737 case '@': /* memory, file-offset or symbol */ 1738 ret = parse_probe_arg_mem_symbol(arg, pcode, end, ctx); 1739 break; 1740 case '\\': /* Immediate value */ 1741 ret = parse_probe_arg_imm(arg, code, ctx); 1742 break; 1743 default: 1744 ret = parse_probe_arg_default(arg, pcode, end, ctx); 1745 break; 1746 } 1747 1748 if (ret) 1749 return ret; 1750 1751 if (code->op == FETCH_OP_NOP) { 1752 /* Parsed, but do not find fetch method */ 1753 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG); 1754 return -EINVAL; 1755 } 1756 1757 return 0; 1758 } 1759 1760 static int unwind_parse_states(struct fetch_insn **pcode, struct fetch_insn *end, 1761 struct traceprobe_parse_context *ctx) 1762 { 1763 struct parse_state *state; 1764 struct fetch_insn *code; 1765 int ret; 1766 1767 while (ctx->depth > 0) { 1768 ctx->depth--; 1769 state = &ctx->stack[ctx->depth]; 1770 1771 if (state->type == STATE_DEREF) { 1772 code = *pcode; 1773 ctx->offset = state->deref.cur_offs; 1774 if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_IMMSTR) { 1775 trace_probe_log_err(ctx->offset, COMM_CANT_DEREF); 1776 return -EINVAL; 1777 } 1778 1779 if (!(state->deref.deref == FETCH_OP_CPU_PTR && 1780 *state->deref.inner_arg == '@')) { 1781 code++; 1782 if (code == end) { 1783 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 1784 return -EINVAL; 1785 } 1786 } 1787 *pcode = code; 1788 1789 code->op = state->deref.deref; 1790 code->offset = state->deref.offset; 1791 ctx->last_type = NULL; 1792 1793 if (state->deref.is_cpu_read) { 1794 code = *pcode; 1795 code++; 1796 if (code == end) { 1797 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 1798 return -EINVAL; 1799 } 1800 code->op = FETCH_OP_DEREF; 1801 code->offset = 0; 1802 *pcode = code; 1803 } 1804 } else if (state->type == STATE_TYPECAST) { 1805 clear_struct_btf(ctx); 1806 1807 /* resolve the typecast struct name */ 1808 ctx->offset = state->typecast.orig_offset + 1; /* for the '(' */ 1809 ret = parse_btf_casttype(state->typecast.casttype, ctx); 1810 if (ret < 0) 1811 return ret; 1812 1813 ctx->offset = state->typecast.orig_offset + 1814 state->typecast.field_offset_diff; 1815 ret = parse_btf_field(state->typecast.fieldname, 1816 ctx->last_struct, pcode, 1817 end, ctx); 1818 ctx->prefix_byteoffs = 0; 1819 if (ret < 0) 1820 return ret; 1821 } 1822 } 1823 1824 return 0; 1825 } 1826 1827 /* Loop-based (non-recursive) argument parser */ 1828 static int 1829 parse_probe_arg(char *arg, const struct fetch_type *type, 1830 struct fetch_insn **pcode, struct fetch_insn *end, 1831 struct traceprobe_parse_context *ctx) 1832 { 1833 int ret; 1834 1835 ctx->depth = 0; 1836 1837 ret = parse_probe_arg_nested(&arg, ctx); 1838 if (ret) 1839 return ret; 1840 1841 ret = parse_probe_arg_leaf(arg, type, pcode, end, ctx); 1842 if (ret) 1843 return ret; 1844 1845 return unwind_parse_states(pcode, end, ctx); 1846 } 1847 1848 /* Bitfield type needs to be parsed into a fetch function */ 1849 static int __parse_bitfield_probe_arg(const char *bf, 1850 const struct fetch_type *t, 1851 struct fetch_insn **pcode) 1852 { 1853 struct fetch_insn *code = *pcode; 1854 unsigned long bw, bo; 1855 char *tail; 1856 1857 if (*bf != 'b') 1858 return 0; 1859 1860 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */ 1861 1862 if (bw == 0 || *tail != '@') 1863 return -EINVAL; 1864 1865 bf = tail + 1; 1866 bo = simple_strtoul(bf, &tail, 0); 1867 1868 if (tail == bf || *tail != '/') 1869 return -EINVAL; 1870 code++; 1871 if (code->op != FETCH_OP_NOP) 1872 return -EINVAL; 1873 *pcode = code; 1874 1875 code->op = FETCH_OP_MOD_BF; 1876 code->lshift = BYTES_TO_BITS(t->size) - (bw + bo); 1877 code->rshift = BYTES_TO_BITS(t->size) - bw; 1878 code->basesize = t->size; 1879 1880 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0; 1881 } 1882 1883 /* Split type part from @arg and return it. */ 1884 static char *parse_probe_arg_type(char *arg, struct probe_arg *parg, 1885 struct traceprobe_parse_context *ctx) 1886 { 1887 char *t = NULL, *t2, *t3; 1888 int offs; 1889 1890 t = strchr(arg, ':'); 1891 if (t) { 1892 *t++ = '\0'; 1893 t2 = strchr(t, '['); 1894 if (t2) { 1895 *t2++ = '\0'; 1896 t3 = strchr(t2, ']'); 1897 if (!t3) { 1898 offs = t2 + strlen(t2) - arg; 1899 1900 trace_probe_log_err(ctx->offset + offs, 1901 ARRAY_NO_CLOSE); 1902 return ERR_PTR(-EINVAL); 1903 } else if (t3[1] != '\0') { 1904 trace_probe_log_err(ctx->offset + t3 + 1 - arg, 1905 BAD_ARRAY_SUFFIX); 1906 return ERR_PTR(-EINVAL); 1907 } 1908 *t3 = '\0'; 1909 if (kstrtouint(t2, 0, &parg->count) || !parg->count) { 1910 trace_probe_log_err(ctx->offset + t2 - arg, 1911 BAD_ARRAY_NUM); 1912 return ERR_PTR(-EINVAL); 1913 } 1914 if (parg->count > MAX_ARRAY_LEN) { 1915 trace_probe_log_err(ctx->offset + t2 - arg, 1916 ARRAY_TOO_BIG); 1917 return ERR_PTR(-EINVAL); 1918 } 1919 } 1920 } 1921 offs = t ? t - arg : 0; 1922 1923 /* 1924 * Since $comm and immediate string can not be dereferenced, 1925 * we can find those by strcmp. But ignore for eprobes. 1926 */ 1927 if (!(ctx->flags & TPARG_FL_TEVENT) && 1928 (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 || 1929 strncmp(arg, "\\\"", 2) == 0)) { 1930 /* The type of $comm must be "string", and not an array type. */ 1931 if (parg->count || (t && strcmp(t, "string"))) { 1932 trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE); 1933 return ERR_PTR(-EINVAL); 1934 } 1935 parg->type = find_fetch_type("string", ctx->flags); 1936 } else 1937 parg->type = find_fetch_type(t, ctx->flags); 1938 1939 if (!parg->type) { 1940 trace_probe_log_err(ctx->offset + offs, BAD_TYPE); 1941 return ERR_PTR(-EINVAL); 1942 } 1943 1944 return t; 1945 } 1946 1947 /* After parsing, adjust the fetch_insn according to the probe_arg */ 1948 static int finalize_fetch_insn(struct fetch_insn *code, 1949 struct probe_arg *parg, 1950 char *type, 1951 int type_offset, 1952 struct traceprobe_parse_context *ctx) 1953 { 1954 struct fetch_insn *scode; 1955 int ret; 1956 1957 /* Store operation */ 1958 if (parg->type->is_string) { 1959 bool is_uderef; 1960 1961 /* Check bad combination of the type and the last fetch_insn. */ 1962 if (!strcmp(parg->type->name, "symstr")) { 1963 if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK && 1964 code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG && 1965 code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) { 1966 trace_probe_log_err(ctx->offset + type_offset, 1967 BAD_SYMSTRING); 1968 return -EINVAL; 1969 } 1970 } else { 1971 if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF && 1972 code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM && 1973 code->op != FETCH_OP_IMMSTR && code->op != FETCH_OP_TP_ARG) { 1974 trace_probe_log_err(ctx->offset + type_offset, 1975 BAD_STRING); 1976 return -EINVAL; 1977 } 1978 } 1979 1980 is_uderef = (code->op == FETCH_OP_UDEREF); 1981 if (!strcmp(parg->type->name, "symstr") || 1982 (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM || 1983 code->op == FETCH_OP_IMMSTR) || code->op == FETCH_OP_TP_ARG || 1984 parg->count) { 1985 /* 1986 * IMM, DATA and COMM is pointing actual address, those 1987 * must be kept, and if parg->count != 0, this is an 1988 * array of string pointers instead of string address 1989 * itself. 1990 * For the symstr, it doesn't need to dereference, thus 1991 * it just get the value. 1992 */ 1993 if (parg->count && (code->op != FETCH_OP_DEREF && 1994 code->op != FETCH_OP_UDEREF)) { 1995 trace_probe_log_err(ctx->offset + type_offset, 1996 BAD_STRING); 1997 return -EINVAL; 1998 } 1999 code++; 2000 if (code->op != FETCH_OP_NOP) { 2001 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 2002 return -EINVAL; 2003 } 2004 } 2005 2006 /* If op == DEREF, replace it with STRING */ 2007 if (!strcmp(parg->type->name, "ustring") || is_uderef) 2008 code->op = FETCH_OP_ST_USTRING; 2009 else if (!strcmp(parg->type->name, "symstr")) 2010 code->op = FETCH_OP_ST_SYMSTR; 2011 else 2012 code->op = FETCH_OP_ST_STRING; 2013 code->size = parg->type->size; 2014 parg->dynamic = true; 2015 } else if (code->op == FETCH_OP_DEREF) { 2016 code->op = FETCH_OP_ST_MEM; 2017 code->size = parg->type->size; 2018 } else if (code->op == FETCH_OP_UDEREF) { 2019 code->op = FETCH_OP_ST_UMEM; 2020 code->size = parg->type->size; 2021 } else { 2022 code++; 2023 if (code->op != FETCH_OP_NOP) { 2024 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 2025 return -E2BIG; 2026 } 2027 code->op = FETCH_OP_ST_RAW; 2028 code->size = parg->type->size; 2029 } 2030 2031 /* Save storing fetch_insn. */ 2032 scode = code; 2033 2034 /* Modify operation */ 2035 if (type != NULL) { 2036 /* Bitfield needs a special fetch_insn. */ 2037 ret = __parse_bitfield_probe_arg(type, parg->type, &code); 2038 if (ret) { 2039 trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD); 2040 return ret; 2041 } 2042 } else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) && 2043 ctx->last_type) { 2044 /* If user not specified the type, try parsing BTF bitfield. */ 2045 ret = parse_btf_bitfield(&code, parg->type, ctx); 2046 if (ret) 2047 return ret; 2048 } 2049 2050 /* Loop(Array) operation */ 2051 if (parg->count) { 2052 if (scode->op != FETCH_OP_ST_MEM && 2053 scode->op != FETCH_OP_ST_STRING && 2054 scode->op != FETCH_OP_ST_USTRING) { 2055 trace_probe_log_err(ctx->offset + type_offset, BAD_STRING); 2056 return -EINVAL; 2057 } 2058 code++; 2059 if (code->op != FETCH_OP_NOP) { 2060 trace_probe_log_err(ctx->offset, TOO_MANY_OPS); 2061 return -E2BIG; 2062 } 2063 code->op = FETCH_OP_LP_ARRAY; 2064 code->param = parg->count; 2065 } 2066 2067 /* Finalize the fetch_insn array. */ 2068 code++; 2069 code->op = FETCH_OP_END; 2070 2071 return 0; 2072 } 2073 2074 /* String length checking wrapper */ 2075 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size, 2076 struct probe_arg *parg, 2077 struct traceprobe_parse_context *ctx) 2078 { 2079 struct fetch_insn *code, *tmp = NULL; 2080 char *type, *arg __free(kfree) = NULL; 2081 int ret, len; 2082 2083 len = strlen(argv); 2084 if (len > MAX_ARGSTR_LEN) { 2085 trace_probe_log_err(ctx->offset, ARG_TOO_LONG); 2086 return -E2BIG; 2087 } else if (len == 0) { 2088 trace_probe_log_err(ctx->offset, NO_ARG_BODY); 2089 return -EINVAL; 2090 } 2091 2092 arg = kstrdup(argv, GFP_KERNEL); 2093 if (!arg) 2094 return -ENOMEM; 2095 2096 parg->comm = kstrdup(arg, GFP_KERNEL); 2097 if (!parg->comm) 2098 return -ENOMEM; 2099 2100 type = parse_probe_arg_type(arg, parg, ctx); 2101 if (IS_ERR(type)) 2102 return PTR_ERR(type); 2103 2104 code = tmp = kzalloc_objs(*code, FETCH_INSN_MAX); 2105 if (!code) 2106 return -ENOMEM; 2107 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END; 2108 2109 ctx->last_type = NULL; 2110 ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1], 2111 ctx); 2112 if (ret < 0) 2113 goto fail; 2114 /* Update storing type if BTF is available */ 2115 if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) && 2116 ctx->last_type) { 2117 if (!type) { 2118 parg->type = find_fetch_type_from_btf_type(ctx); 2119 } else if (strstr(type, "string")) { 2120 ret = check_prepare_btf_string_fetch(type, &code, ctx); 2121 if (ret) 2122 goto fail; 2123 } 2124 } 2125 parg->offset = *size; 2126 *size += parg->type->size * (parg->count ?: 1); 2127 2128 if (*size > MAX_PROBE_EVENT_SIZE) { 2129 ret = -E2BIG; 2130 trace_probe_log_err(ctx->offset, EVENT_TOO_BIG); 2131 goto fail; 2132 } 2133 2134 if (parg->count) { 2135 len = strlen(parg->type->fmttype) + 6; 2136 parg->fmt = kmalloc(len, GFP_KERNEL); 2137 if (!parg->fmt) { 2138 ret = -ENOMEM; 2139 goto fail; 2140 } 2141 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype, 2142 parg->count); 2143 } 2144 2145 ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx); 2146 if (ret < 0) 2147 goto fail; 2148 2149 for (; code < tmp + FETCH_INSN_MAX; code++) 2150 if (code->op == FETCH_OP_END) 2151 break; 2152 /* Shrink down the code buffer */ 2153 parg->code = kzalloc_objs(*code, code - tmp + 1); 2154 if (!parg->code) 2155 ret = -ENOMEM; 2156 else 2157 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1)); 2158 2159 fail: 2160 if (ret < 0) { 2161 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++) 2162 if (code->op == FETCH_NOP_SYMBOL || 2163 code->op == FETCH_OP_IMMSTR) 2164 kfree(code->data); 2165 } 2166 kfree(tmp); 2167 2168 /* struct_btf should not be passed to other arguments */ 2169 clear_struct_btf(ctx); 2170 2171 return ret; 2172 } 2173 2174 /* Return 1 if name is reserved or already used by another argument */ 2175 static int traceprobe_conflict_field_name(const char *name, 2176 struct probe_arg *args, int narg) 2177 { 2178 int i; 2179 2180 for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++) 2181 if (strcmp(reserved_field_names[i], name) == 0) 2182 return 1; 2183 2184 for (i = 0; i < narg; i++) 2185 if (strcmp(args[i].name, name) == 0) 2186 return 1; 2187 2188 return 0; 2189 } 2190 2191 static char *generate_probe_arg_name(const char *arg, int idx) 2192 { 2193 char *name = NULL; 2194 const char *end; 2195 2196 /* 2197 * If argument name is omitted, try arg as a name (BTF variable) 2198 * or "argN". 2199 */ 2200 if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) { 2201 end = strchr(arg, ':'); 2202 if (!end) 2203 end = arg + strlen(arg); 2204 2205 name = kmemdup_nul(arg, end - arg, GFP_KERNEL); 2206 if (!name || !is_good_name(name)) { 2207 kfree(name); 2208 name = NULL; 2209 } 2210 } 2211 2212 if (!name) 2213 name = kasprintf(GFP_KERNEL, "arg%d", idx + 1); 2214 2215 return name; 2216 } 2217 2218 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg, 2219 struct traceprobe_parse_context *ctx) 2220 { 2221 struct probe_arg *parg = &tp->args[i]; 2222 const char *body; 2223 2224 ctx->tp = tp; 2225 body = strchr(arg, '='); 2226 if (body) { 2227 if (body - arg > MAX_ARG_NAME_LEN) { 2228 trace_probe_log_err(0, ARG_NAME_TOO_LONG); 2229 return -EINVAL; 2230 } else if (body == arg) { 2231 trace_probe_log_err(0, NO_ARG_NAME); 2232 return -EINVAL; 2233 } 2234 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL); 2235 body++; 2236 } else { 2237 parg->name = generate_probe_arg_name(arg, i); 2238 body = arg; 2239 } 2240 if (!parg->name) 2241 return -ENOMEM; 2242 2243 if (!is_good_name(parg->name)) { 2244 trace_probe_log_err(0, BAD_ARG_NAME); 2245 return -EINVAL; 2246 } 2247 if (traceprobe_conflict_field_name(parg->name, tp->args, i)) { 2248 trace_probe_log_err(0, USED_ARG_NAME); 2249 return -EINVAL; 2250 } 2251 ctx->offset = body - arg; 2252 /* Parse fetch argument */ 2253 return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx); 2254 } 2255 2256 void traceprobe_free_probe_arg(struct probe_arg *arg) 2257 { 2258 struct fetch_insn *code = arg->code; 2259 2260 while (code && code->op != FETCH_OP_END) { 2261 if (code->op == FETCH_NOP_SYMBOL || 2262 code->op == FETCH_OP_IMMSTR) 2263 kfree(code->data); 2264 code++; 2265 } 2266 kfree(arg->code); 2267 kfree(arg->name); 2268 kfree(arg->comm); 2269 kfree(arg->fmt); 2270 } 2271 2272 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx, 2273 struct traceprobe_parse_context *ctx) 2274 { 2275 int i, found = 0; 2276 2277 for (i = 0; i < argc; i++) 2278 if (str_has_prefix(argv[i], "$arg")) { 2279 trace_probe_log_set_index(i + 2); 2280 2281 if (!tparg_is_function_entry(ctx->flags) && 2282 !tparg_is_function_return(ctx->flags)) { 2283 trace_probe_log_err(0, NOFENTRY_ARGS); 2284 return -EINVAL; 2285 } 2286 2287 if (isdigit(argv[i][4])) { 2288 found = 1; 2289 continue; 2290 } 2291 2292 if (argv[i][4] != '*') { 2293 trace_probe_log_err(0, BAD_VAR); 2294 return -EINVAL; 2295 } 2296 2297 if (*args_idx >= 0 && *args_idx < argc) { 2298 trace_probe_log_err(0, DOUBLE_ARGS); 2299 return -EINVAL; 2300 } 2301 found = 1; 2302 *args_idx = i; 2303 } 2304 2305 return found; 2306 } 2307 2308 static int sprint_nth_btf_arg(int idx, const char *type, 2309 char *buf, int bufsize, 2310 struct traceprobe_parse_context *ctx) 2311 { 2312 const char *name; 2313 int ret; 2314 2315 if (idx >= ctx->nr_params) { 2316 trace_probe_log_err(0, NO_BTFARG); 2317 return -ENOENT; 2318 } 2319 name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off); 2320 if (!name) { 2321 trace_probe_log_err(0, NO_BTF_ENTRY); 2322 return -ENOENT; 2323 } 2324 ret = snprintf(buf, bufsize, "%s%s", name, type); 2325 if (ret >= bufsize) { 2326 trace_probe_log_err(0, ARGS_2LONG); 2327 return -E2BIG; 2328 } 2329 return ret; 2330 } 2331 2332 /* Return new_argv which must be freed after use */ 2333 const char **traceprobe_expand_meta_args(int argc, const char *argv[], 2334 int *new_argc, char *buf, int bufsize, 2335 struct traceprobe_parse_context *ctx) 2336 { 2337 int i, j, n, used, ret, args_idx = -1; 2338 const char **new_argv __free(kfree) = NULL; 2339 2340 ret = argv_has_var_arg(argc, argv, &args_idx, ctx); 2341 if (ret < 0) 2342 return ERR_PTR(ret); 2343 2344 if (!ret) { 2345 *new_argc = argc; 2346 return NULL; 2347 } 2348 2349 ret = query_btf_context(ctx); 2350 if (ret < 0 || ctx->nr_params == 0) { 2351 if (args_idx != -1) { 2352 /* $arg* requires BTF info */ 2353 trace_probe_log_err(0, NOSUP_BTFARG); 2354 return ERR_PTR(-EOPNOTSUPP); 2355 } 2356 *new_argc = argc; 2357 return NULL; 2358 } 2359 2360 if (args_idx >= 0) 2361 *new_argc = argc + ctx->nr_params - 1; 2362 else 2363 *new_argc = argc; 2364 2365 new_argv = kzalloc_objs(*new_argv, *new_argc); 2366 if (!new_argv) 2367 return ERR_PTR(-ENOMEM); 2368 2369 used = 0; 2370 for (i = 0, j = 0; i < argc; i++) { 2371 trace_probe_log_set_index(i + 2); 2372 if (i == args_idx) { 2373 for (n = 0; n < ctx->nr_params; n++) { 2374 ret = sprint_nth_btf_arg(n, "", buf + used, 2375 bufsize - used, ctx); 2376 if (ret < 0) 2377 return ERR_PTR(ret); 2378 2379 new_argv[j++] = buf + used; 2380 used += ret + 1; 2381 } 2382 continue; 2383 } 2384 2385 if (str_has_prefix(argv[i], "$arg")) { 2386 char *type = NULL; 2387 2388 n = simple_strtoul(argv[i] + 4, &type, 10); 2389 if (type && !(*type == ':' || *type == '\0')) { 2390 trace_probe_log_err(0, BAD_VAR); 2391 return ERR_PTR(-ENOENT); 2392 } 2393 /* Note: $argN starts from $arg1, so $arg0 is invalid. */ 2394 if (n == 0) { 2395 trace_probe_log_err(0, BAD_ARG_NUM); 2396 return ERR_PTR(-EINVAL); 2397 } 2398 ret = sprint_nth_btf_arg(n - 1, type, buf + used, 2399 bufsize - used, ctx); 2400 if (ret < 0) 2401 return ERR_PTR(ret); 2402 new_argv[j++] = buf + used; 2403 used += ret + 1; 2404 } else 2405 new_argv[j++] = argv[i]; 2406 } 2407 2408 return_ptr(new_argv); 2409 } 2410 2411 /* @buf: *buf must be equal to NULL. Caller must to free *buf */ 2412 int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf) 2413 { 2414 int i, used, ret; 2415 const int bufsize = MAX_DENTRY_ARGS_LEN; 2416 char *tmpbuf __free(kfree) = NULL; 2417 2418 if (*buf) 2419 return -EINVAL; 2420 2421 used = 0; 2422 for (i = 0; i < argc; i++) { 2423 char *tmp __free(kfree) = NULL; 2424 char *equal; 2425 size_t arg_len; 2426 2427 if (!glob_match("*:%p[dD]", argv[i])) 2428 continue; 2429 2430 if (!tmpbuf) { 2431 tmpbuf = kmalloc(bufsize, GFP_KERNEL); 2432 if (!tmpbuf) 2433 return -ENOMEM; 2434 } 2435 2436 tmp = kstrdup(argv[i], GFP_KERNEL); 2437 if (!tmp) 2438 return -ENOMEM; 2439 2440 equal = strchr(tmp, '='); 2441 if (equal) 2442 *equal = '\0'; 2443 arg_len = strlen(argv[i]); 2444 tmp[arg_len - 4] = '\0'; 2445 if (argv[i][arg_len - 1] == 'd') 2446 ret = snprintf(tmpbuf + used, bufsize - used, 2447 "%s%s+0x0(+0x%zx(%s)):string", 2448 equal ? tmp : "", equal ? "=" : "", 2449 offsetof(struct dentry, d_name.name), 2450 equal ? equal + 1 : tmp); 2451 else 2452 ret = snprintf(tmpbuf + used, bufsize - used, 2453 "%s%s+0x0(+0x%zx(+0x%zx(%s))):string", 2454 equal ? tmp : "", equal ? "=" : "", 2455 offsetof(struct dentry, d_name.name), 2456 offsetof(struct file, f_path.dentry), 2457 equal ? equal + 1 : tmp); 2458 2459 if (ret >= bufsize - used) 2460 return -ENOMEM; 2461 argv[i] = tmpbuf + used; 2462 used += ret + 1; 2463 } 2464 2465 *buf = no_free_ptr(tmpbuf); 2466 return 0; 2467 } 2468 2469 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx) 2470 { 2471 clear_btf_context(ctx); 2472 } 2473 2474 int traceprobe_update_arg(struct probe_arg *arg) 2475 { 2476 struct fetch_insn *code = arg->code; 2477 long offset; 2478 char *tmp; 2479 char c; 2480 int ret = 0; 2481 2482 while (code && code->op != FETCH_OP_END) { 2483 if (code->op == FETCH_NOP_SYMBOL) { 2484 if (code[1].op != FETCH_OP_IMM) 2485 return -EINVAL; 2486 2487 tmp = strpbrk(code->data, "+-"); 2488 if (tmp) 2489 c = *tmp; 2490 ret = traceprobe_split_symbol_offset(code->data, 2491 &offset); 2492 if (ret) 2493 return ret; 2494 2495 code[1].immediate = 2496 (unsigned long)kallsyms_lookup_name(code->data); 2497 if (tmp) 2498 *tmp = c; 2499 if (!code[1].immediate) 2500 return -ENOENT; 2501 code[1].immediate += offset; 2502 } 2503 code++; 2504 } 2505 return 0; 2506 } 2507 2508 /* When len=0, we just calculate the needed length */ 2509 #define LEN_OR_ZERO (len > pos ? len - pos : 0) 2510 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len, 2511 enum probe_print_type ptype) 2512 { 2513 struct probe_arg *parg; 2514 int i, j; 2515 int pos = 0; 2516 const char *fmt, *arg; 2517 2518 switch (ptype) { 2519 case PROBE_PRINT_NORMAL: 2520 fmt = "(%lx)"; 2521 arg = ", REC->" FIELD_STRING_IP; 2522 break; 2523 case PROBE_PRINT_RETURN: 2524 fmt = "(%lx <- %lx)"; 2525 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP; 2526 break; 2527 case PROBE_PRINT_EVENT: 2528 fmt = ""; 2529 arg = ""; 2530 break; 2531 default: 2532 WARN_ON_ONCE(1); 2533 return 0; 2534 } 2535 2536 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt); 2537 2538 for (i = 0; i < tp->nr_args; i++) { 2539 parg = tp->args + i; 2540 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name); 2541 if (parg->count) { 2542 pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s", 2543 parg->type->fmt); 2544 for (j = 1; j < parg->count; j++) 2545 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s", 2546 parg->type->fmt); 2547 pos += snprintf(buf + pos, LEN_OR_ZERO, "}"); 2548 } else 2549 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", 2550 parg->type->fmt); 2551 } 2552 2553 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg); 2554 2555 for (i = 0; i < tp->nr_args; i++) { 2556 parg = tp->args + i; 2557 if (parg->count) { 2558 if (parg->type->is_string) 2559 fmt = ", __get_str(%s[%d])"; 2560 else 2561 fmt = ", REC->%s[%d]"; 2562 for (j = 0; j < parg->count; j++) 2563 pos += snprintf(buf + pos, LEN_OR_ZERO, 2564 fmt, parg->name, j); 2565 } else { 2566 if (parg->type->is_string) 2567 fmt = ", __get_str(%s)"; 2568 else 2569 fmt = ", REC->%s"; 2570 pos += snprintf(buf + pos, LEN_OR_ZERO, 2571 fmt, parg->name); 2572 } 2573 } 2574 2575 /* return the length of print_fmt */ 2576 return pos; 2577 } 2578 #undef LEN_OR_ZERO 2579 2580 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype) 2581 { 2582 struct trace_event_call *call = trace_probe_event_call(tp); 2583 int len; 2584 char *print_fmt; 2585 2586 /* First: called with 0 length to calculate the needed length */ 2587 len = __set_print_fmt(tp, NULL, 0, ptype); 2588 print_fmt = kmalloc(len + 1, GFP_KERNEL); 2589 if (!print_fmt) 2590 return -ENOMEM; 2591 2592 /* Second: actually write the @print_fmt */ 2593 __set_print_fmt(tp, print_fmt, len + 1, ptype); 2594 call->print_fmt = print_fmt; 2595 2596 return 0; 2597 } 2598 2599 int traceprobe_define_arg_fields(struct trace_event_call *event_call, 2600 size_t offset, struct trace_probe *tp) 2601 { 2602 struct trace_probe_event *tpe = trace_probe_event_from_call(event_call); 2603 int ret, i; 2604 2605 /* 2606 * A field created by trace_define_field() only stores the name and 2607 * type pointers, it does not copy the strings. Here they point into 2608 * the probe_arg of @tp, which is freed when @tp is removed. For an 2609 * event with multiple probes attached, the field list is defined 2610 * once by the first probe but kept alive by the surviving siblings, 2611 * so removing that first probe would leave the fields referencing 2612 * freed memory. Duplicate the strings and anchor the copies on the 2613 * trace_probe_event, which lives as long as the field list itself. 2614 * 2615 * event_define_fields() ignores the return value of this hook, so 2616 * if a previous attempt failed before creating any field, it may 2617 * call here again. Release duplicates left behind by such an 2618 * attempt before starting over. 2619 */ 2620 for (i = 0; i < tpe->nr_field_strings; i++) 2621 kfree(tpe->field_strings[i]); 2622 kfree(tpe->field_strings); 2623 tpe->field_strings = NULL; 2624 tpe->nr_field_strings = 0; 2625 2626 if (tp->nr_args) { 2627 tpe->field_strings = kcalloc(tp->nr_args * 2, sizeof(char *), 2628 GFP_KERNEL); 2629 if (!tpe->field_strings) 2630 return -ENOMEM; 2631 } 2632 2633 /* Set argument names as fields */ 2634 for (i = 0; i < tp->nr_args; i++) { 2635 struct probe_arg *parg = &tp->args[i]; 2636 const char *fmt = parg->type->fmttype; 2637 int size = parg->type->size; 2638 char *name, *type; 2639 2640 if (parg->fmt) 2641 fmt = parg->fmt; 2642 if (parg->count) 2643 size *= parg->count; 2644 2645 name = kstrdup(parg->name, GFP_KERNEL); 2646 type = kstrdup(fmt, GFP_KERNEL); 2647 if (!name || !type) { 2648 kfree(name); 2649 kfree(type); 2650 return -ENOMEM; 2651 } 2652 tpe->field_strings[tpe->nr_field_strings++] = name; 2653 tpe->field_strings[tpe->nr_field_strings++] = type; 2654 2655 ret = trace_define_field(event_call, type, name, 2656 offset + parg->offset, size, 2657 parg->type->is_signed, 2658 FILTER_OTHER); 2659 if (ret) 2660 return ret; 2661 } 2662 return 0; 2663 } 2664 2665 static void trace_probe_event_free(struct trace_probe_event *tpe) 2666 { 2667 int i; 2668 2669 for (i = 0; i < tpe->nr_field_strings; i++) 2670 kfree(tpe->field_strings[i]); 2671 kfree(tpe->field_strings); 2672 kfree(tpe->class.system); 2673 kfree(tpe->call.name); 2674 kfree(tpe->call.print_fmt); 2675 kfree(tpe); 2676 } 2677 2678 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to) 2679 { 2680 if (trace_probe_has_sibling(tp)) 2681 return -EBUSY; 2682 2683 list_del_init(&tp->list); 2684 trace_probe_event_free(tp->event); 2685 2686 tp->event = to->event; 2687 list_add_tail(&tp->list, trace_probe_probe_list(to)); 2688 2689 return 0; 2690 } 2691 2692 void trace_probe_unlink(struct trace_probe *tp) 2693 { 2694 list_del_init(&tp->list); 2695 if (list_empty(trace_probe_probe_list(tp))) 2696 trace_probe_event_free(tp->event); 2697 tp->event = NULL; 2698 } 2699 2700 void trace_probe_cleanup(struct trace_probe *tp) 2701 { 2702 int i; 2703 2704 for (i = 0; i < tp->nr_args; i++) 2705 traceprobe_free_probe_arg(&tp->args[i]); 2706 2707 if (tp->entry_arg) { 2708 kfree(tp->entry_arg); 2709 tp->entry_arg = NULL; 2710 } 2711 2712 if (tp->event) 2713 trace_probe_unlink(tp); 2714 } 2715 2716 int trace_probe_init(struct trace_probe *tp, const char *event, 2717 const char *group, bool alloc_filter, int nargs) 2718 { 2719 struct trace_event_call *call; 2720 size_t size = sizeof(struct trace_probe_event); 2721 int ret = 0; 2722 2723 if (!event || !group) 2724 return -EINVAL; 2725 2726 if (alloc_filter) 2727 size += sizeof(struct trace_uprobe_filter); 2728 2729 tp->event = kzalloc(size, GFP_KERNEL); 2730 if (!tp->event) 2731 return -ENOMEM; 2732 2733 INIT_LIST_HEAD(&tp->event->files); 2734 INIT_LIST_HEAD(&tp->event->class.fields); 2735 INIT_LIST_HEAD(&tp->event->probes); 2736 INIT_LIST_HEAD(&tp->list); 2737 list_add(&tp->list, &tp->event->probes); 2738 2739 call = trace_probe_event_call(tp); 2740 call->class = &tp->event->class; 2741 call->name = kstrdup(event, GFP_KERNEL); 2742 if (!call->name) { 2743 ret = -ENOMEM; 2744 goto error; 2745 } 2746 2747 tp->event->class.system = kstrdup(group, GFP_KERNEL); 2748 if (!tp->event->class.system) { 2749 ret = -ENOMEM; 2750 goto error; 2751 } 2752 2753 tp->nr_args = nargs; 2754 /* Make sure pointers in args[] are NULL */ 2755 if (nargs) 2756 memset(tp->args, 0, sizeof(tp->args[0]) * nargs); 2757 2758 return 0; 2759 2760 error: 2761 trace_probe_cleanup(tp); 2762 return ret; 2763 } 2764 2765 static struct trace_event_call * 2766 find_trace_event_call(const char *system, const char *event_name) 2767 { 2768 struct trace_event_call *tp_event; 2769 const char *name; 2770 2771 list_for_each_entry(tp_event, &ftrace_events, list) { 2772 if (!tp_event->class->system || 2773 strcmp(system, tp_event->class->system)) 2774 continue; 2775 name = trace_event_name(tp_event); 2776 if (!name || strcmp(event_name, name)) 2777 continue; 2778 return tp_event; 2779 } 2780 2781 return NULL; 2782 } 2783 2784 int trace_probe_register_event_call(struct trace_probe *tp) 2785 { 2786 struct trace_event_call *call = trace_probe_event_call(tp); 2787 int ret; 2788 2789 lockdep_assert_held(&event_mutex); 2790 2791 if (find_trace_event_call(trace_probe_group_name(tp), 2792 trace_probe_name(tp))) 2793 return -EEXIST; 2794 2795 ret = register_trace_event(&call->event); 2796 if (!ret) 2797 return -ENODEV; 2798 2799 ret = trace_add_event_call(call); 2800 if (ret) 2801 unregister_trace_event(&call->event); 2802 2803 return ret; 2804 } 2805 2806 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file) 2807 { 2808 struct event_file_link *link; 2809 2810 link = kmalloc_obj(*link); 2811 if (!link) 2812 return -ENOMEM; 2813 2814 link->file = file; 2815 INIT_LIST_HEAD(&link->list); 2816 list_add_tail_rcu(&link->list, &tp->event->files); 2817 trace_probe_set_flag(tp, TP_FLAG_TRACE); 2818 return 0; 2819 } 2820 2821 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp, 2822 struct trace_event_file *file) 2823 { 2824 struct event_file_link *link; 2825 2826 trace_probe_for_each_link(link, tp) { 2827 if (link->file == file) 2828 return link; 2829 } 2830 2831 return NULL; 2832 } 2833 2834 int trace_probe_remove_file(struct trace_probe *tp, 2835 struct trace_event_file *file) 2836 { 2837 struct event_file_link *link; 2838 2839 link = trace_probe_get_file_link(tp, file); 2840 if (!link) 2841 return -ENOENT; 2842 2843 list_del_rcu(&link->list); 2844 kvfree_rcu_mightsleep(link); 2845 2846 if (list_empty(&tp->event->files)) 2847 trace_probe_clear_flag(tp, TP_FLAG_TRACE); 2848 2849 return 0; 2850 } 2851 2852 /* 2853 * Return the smallest index of different type argument (start from 1). 2854 * If all argument types and name are same, return 0. 2855 */ 2856 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b) 2857 { 2858 int i; 2859 2860 /* In case of more arguments */ 2861 if (a->nr_args < b->nr_args) 2862 return a->nr_args + 1; 2863 if (a->nr_args > b->nr_args) 2864 return b->nr_args + 1; 2865 2866 for (i = 0; i < a->nr_args; i++) { 2867 if ((a->args[i].type != b->args[i].type) || 2868 (a->args[i].count != b->args[i].count) || 2869 strcmp(a->args[i].name, b->args[i].name)) 2870 return i + 1; 2871 } 2872 2873 return 0; 2874 } 2875 2876 bool trace_probe_match_command_args(struct trace_probe *tp, 2877 int argc, const char **argv) 2878 { 2879 int i; 2880 2881 if (tp->nr_args < argc) 2882 return false; 2883 2884 for (i = 0; i < argc; i++) { 2885 int len = strlen(tp->args[i].name); 2886 2887 if (strncmp(argv[i], tp->args[i].name, len) || 2888 argv[i][len] != '=' || 2889 strcmp(argv[i] + len + 1, tp->args[i].comm)) 2890 return false; 2891 } 2892 return true; 2893 } 2894 2895 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **)) 2896 { 2897 int argc = 0, ret = 0; 2898 char **argv; 2899 2900 argv = argv_split(GFP_KERNEL, raw_command, &argc); 2901 if (!argv) 2902 return -ENOMEM; 2903 2904 if (argc) 2905 ret = createfn(argc, (const char **)argv); 2906 2907 argv_free(argv); 2908 2909 return ret; 2910 } 2911 2912 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args, 2913 u8 *data, void *field) 2914 { 2915 void *p; 2916 int i, j; 2917 2918 for (i = 0; i < nr_args; i++) { 2919 struct probe_arg *a = args + i; 2920 2921 trace_seq_printf(s, " %s=", a->name); 2922 if (likely(!a->count)) { 2923 if (!a->type->print(s, data + a->offset, field)) 2924 return -ENOMEM; 2925 continue; 2926 } 2927 trace_seq_putc(s, '{'); 2928 p = data + a->offset; 2929 for (j = 0; j < a->count; j++) { 2930 if (!a->type->print(s, p, field)) 2931 return -ENOMEM; 2932 trace_seq_putc(s, j == a->count - 1 ? '}' : ','); 2933 p += a->type->size; 2934 } 2935 } 2936 return 0; 2937 } 2938 2939 #ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG 2940 2941 struct fetch_op_decode { 2942 const char *name; 2943 void (*decode)(struct seq_file *m, struct fetch_insn *insn); 2944 }; 2945 2946 static const struct fetch_op_decode fetch_op_decode[]; 2947 2948 static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn) 2949 { 2950 seq_puts(m, fetch_op_decode[insn->op].name); 2951 } 2952 2953 static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn) 2954 { 2955 seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param); 2956 } 2957 2958 static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn) 2959 { 2960 seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate); 2961 } 2962 2963 static void fetcharg_decode_string(struct seq_file *m, struct fetch_insn *insn) 2964 { 2965 seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data); 2966 } 2967 2968 static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn) 2969 { 2970 seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data); 2971 } 2972 2973 static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn) 2974 { 2975 seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset); 2976 } 2977 2978 static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn) 2979 { 2980 if (insn->op == FETCH_OP_ST_RAW) 2981 seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size); 2982 else 2983 seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name, 2984 insn->offset, insn->size); 2985 } 2986 2987 static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn) 2988 { 2989 seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)", 2990 fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift); 2991 } 2992 2993 static void fetcharg_decode_tp_arg(struct seq_file *m, struct fetch_insn *insn) 2994 { 2995 struct ftrace_event_field *field = insn->data; 2996 2997 seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, field->name); 2998 } 2999 3000 #define FETCH_OP(opname, decode_fn) \ 3001 [FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn } 3002 3003 static const struct fetch_op_decode fetch_op_decode[] = FETCH_OP_LIST; 3004 #undef FETCH_OP 3005 3006 static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg) 3007 { 3008 int i; 3009 3010 seq_printf(m, "# %s: ", parg->name); 3011 for (i = 0; i < FETCH_INSN_MAX; i++) { 3012 struct fetch_insn *insn = parg->code + i; 3013 3014 if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode) 3015 seq_printf(m, "unknown(%d)", insn->op); 3016 else 3017 fetch_op_decode[insn->op].decode(m, insn); 3018 3019 if (insn->op == FETCH_OP_END) 3020 break; 3021 seq_puts(m, " -> "); 3022 } 3023 seq_putc(m, '\n'); 3024 } 3025 3026 void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp) 3027 { 3028 int i; 3029 3030 for (i = 0; i < tp->nr_args; i++) 3031 trace_probe_dump_arg(m, &tp->args[i]); 3032 } 3033 #endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */ 3034