1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/mutex.h> 11 #include <linux/slab.h> 12 #include <linux/stacktrace.h> 13 #include <linux/rculist.h> 14 #include <linux/tracefs.h> 15 16 #include "tracing_map.h" 17 #include "trace.h" 18 19 #define SYNTH_SYSTEM "synthetic" 20 #define SYNTH_FIELDS_MAX 16 21 22 #define STR_VAR_LEN_MAX 32 /* must be multiple of sizeof(u64) */ 23 24 struct hist_field; 25 26 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 27 struct tracing_map_elt *elt, 28 struct ring_buffer_event *rbe, 29 void *event); 30 31 #define HIST_FIELD_OPERANDS_MAX 2 32 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 33 #define HIST_ACTIONS_MAX 8 34 35 enum field_op_id { 36 FIELD_OP_NONE, 37 FIELD_OP_PLUS, 38 FIELD_OP_MINUS, 39 FIELD_OP_UNARY_MINUS, 40 }; 41 42 struct hist_var { 43 char *name; 44 struct hist_trigger_data *hist_data; 45 unsigned int idx; 46 }; 47 48 struct hist_field { 49 struct ftrace_event_field *field; 50 unsigned long flags; 51 hist_field_fn_t fn; 52 unsigned int size; 53 unsigned int offset; 54 unsigned int is_signed; 55 const char *type; 56 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 57 struct hist_trigger_data *hist_data; 58 struct hist_var var; 59 enum field_op_id operator; 60 char *system; 61 char *event_name; 62 char *name; 63 unsigned int var_idx; 64 unsigned int var_ref_idx; 65 bool read_once; 66 }; 67 68 static u64 hist_field_none(struct hist_field *field, 69 struct tracing_map_elt *elt, 70 struct ring_buffer_event *rbe, 71 void *event) 72 { 73 return 0; 74 } 75 76 static u64 hist_field_counter(struct hist_field *field, 77 struct tracing_map_elt *elt, 78 struct ring_buffer_event *rbe, 79 void *event) 80 { 81 return 1; 82 } 83 84 static u64 hist_field_string(struct hist_field *hist_field, 85 struct tracing_map_elt *elt, 86 struct ring_buffer_event *rbe, 87 void *event) 88 { 89 char *addr = (char *)(event + hist_field->field->offset); 90 91 return (u64)(unsigned long)addr; 92 } 93 94 static u64 hist_field_dynstring(struct hist_field *hist_field, 95 struct tracing_map_elt *elt, 96 struct ring_buffer_event *rbe, 97 void *event) 98 { 99 u32 str_item = *(u32 *)(event + hist_field->field->offset); 100 int str_loc = str_item & 0xffff; 101 char *addr = (char *)(event + str_loc); 102 103 return (u64)(unsigned long)addr; 104 } 105 106 static u64 hist_field_pstring(struct hist_field *hist_field, 107 struct tracing_map_elt *elt, 108 struct ring_buffer_event *rbe, 109 void *event) 110 { 111 char **addr = (char **)(event + hist_field->field->offset); 112 113 return (u64)(unsigned long)*addr; 114 } 115 116 static u64 hist_field_log2(struct hist_field *hist_field, 117 struct tracing_map_elt *elt, 118 struct ring_buffer_event *rbe, 119 void *event) 120 { 121 struct hist_field *operand = hist_field->operands[0]; 122 123 u64 val = operand->fn(operand, elt, rbe, event); 124 125 return (u64) ilog2(roundup_pow_of_two(val)); 126 } 127 128 static u64 hist_field_plus(struct hist_field *hist_field, 129 struct tracing_map_elt *elt, 130 struct ring_buffer_event *rbe, 131 void *event) 132 { 133 struct hist_field *operand1 = hist_field->operands[0]; 134 struct hist_field *operand2 = hist_field->operands[1]; 135 136 u64 val1 = operand1->fn(operand1, elt, rbe, event); 137 u64 val2 = operand2->fn(operand2, elt, rbe, event); 138 139 return val1 + val2; 140 } 141 142 static u64 hist_field_minus(struct hist_field *hist_field, 143 struct tracing_map_elt *elt, 144 struct ring_buffer_event *rbe, 145 void *event) 146 { 147 struct hist_field *operand1 = hist_field->operands[0]; 148 struct hist_field *operand2 = hist_field->operands[1]; 149 150 u64 val1 = operand1->fn(operand1, elt, rbe, event); 151 u64 val2 = operand2->fn(operand2, elt, rbe, event); 152 153 return val1 - val2; 154 } 155 156 static u64 hist_field_unary_minus(struct hist_field *hist_field, 157 struct tracing_map_elt *elt, 158 struct ring_buffer_event *rbe, 159 void *event) 160 { 161 struct hist_field *operand = hist_field->operands[0]; 162 163 s64 sval = (s64)operand->fn(operand, elt, rbe, event); 164 u64 val = (u64)-sval; 165 166 return val; 167 } 168 169 #define DEFINE_HIST_FIELD_FN(type) \ 170 static u64 hist_field_##type(struct hist_field *hist_field, \ 171 struct tracing_map_elt *elt, \ 172 struct ring_buffer_event *rbe, \ 173 void *event) \ 174 { \ 175 type *addr = (type *)(event + hist_field->field->offset); \ 176 \ 177 return (u64)(unsigned long)*addr; \ 178 } 179 180 DEFINE_HIST_FIELD_FN(s64); 181 DEFINE_HIST_FIELD_FN(u64); 182 DEFINE_HIST_FIELD_FN(s32); 183 DEFINE_HIST_FIELD_FN(u32); 184 DEFINE_HIST_FIELD_FN(s16); 185 DEFINE_HIST_FIELD_FN(u16); 186 DEFINE_HIST_FIELD_FN(s8); 187 DEFINE_HIST_FIELD_FN(u8); 188 189 #define for_each_hist_field(i, hist_data) \ 190 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 191 192 #define for_each_hist_val_field(i, hist_data) \ 193 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 194 195 #define for_each_hist_key_field(i, hist_data) \ 196 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 197 198 #define HIST_STACKTRACE_DEPTH 16 199 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 200 #define HIST_STACKTRACE_SKIP 5 201 202 #define HITCOUNT_IDX 0 203 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 204 205 enum hist_field_flags { 206 HIST_FIELD_FL_HITCOUNT = 1 << 0, 207 HIST_FIELD_FL_KEY = 1 << 1, 208 HIST_FIELD_FL_STRING = 1 << 2, 209 HIST_FIELD_FL_HEX = 1 << 3, 210 HIST_FIELD_FL_SYM = 1 << 4, 211 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 212 HIST_FIELD_FL_EXECNAME = 1 << 6, 213 HIST_FIELD_FL_SYSCALL = 1 << 7, 214 HIST_FIELD_FL_STACKTRACE = 1 << 8, 215 HIST_FIELD_FL_LOG2 = 1 << 9, 216 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 217 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 218 HIST_FIELD_FL_VAR = 1 << 12, 219 HIST_FIELD_FL_EXPR = 1 << 13, 220 HIST_FIELD_FL_VAR_REF = 1 << 14, 221 HIST_FIELD_FL_CPU = 1 << 15, 222 HIST_FIELD_FL_ALIAS = 1 << 16, 223 }; 224 225 struct var_defs { 226 unsigned int n_vars; 227 char *name[TRACING_MAP_VARS_MAX]; 228 char *expr[TRACING_MAP_VARS_MAX]; 229 }; 230 231 struct hist_trigger_attrs { 232 char *keys_str; 233 char *vals_str; 234 char *sort_key_str; 235 char *name; 236 char *clock; 237 bool pause; 238 bool cont; 239 bool clear; 240 bool ts_in_usecs; 241 unsigned int map_bits; 242 243 char *assignment_str[TRACING_MAP_VARS_MAX]; 244 unsigned int n_assignments; 245 246 char *action_str[HIST_ACTIONS_MAX]; 247 unsigned int n_actions; 248 249 struct var_defs var_defs; 250 }; 251 252 struct field_var { 253 struct hist_field *var; 254 struct hist_field *val; 255 }; 256 257 struct field_var_hist { 258 struct hist_trigger_data *hist_data; 259 char *cmd; 260 }; 261 262 struct hist_trigger_data { 263 struct hist_field *fields[HIST_FIELDS_MAX]; 264 unsigned int n_vals; 265 unsigned int n_keys; 266 unsigned int n_fields; 267 unsigned int n_vars; 268 unsigned int key_size; 269 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 270 unsigned int n_sort_keys; 271 struct trace_event_file *event_file; 272 struct hist_trigger_attrs *attrs; 273 struct tracing_map *map; 274 bool enable_timestamps; 275 bool remove; 276 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 277 unsigned int n_var_refs; 278 279 struct action_data *actions[HIST_ACTIONS_MAX]; 280 unsigned int n_actions; 281 282 struct hist_field *synth_var_refs[SYNTH_FIELDS_MAX]; 283 unsigned int n_synth_var_refs; 284 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 285 unsigned int n_field_vars; 286 unsigned int n_field_var_str; 287 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 288 unsigned int n_field_var_hists; 289 290 struct field_var *max_vars[SYNTH_FIELDS_MAX]; 291 unsigned int n_max_vars; 292 unsigned int n_max_var_str; 293 }; 294 295 struct synth_field { 296 char *type; 297 char *name; 298 size_t size; 299 bool is_signed; 300 bool is_string; 301 }; 302 303 struct synth_event { 304 struct list_head list; 305 int ref; 306 char *name; 307 struct synth_field **fields; 308 unsigned int n_fields; 309 unsigned int n_u64; 310 struct trace_event_class class; 311 struct trace_event_call call; 312 struct tracepoint *tp; 313 }; 314 315 struct action_data; 316 317 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 318 struct tracing_map_elt *elt, void *rec, 319 struct ring_buffer_event *rbe, 320 struct action_data *data, u64 *var_ref_vals); 321 322 struct action_data { 323 action_fn_t fn; 324 unsigned int n_params; 325 char *params[SYNTH_FIELDS_MAX]; 326 327 union { 328 struct { 329 unsigned int var_ref_idx; 330 char *match_event; 331 char *match_event_system; 332 char *synth_event_name; 333 struct synth_event *synth_event; 334 } onmatch; 335 336 struct { 337 char *var_str; 338 char *fn_name; 339 unsigned int max_var_ref_idx; 340 struct hist_field *max_var; 341 struct hist_field *var; 342 } onmax; 343 }; 344 }; 345 346 347 static char last_hist_cmd[MAX_FILTER_STR_VAL]; 348 static char hist_err_str[MAX_FILTER_STR_VAL]; 349 350 static void last_cmd_set(char *str) 351 { 352 if (!str) 353 return; 354 355 strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1); 356 } 357 358 static void hist_err(char *str, char *var) 359 { 360 int maxlen = MAX_FILTER_STR_VAL - 1; 361 362 if (!str) 363 return; 364 365 if (strlen(hist_err_str)) 366 return; 367 368 if (!var) 369 var = ""; 370 371 if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen) 372 return; 373 374 strcat(hist_err_str, str); 375 strcat(hist_err_str, var); 376 } 377 378 static void hist_err_event(char *str, char *system, char *event, char *var) 379 { 380 char err[MAX_FILTER_STR_VAL]; 381 382 if (system && var) 383 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var); 384 else if (system) 385 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event); 386 else 387 strscpy(err, var, MAX_FILTER_STR_VAL); 388 389 hist_err(str, err); 390 } 391 392 static void hist_err_clear(void) 393 { 394 hist_err_str[0] = '\0'; 395 } 396 397 static bool have_hist_err(void) 398 { 399 if (strlen(hist_err_str)) 400 return true; 401 402 return false; 403 } 404 405 static LIST_HEAD(synth_event_list); 406 static DEFINE_MUTEX(synth_event_mutex); 407 408 struct synth_trace_event { 409 struct trace_entry ent; 410 u64 fields[]; 411 }; 412 413 static int synth_event_define_fields(struct trace_event_call *call) 414 { 415 struct synth_trace_event trace; 416 int offset = offsetof(typeof(trace), fields); 417 struct synth_event *event = call->data; 418 unsigned int i, size, n_u64; 419 char *name, *type; 420 bool is_signed; 421 int ret = 0; 422 423 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 424 size = event->fields[i]->size; 425 is_signed = event->fields[i]->is_signed; 426 type = event->fields[i]->type; 427 name = event->fields[i]->name; 428 ret = trace_define_field(call, type, name, offset, size, 429 is_signed, FILTER_OTHER); 430 if (ret) 431 break; 432 433 if (event->fields[i]->is_string) { 434 offset += STR_VAR_LEN_MAX; 435 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 436 } else { 437 offset += sizeof(u64); 438 n_u64++; 439 } 440 } 441 442 event->n_u64 = n_u64; 443 444 return ret; 445 } 446 447 static bool synth_field_signed(char *type) 448 { 449 if (strncmp(type, "u", 1) == 0) 450 return false; 451 452 return true; 453 } 454 455 static int synth_field_is_string(char *type) 456 { 457 if (strstr(type, "char[") != NULL) 458 return true; 459 460 return false; 461 } 462 463 static int synth_field_string_size(char *type) 464 { 465 char buf[4], *end, *start; 466 unsigned int len; 467 int size, err; 468 469 start = strstr(type, "char["); 470 if (start == NULL) 471 return -EINVAL; 472 start += strlen("char["); 473 474 end = strchr(type, ']'); 475 if (!end || end < start) 476 return -EINVAL; 477 478 len = end - start; 479 if (len > 3) 480 return -EINVAL; 481 482 strncpy(buf, start, len); 483 buf[len] = '\0'; 484 485 err = kstrtouint(buf, 0, &size); 486 if (err) 487 return err; 488 489 if (size > STR_VAR_LEN_MAX) 490 return -EINVAL; 491 492 return size; 493 } 494 495 static int synth_field_size(char *type) 496 { 497 int size = 0; 498 499 if (strcmp(type, "s64") == 0) 500 size = sizeof(s64); 501 else if (strcmp(type, "u64") == 0) 502 size = sizeof(u64); 503 else if (strcmp(type, "s32") == 0) 504 size = sizeof(s32); 505 else if (strcmp(type, "u32") == 0) 506 size = sizeof(u32); 507 else if (strcmp(type, "s16") == 0) 508 size = sizeof(s16); 509 else if (strcmp(type, "u16") == 0) 510 size = sizeof(u16); 511 else if (strcmp(type, "s8") == 0) 512 size = sizeof(s8); 513 else if (strcmp(type, "u8") == 0) 514 size = sizeof(u8); 515 else if (strcmp(type, "char") == 0) 516 size = sizeof(char); 517 else if (strcmp(type, "unsigned char") == 0) 518 size = sizeof(unsigned char); 519 else if (strcmp(type, "int") == 0) 520 size = sizeof(int); 521 else if (strcmp(type, "unsigned int") == 0) 522 size = sizeof(unsigned int); 523 else if (strcmp(type, "long") == 0) 524 size = sizeof(long); 525 else if (strcmp(type, "unsigned long") == 0) 526 size = sizeof(unsigned long); 527 else if (strcmp(type, "pid_t") == 0) 528 size = sizeof(pid_t); 529 else if (synth_field_is_string(type)) 530 size = synth_field_string_size(type); 531 532 return size; 533 } 534 535 static const char *synth_field_fmt(char *type) 536 { 537 const char *fmt = "%llu"; 538 539 if (strcmp(type, "s64") == 0) 540 fmt = "%lld"; 541 else if (strcmp(type, "u64") == 0) 542 fmt = "%llu"; 543 else if (strcmp(type, "s32") == 0) 544 fmt = "%d"; 545 else if (strcmp(type, "u32") == 0) 546 fmt = "%u"; 547 else if (strcmp(type, "s16") == 0) 548 fmt = "%d"; 549 else if (strcmp(type, "u16") == 0) 550 fmt = "%u"; 551 else if (strcmp(type, "s8") == 0) 552 fmt = "%d"; 553 else if (strcmp(type, "u8") == 0) 554 fmt = "%u"; 555 else if (strcmp(type, "char") == 0) 556 fmt = "%d"; 557 else if (strcmp(type, "unsigned char") == 0) 558 fmt = "%u"; 559 else if (strcmp(type, "int") == 0) 560 fmt = "%d"; 561 else if (strcmp(type, "unsigned int") == 0) 562 fmt = "%u"; 563 else if (strcmp(type, "long") == 0) 564 fmt = "%ld"; 565 else if (strcmp(type, "unsigned long") == 0) 566 fmt = "%lu"; 567 else if (strcmp(type, "pid_t") == 0) 568 fmt = "%d"; 569 else if (synth_field_is_string(type)) 570 fmt = "%s"; 571 572 return fmt; 573 } 574 575 static enum print_line_t print_synth_event(struct trace_iterator *iter, 576 int flags, 577 struct trace_event *event) 578 { 579 struct trace_array *tr = iter->tr; 580 struct trace_seq *s = &iter->seq; 581 struct synth_trace_event *entry; 582 struct synth_event *se; 583 unsigned int i, n_u64; 584 char print_fmt[32]; 585 const char *fmt; 586 587 entry = (struct synth_trace_event *)iter->ent; 588 se = container_of(event, struct synth_event, call.event); 589 590 trace_seq_printf(s, "%s: ", se->name); 591 592 for (i = 0, n_u64 = 0; i < se->n_fields; i++) { 593 if (trace_seq_has_overflowed(s)) 594 goto end; 595 596 fmt = synth_field_fmt(se->fields[i]->type); 597 598 /* parameter types */ 599 if (tr->trace_flags & TRACE_ITER_VERBOSE) 600 trace_seq_printf(s, "%s ", fmt); 601 602 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt); 603 604 /* parameter values */ 605 if (se->fields[i]->is_string) { 606 trace_seq_printf(s, print_fmt, se->fields[i]->name, 607 (char *)&entry->fields[n_u64], 608 i == se->n_fields - 1 ? "" : " "); 609 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 610 } else { 611 trace_seq_printf(s, print_fmt, se->fields[i]->name, 612 entry->fields[n_u64], 613 i == se->n_fields - 1 ? "" : " "); 614 n_u64++; 615 } 616 } 617 end: 618 trace_seq_putc(s, '\n'); 619 620 return trace_handle_return(s); 621 } 622 623 static struct trace_event_functions synth_event_funcs = { 624 .trace = print_synth_event 625 }; 626 627 static notrace void trace_event_raw_event_synth(void *__data, 628 u64 *var_ref_vals, 629 unsigned int var_ref_idx) 630 { 631 struct trace_event_file *trace_file = __data; 632 struct synth_trace_event *entry; 633 struct trace_event_buffer fbuffer; 634 struct ring_buffer *buffer; 635 struct synth_event *event; 636 unsigned int i, n_u64; 637 int fields_size = 0; 638 639 event = trace_file->event_call->data; 640 641 if (trace_trigger_soft_disabled(trace_file)) 642 return; 643 644 fields_size = event->n_u64 * sizeof(u64); 645 646 /* 647 * Avoid ring buffer recursion detection, as this event 648 * is being performed within another event. 649 */ 650 buffer = trace_file->tr->trace_buffer.buffer; 651 ring_buffer_nest_start(buffer); 652 653 entry = trace_event_buffer_reserve(&fbuffer, trace_file, 654 sizeof(*entry) + fields_size); 655 if (!entry) 656 goto out; 657 658 for (i = 0, n_u64 = 0; i < event->n_fields; i++) { 659 if (event->fields[i]->is_string) { 660 char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i]; 661 char *str_field = (char *)&entry->fields[n_u64]; 662 663 strscpy(str_field, str_val, STR_VAR_LEN_MAX); 664 n_u64 += STR_VAR_LEN_MAX / sizeof(u64); 665 } else { 666 entry->fields[n_u64] = var_ref_vals[var_ref_idx + i]; 667 n_u64++; 668 } 669 } 670 671 trace_event_buffer_commit(&fbuffer); 672 out: 673 ring_buffer_nest_end(buffer); 674 } 675 676 static void free_synth_event_print_fmt(struct trace_event_call *call) 677 { 678 if (call) { 679 kfree(call->print_fmt); 680 call->print_fmt = NULL; 681 } 682 } 683 684 static int __set_synth_event_print_fmt(struct synth_event *event, 685 char *buf, int len) 686 { 687 const char *fmt; 688 int pos = 0; 689 int i; 690 691 /* When len=0, we just calculate the needed length */ 692 #define LEN_OR_ZERO (len ? len - pos : 0) 693 694 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 695 for (i = 0; i < event->n_fields; i++) { 696 fmt = synth_field_fmt(event->fields[i]->type); 697 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s", 698 event->fields[i]->name, fmt, 699 i == event->n_fields - 1 ? "" : ", "); 700 } 701 pos += snprintf(buf + pos, LEN_OR_ZERO, "\""); 702 703 for (i = 0; i < event->n_fields; i++) { 704 pos += snprintf(buf + pos, LEN_OR_ZERO, 705 ", REC->%s", event->fields[i]->name); 706 } 707 708 #undef LEN_OR_ZERO 709 710 /* return the length of print_fmt */ 711 return pos; 712 } 713 714 static int set_synth_event_print_fmt(struct trace_event_call *call) 715 { 716 struct synth_event *event = call->data; 717 char *print_fmt; 718 int len; 719 720 /* First: called with 0 length to calculate the needed length */ 721 len = __set_synth_event_print_fmt(event, NULL, 0); 722 723 print_fmt = kmalloc(len + 1, GFP_KERNEL); 724 if (!print_fmt) 725 return -ENOMEM; 726 727 /* Second: actually write the @print_fmt */ 728 __set_synth_event_print_fmt(event, print_fmt, len + 1); 729 call->print_fmt = print_fmt; 730 731 return 0; 732 } 733 734 static void free_synth_field(struct synth_field *field) 735 { 736 kfree(field->type); 737 kfree(field->name); 738 kfree(field); 739 } 740 741 static struct synth_field *parse_synth_field(int argc, char **argv, 742 int *consumed) 743 { 744 struct synth_field *field; 745 const char *prefix = NULL; 746 char *field_type = argv[0], *field_name; 747 int len, ret = 0; 748 char *array; 749 750 if (field_type[0] == ';') 751 field_type++; 752 753 if (!strcmp(field_type, "unsigned")) { 754 if (argc < 3) 755 return ERR_PTR(-EINVAL); 756 prefix = "unsigned "; 757 field_type = argv[1]; 758 field_name = argv[2]; 759 *consumed = 3; 760 } else { 761 field_name = argv[1]; 762 *consumed = 2; 763 } 764 765 len = strlen(field_name); 766 if (field_name[len - 1] == ';') 767 field_name[len - 1] = '\0'; 768 769 field = kzalloc(sizeof(*field), GFP_KERNEL); 770 if (!field) 771 return ERR_PTR(-ENOMEM); 772 773 len = strlen(field_type) + 1; 774 array = strchr(field_name, '['); 775 if (array) 776 len += strlen(array); 777 if (prefix) 778 len += strlen(prefix); 779 field->type = kzalloc(len, GFP_KERNEL); 780 if (!field->type) { 781 ret = -ENOMEM; 782 goto free; 783 } 784 if (prefix) 785 strcat(field->type, prefix); 786 strcat(field->type, field_type); 787 if (array) { 788 strcat(field->type, array); 789 *array = '\0'; 790 } 791 792 field->size = synth_field_size(field->type); 793 if (!field->size) { 794 ret = -EINVAL; 795 goto free; 796 } 797 798 if (synth_field_is_string(field->type)) 799 field->is_string = true; 800 801 field->is_signed = synth_field_signed(field->type); 802 803 field->name = kstrdup(field_name, GFP_KERNEL); 804 if (!field->name) { 805 ret = -ENOMEM; 806 goto free; 807 } 808 out: 809 return field; 810 free: 811 free_synth_field(field); 812 field = ERR_PTR(ret); 813 goto out; 814 } 815 816 static void free_synth_tracepoint(struct tracepoint *tp) 817 { 818 if (!tp) 819 return; 820 821 kfree(tp->name); 822 kfree(tp); 823 } 824 825 static struct tracepoint *alloc_synth_tracepoint(char *name) 826 { 827 struct tracepoint *tp; 828 829 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 830 if (!tp) 831 return ERR_PTR(-ENOMEM); 832 833 tp->name = kstrdup(name, GFP_KERNEL); 834 if (!tp->name) { 835 kfree(tp); 836 return ERR_PTR(-ENOMEM); 837 } 838 839 return tp; 840 } 841 842 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 843 unsigned int var_ref_idx); 844 845 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 846 unsigned int var_ref_idx) 847 { 848 struct tracepoint *tp = event->tp; 849 850 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 851 struct tracepoint_func *probe_func_ptr; 852 synth_probe_func_t probe_func; 853 void *__data; 854 855 if (!(cpu_online(raw_smp_processor_id()))) 856 return; 857 858 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 859 if (probe_func_ptr) { 860 do { 861 probe_func = probe_func_ptr->func; 862 __data = probe_func_ptr->data; 863 probe_func(__data, var_ref_vals, var_ref_idx); 864 } while ((++probe_func_ptr)->func); 865 } 866 } 867 } 868 869 static struct synth_event *find_synth_event(const char *name) 870 { 871 struct synth_event *event; 872 873 list_for_each_entry(event, &synth_event_list, list) { 874 if (strcmp(event->name, name) == 0) 875 return event; 876 } 877 878 return NULL; 879 } 880 881 static int register_synth_event(struct synth_event *event) 882 { 883 struct trace_event_call *call = &event->call; 884 int ret = 0; 885 886 event->call.class = &event->class; 887 event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL); 888 if (!event->class.system) { 889 ret = -ENOMEM; 890 goto out; 891 } 892 893 event->tp = alloc_synth_tracepoint(event->name); 894 if (IS_ERR(event->tp)) { 895 ret = PTR_ERR(event->tp); 896 event->tp = NULL; 897 goto out; 898 } 899 900 INIT_LIST_HEAD(&call->class->fields); 901 call->event.funcs = &synth_event_funcs; 902 call->class->define_fields = synth_event_define_fields; 903 904 ret = register_trace_event(&call->event); 905 if (!ret) { 906 ret = -ENODEV; 907 goto out; 908 } 909 call->flags = TRACE_EVENT_FL_TRACEPOINT; 910 call->class->reg = trace_event_reg; 911 call->class->probe = trace_event_raw_event_synth; 912 call->data = event; 913 call->tp = event->tp; 914 915 ret = trace_add_event_call(call); 916 if (ret) { 917 pr_warn("Failed to register synthetic event: %s\n", 918 trace_event_name(call)); 919 goto err; 920 } 921 922 ret = set_synth_event_print_fmt(call); 923 if (ret < 0) { 924 trace_remove_event_call(call); 925 goto err; 926 } 927 out: 928 return ret; 929 err: 930 unregister_trace_event(&call->event); 931 goto out; 932 } 933 934 static int unregister_synth_event(struct synth_event *event) 935 { 936 struct trace_event_call *call = &event->call; 937 int ret; 938 939 ret = trace_remove_event_call(call); 940 941 return ret; 942 } 943 944 static void free_synth_event(struct synth_event *event) 945 { 946 unsigned int i; 947 948 if (!event) 949 return; 950 951 for (i = 0; i < event->n_fields; i++) 952 free_synth_field(event->fields[i]); 953 954 kfree(event->fields); 955 kfree(event->name); 956 kfree(event->class.system); 957 free_synth_tracepoint(event->tp); 958 free_synth_event_print_fmt(&event->call); 959 kfree(event); 960 } 961 962 static struct synth_event *alloc_synth_event(char *event_name, int n_fields, 963 struct synth_field **fields) 964 { 965 struct synth_event *event; 966 unsigned int i; 967 968 event = kzalloc(sizeof(*event), GFP_KERNEL); 969 if (!event) { 970 event = ERR_PTR(-ENOMEM); 971 goto out; 972 } 973 974 event->name = kstrdup(event_name, GFP_KERNEL); 975 if (!event->name) { 976 kfree(event); 977 event = ERR_PTR(-ENOMEM); 978 goto out; 979 } 980 981 event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL); 982 if (!event->fields) { 983 free_synth_event(event); 984 event = ERR_PTR(-ENOMEM); 985 goto out; 986 } 987 988 for (i = 0; i < n_fields; i++) 989 event->fields[i] = fields[i]; 990 991 event->n_fields = n_fields; 992 out: 993 return event; 994 } 995 996 static void action_trace(struct hist_trigger_data *hist_data, 997 struct tracing_map_elt *elt, void *rec, 998 struct ring_buffer_event *rbe, 999 struct action_data *data, u64 *var_ref_vals) 1000 { 1001 struct synth_event *event = data->onmatch.synth_event; 1002 1003 trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx); 1004 } 1005 1006 struct hist_var_data { 1007 struct list_head list; 1008 struct hist_trigger_data *hist_data; 1009 }; 1010 1011 static void add_or_delete_synth_event(struct synth_event *event, int delete) 1012 { 1013 if (delete) 1014 free_synth_event(event); 1015 else { 1016 mutex_lock(&synth_event_mutex); 1017 if (!find_synth_event(event->name)) 1018 list_add(&event->list, &synth_event_list); 1019 else 1020 free_synth_event(event); 1021 mutex_unlock(&synth_event_mutex); 1022 } 1023 } 1024 1025 static int create_synth_event(int argc, char **argv) 1026 { 1027 struct synth_field *field, *fields[SYNTH_FIELDS_MAX]; 1028 struct synth_event *event = NULL; 1029 bool delete_event = false; 1030 int i, consumed = 0, n_fields = 0, ret = 0; 1031 char *name; 1032 1033 mutex_lock(&synth_event_mutex); 1034 1035 /* 1036 * Argument syntax: 1037 * - Add synthetic event: <event_name> field[;field] ... 1038 * - Remove synthetic event: !<event_name> field[;field] ... 1039 * where 'field' = type field_name 1040 */ 1041 if (argc < 1) { 1042 ret = -EINVAL; 1043 goto out; 1044 } 1045 1046 name = argv[0]; 1047 if (name[0] == '!') { 1048 delete_event = true; 1049 name++; 1050 } 1051 1052 event = find_synth_event(name); 1053 if (event) { 1054 if (delete_event) { 1055 if (event->ref) { 1056 event = NULL; 1057 ret = -EBUSY; 1058 goto out; 1059 } 1060 list_del(&event->list); 1061 goto out; 1062 } 1063 event = NULL; 1064 ret = -EEXIST; 1065 goto out; 1066 } else if (delete_event) 1067 goto out; 1068 1069 if (argc < 2) { 1070 ret = -EINVAL; 1071 goto out; 1072 } 1073 1074 for (i = 1; i < argc - 1; i++) { 1075 if (strcmp(argv[i], ";") == 0) 1076 continue; 1077 if (n_fields == SYNTH_FIELDS_MAX) { 1078 ret = -EINVAL; 1079 goto err; 1080 } 1081 1082 field = parse_synth_field(argc - i, &argv[i], &consumed); 1083 if (IS_ERR(field)) { 1084 ret = PTR_ERR(field); 1085 goto err; 1086 } 1087 fields[n_fields++] = field; 1088 i += consumed - 1; 1089 } 1090 1091 if (i < argc && strcmp(argv[i], ";") != 0) { 1092 ret = -EINVAL; 1093 goto err; 1094 } 1095 1096 event = alloc_synth_event(name, n_fields, fields); 1097 if (IS_ERR(event)) { 1098 ret = PTR_ERR(event); 1099 event = NULL; 1100 goto err; 1101 } 1102 out: 1103 mutex_unlock(&synth_event_mutex); 1104 1105 if (event) { 1106 if (delete_event) { 1107 ret = unregister_synth_event(event); 1108 add_or_delete_synth_event(event, !ret); 1109 } else { 1110 ret = register_synth_event(event); 1111 add_or_delete_synth_event(event, ret); 1112 } 1113 } 1114 1115 return ret; 1116 err: 1117 mutex_unlock(&synth_event_mutex); 1118 1119 for (i = 0; i < n_fields; i++) 1120 free_synth_field(fields[i]); 1121 free_synth_event(event); 1122 1123 return ret; 1124 } 1125 1126 static int release_all_synth_events(void) 1127 { 1128 struct list_head release_events; 1129 struct synth_event *event, *e; 1130 int ret = 0; 1131 1132 INIT_LIST_HEAD(&release_events); 1133 1134 mutex_lock(&synth_event_mutex); 1135 1136 list_for_each_entry(event, &synth_event_list, list) { 1137 if (event->ref) { 1138 mutex_unlock(&synth_event_mutex); 1139 return -EBUSY; 1140 } 1141 } 1142 1143 list_splice_init(&event->list, &release_events); 1144 1145 mutex_unlock(&synth_event_mutex); 1146 1147 list_for_each_entry_safe(event, e, &release_events, list) { 1148 list_del(&event->list); 1149 1150 ret = unregister_synth_event(event); 1151 add_or_delete_synth_event(event, !ret); 1152 } 1153 1154 return ret; 1155 } 1156 1157 1158 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos) 1159 { 1160 mutex_lock(&synth_event_mutex); 1161 1162 return seq_list_start(&synth_event_list, *pos); 1163 } 1164 1165 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos) 1166 { 1167 return seq_list_next(v, &synth_event_list, pos); 1168 } 1169 1170 static void synth_events_seq_stop(struct seq_file *m, void *v) 1171 { 1172 mutex_unlock(&synth_event_mutex); 1173 } 1174 1175 static int synth_events_seq_show(struct seq_file *m, void *v) 1176 { 1177 struct synth_field *field; 1178 struct synth_event *event = v; 1179 unsigned int i; 1180 1181 seq_printf(m, "%s\t", event->name); 1182 1183 for (i = 0; i < event->n_fields; i++) { 1184 field = event->fields[i]; 1185 1186 /* parameter values */ 1187 seq_printf(m, "%s %s%s", field->type, field->name, 1188 i == event->n_fields - 1 ? "" : "; "); 1189 } 1190 1191 seq_putc(m, '\n'); 1192 1193 return 0; 1194 } 1195 1196 static const struct seq_operations synth_events_seq_op = { 1197 .start = synth_events_seq_start, 1198 .next = synth_events_seq_next, 1199 .stop = synth_events_seq_stop, 1200 .show = synth_events_seq_show 1201 }; 1202 1203 static int synth_events_open(struct inode *inode, struct file *file) 1204 { 1205 int ret; 1206 1207 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 1208 ret = release_all_synth_events(); 1209 if (ret < 0) 1210 return ret; 1211 } 1212 1213 return seq_open(file, &synth_events_seq_op); 1214 } 1215 1216 static ssize_t synth_events_write(struct file *file, 1217 const char __user *buffer, 1218 size_t count, loff_t *ppos) 1219 { 1220 return trace_parse_run_command(file, buffer, count, ppos, 1221 create_synth_event); 1222 } 1223 1224 static const struct file_operations synth_events_fops = { 1225 .open = synth_events_open, 1226 .write = synth_events_write, 1227 .read = seq_read, 1228 .llseek = seq_lseek, 1229 .release = seq_release, 1230 }; 1231 1232 static u64 hist_field_timestamp(struct hist_field *hist_field, 1233 struct tracing_map_elt *elt, 1234 struct ring_buffer_event *rbe, 1235 void *event) 1236 { 1237 struct hist_trigger_data *hist_data = hist_field->hist_data; 1238 struct trace_array *tr = hist_data->event_file->tr; 1239 1240 u64 ts = ring_buffer_event_time_stamp(rbe); 1241 1242 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 1243 ts = ns2usecs(ts); 1244 1245 return ts; 1246 } 1247 1248 static u64 hist_field_cpu(struct hist_field *hist_field, 1249 struct tracing_map_elt *elt, 1250 struct ring_buffer_event *rbe, 1251 void *event) 1252 { 1253 int cpu = smp_processor_id(); 1254 1255 return cpu; 1256 } 1257 1258 static struct hist_field * 1259 check_field_for_var_ref(struct hist_field *hist_field, 1260 struct hist_trigger_data *var_data, 1261 unsigned int var_idx) 1262 { 1263 struct hist_field *found = NULL; 1264 1265 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) { 1266 if (hist_field->var.idx == var_idx && 1267 hist_field->var.hist_data == var_data) { 1268 found = hist_field; 1269 } 1270 } 1271 1272 return found; 1273 } 1274 1275 static struct hist_field * 1276 check_field_for_var_refs(struct hist_trigger_data *hist_data, 1277 struct hist_field *hist_field, 1278 struct hist_trigger_data *var_data, 1279 unsigned int var_idx, 1280 unsigned int level) 1281 { 1282 struct hist_field *found = NULL; 1283 unsigned int i; 1284 1285 if (level > 3) 1286 return found; 1287 1288 if (!hist_field) 1289 return found; 1290 1291 found = check_field_for_var_ref(hist_field, var_data, var_idx); 1292 if (found) 1293 return found; 1294 1295 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1296 struct hist_field *operand; 1297 1298 operand = hist_field->operands[i]; 1299 found = check_field_for_var_refs(hist_data, operand, var_data, 1300 var_idx, level + 1); 1301 if (found) 1302 return found; 1303 } 1304 1305 return found; 1306 } 1307 1308 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 1309 struct hist_trigger_data *var_data, 1310 unsigned int var_idx) 1311 { 1312 struct hist_field *hist_field, *found = NULL; 1313 unsigned int i; 1314 1315 for_each_hist_field(i, hist_data) { 1316 hist_field = hist_data->fields[i]; 1317 found = check_field_for_var_refs(hist_data, hist_field, 1318 var_data, var_idx, 0); 1319 if (found) 1320 return found; 1321 } 1322 1323 for (i = 0; i < hist_data->n_synth_var_refs; i++) { 1324 hist_field = hist_data->synth_var_refs[i]; 1325 found = check_field_for_var_refs(hist_data, hist_field, 1326 var_data, var_idx, 0); 1327 if (found) 1328 return found; 1329 } 1330 1331 return found; 1332 } 1333 1334 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 1335 unsigned int var_idx) 1336 { 1337 struct trace_array *tr = hist_data->event_file->tr; 1338 struct hist_field *found = NULL; 1339 struct hist_var_data *var_data; 1340 1341 list_for_each_entry(var_data, &tr->hist_vars, list) { 1342 if (var_data->hist_data == hist_data) 1343 continue; 1344 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 1345 if (found) 1346 break; 1347 } 1348 1349 return found; 1350 } 1351 1352 static bool check_var_refs(struct hist_trigger_data *hist_data) 1353 { 1354 struct hist_field *field; 1355 bool found = false; 1356 int i; 1357 1358 for_each_hist_field(i, hist_data) { 1359 field = hist_data->fields[i]; 1360 if (field && field->flags & HIST_FIELD_FL_VAR) { 1361 if (find_any_var_ref(hist_data, field->var.idx)) { 1362 found = true; 1363 break; 1364 } 1365 } 1366 } 1367 1368 return found; 1369 } 1370 1371 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1372 { 1373 struct trace_array *tr = hist_data->event_file->tr; 1374 struct hist_var_data *var_data, *found = NULL; 1375 1376 list_for_each_entry(var_data, &tr->hist_vars, list) { 1377 if (var_data->hist_data == hist_data) { 1378 found = var_data; 1379 break; 1380 } 1381 } 1382 1383 return found; 1384 } 1385 1386 static bool field_has_hist_vars(struct hist_field *hist_field, 1387 unsigned int level) 1388 { 1389 int i; 1390 1391 if (level > 3) 1392 return false; 1393 1394 if (!hist_field) 1395 return false; 1396 1397 if (hist_field->flags & HIST_FIELD_FL_VAR || 1398 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1399 return true; 1400 1401 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1402 struct hist_field *operand; 1403 1404 operand = hist_field->operands[i]; 1405 if (field_has_hist_vars(operand, level + 1)) 1406 return true; 1407 } 1408 1409 return false; 1410 } 1411 1412 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1413 { 1414 struct hist_field *hist_field; 1415 int i; 1416 1417 for_each_hist_field(i, hist_data) { 1418 hist_field = hist_data->fields[i]; 1419 if (field_has_hist_vars(hist_field, 0)) 1420 return true; 1421 } 1422 1423 return false; 1424 } 1425 1426 static int save_hist_vars(struct hist_trigger_data *hist_data) 1427 { 1428 struct trace_array *tr = hist_data->event_file->tr; 1429 struct hist_var_data *var_data; 1430 1431 var_data = find_hist_vars(hist_data); 1432 if (var_data) 1433 return 0; 1434 1435 if (trace_array_get(tr) < 0) 1436 return -ENODEV; 1437 1438 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1439 if (!var_data) { 1440 trace_array_put(tr); 1441 return -ENOMEM; 1442 } 1443 1444 var_data->hist_data = hist_data; 1445 list_add(&var_data->list, &tr->hist_vars); 1446 1447 return 0; 1448 } 1449 1450 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1451 { 1452 struct trace_array *tr = hist_data->event_file->tr; 1453 struct hist_var_data *var_data; 1454 1455 var_data = find_hist_vars(hist_data); 1456 if (!var_data) 1457 return; 1458 1459 if (WARN_ON(check_var_refs(hist_data))) 1460 return; 1461 1462 list_del(&var_data->list); 1463 1464 kfree(var_data); 1465 1466 trace_array_put(tr); 1467 } 1468 1469 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1470 const char *var_name) 1471 { 1472 struct hist_field *hist_field, *found = NULL; 1473 int i; 1474 1475 for_each_hist_field(i, hist_data) { 1476 hist_field = hist_data->fields[i]; 1477 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1478 strcmp(hist_field->var.name, var_name) == 0) { 1479 found = hist_field; 1480 break; 1481 } 1482 } 1483 1484 return found; 1485 } 1486 1487 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1488 struct trace_event_file *file, 1489 const char *var_name) 1490 { 1491 struct hist_trigger_data *test_data; 1492 struct event_trigger_data *test; 1493 struct hist_field *hist_field; 1494 1495 hist_field = find_var_field(hist_data, var_name); 1496 if (hist_field) 1497 return hist_field; 1498 1499 list_for_each_entry_rcu(test, &file->triggers, list) { 1500 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1501 test_data = test->private_data; 1502 hist_field = find_var_field(test_data, var_name); 1503 if (hist_field) 1504 return hist_field; 1505 } 1506 } 1507 1508 return NULL; 1509 } 1510 1511 static struct trace_event_file *find_var_file(struct trace_array *tr, 1512 char *system, 1513 char *event_name, 1514 char *var_name) 1515 { 1516 struct hist_trigger_data *var_hist_data; 1517 struct hist_var_data *var_data; 1518 struct trace_event_file *file, *found = NULL; 1519 1520 if (system) 1521 return find_event_file(tr, system, event_name); 1522 1523 list_for_each_entry(var_data, &tr->hist_vars, list) { 1524 var_hist_data = var_data->hist_data; 1525 file = var_hist_data->event_file; 1526 if (file == found) 1527 continue; 1528 1529 if (find_var_field(var_hist_data, var_name)) { 1530 if (found) { 1531 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name); 1532 return NULL; 1533 } 1534 1535 found = file; 1536 } 1537 } 1538 1539 return found; 1540 } 1541 1542 static struct hist_field *find_file_var(struct trace_event_file *file, 1543 const char *var_name) 1544 { 1545 struct hist_trigger_data *test_data; 1546 struct event_trigger_data *test; 1547 struct hist_field *hist_field; 1548 1549 list_for_each_entry_rcu(test, &file->triggers, list) { 1550 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1551 test_data = test->private_data; 1552 hist_field = find_var_field(test_data, var_name); 1553 if (hist_field) 1554 return hist_field; 1555 } 1556 } 1557 1558 return NULL; 1559 } 1560 1561 static struct hist_field * 1562 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1563 { 1564 struct trace_array *tr = hist_data->event_file->tr; 1565 struct hist_field *hist_field, *found = NULL; 1566 struct trace_event_file *file; 1567 unsigned int i; 1568 1569 for (i = 0; i < hist_data->n_actions; i++) { 1570 struct action_data *data = hist_data->actions[i]; 1571 1572 if (data->fn == action_trace) { 1573 char *system = data->onmatch.match_event_system; 1574 char *event_name = data->onmatch.match_event; 1575 1576 file = find_var_file(tr, system, event_name, var_name); 1577 if (!file) 1578 continue; 1579 hist_field = find_file_var(file, var_name); 1580 if (hist_field) { 1581 if (found) { 1582 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name); 1583 return ERR_PTR(-EINVAL); 1584 } 1585 1586 found = hist_field; 1587 } 1588 } 1589 } 1590 return found; 1591 } 1592 1593 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1594 char *system, 1595 char *event_name, 1596 char *var_name) 1597 { 1598 struct trace_array *tr = hist_data->event_file->tr; 1599 struct hist_field *hist_field = NULL; 1600 struct trace_event_file *file; 1601 1602 if (!system || !event_name) { 1603 hist_field = find_match_var(hist_data, var_name); 1604 if (IS_ERR(hist_field)) 1605 return NULL; 1606 if (hist_field) 1607 return hist_field; 1608 } 1609 1610 file = find_var_file(tr, system, event_name, var_name); 1611 if (!file) 1612 return NULL; 1613 1614 hist_field = find_file_var(file, var_name); 1615 1616 return hist_field; 1617 } 1618 1619 struct hist_elt_data { 1620 char *comm; 1621 u64 *var_ref_vals; 1622 char *field_var_str[SYNTH_FIELDS_MAX]; 1623 }; 1624 1625 static u64 hist_field_var_ref(struct hist_field *hist_field, 1626 struct tracing_map_elt *elt, 1627 struct ring_buffer_event *rbe, 1628 void *event) 1629 { 1630 struct hist_elt_data *elt_data; 1631 u64 var_val = 0; 1632 1633 elt_data = elt->private_data; 1634 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1635 1636 return var_val; 1637 } 1638 1639 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1640 u64 *var_ref_vals, bool self) 1641 { 1642 struct hist_trigger_data *var_data; 1643 struct tracing_map_elt *var_elt; 1644 struct hist_field *hist_field; 1645 unsigned int i, var_idx; 1646 bool resolved = true; 1647 u64 var_val = 0; 1648 1649 for (i = 0; i < hist_data->n_var_refs; i++) { 1650 hist_field = hist_data->var_refs[i]; 1651 var_idx = hist_field->var.idx; 1652 var_data = hist_field->var.hist_data; 1653 1654 if (var_data == NULL) { 1655 resolved = false; 1656 break; 1657 } 1658 1659 if ((self && var_data != hist_data) || 1660 (!self && var_data == hist_data)) 1661 continue; 1662 1663 var_elt = tracing_map_lookup(var_data->map, key); 1664 if (!var_elt) { 1665 resolved = false; 1666 break; 1667 } 1668 1669 if (!tracing_map_var_set(var_elt, var_idx)) { 1670 resolved = false; 1671 break; 1672 } 1673 1674 if (self || !hist_field->read_once) 1675 var_val = tracing_map_read_var(var_elt, var_idx); 1676 else 1677 var_val = tracing_map_read_var_once(var_elt, var_idx); 1678 1679 var_ref_vals[i] = var_val; 1680 } 1681 1682 return resolved; 1683 } 1684 1685 static const char *hist_field_name(struct hist_field *field, 1686 unsigned int level) 1687 { 1688 const char *field_name = ""; 1689 1690 if (level > 1) 1691 return field_name; 1692 1693 if (field->field) 1694 field_name = field->field->name; 1695 else if (field->flags & HIST_FIELD_FL_LOG2 || 1696 field->flags & HIST_FIELD_FL_ALIAS) 1697 field_name = hist_field_name(field->operands[0], ++level); 1698 else if (field->flags & HIST_FIELD_FL_CPU) 1699 field_name = "cpu"; 1700 else if (field->flags & HIST_FIELD_FL_EXPR || 1701 field->flags & HIST_FIELD_FL_VAR_REF) { 1702 if (field->system) { 1703 static char full_name[MAX_FILTER_STR_VAL]; 1704 1705 strcat(full_name, field->system); 1706 strcat(full_name, "."); 1707 strcat(full_name, field->event_name); 1708 strcat(full_name, "."); 1709 strcat(full_name, field->name); 1710 field_name = full_name; 1711 } else 1712 field_name = field->name; 1713 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1714 field_name = "common_timestamp"; 1715 1716 if (field_name == NULL) 1717 field_name = ""; 1718 1719 return field_name; 1720 } 1721 1722 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1723 { 1724 hist_field_fn_t fn = NULL; 1725 1726 switch (field_size) { 1727 case 8: 1728 if (field_is_signed) 1729 fn = hist_field_s64; 1730 else 1731 fn = hist_field_u64; 1732 break; 1733 case 4: 1734 if (field_is_signed) 1735 fn = hist_field_s32; 1736 else 1737 fn = hist_field_u32; 1738 break; 1739 case 2: 1740 if (field_is_signed) 1741 fn = hist_field_s16; 1742 else 1743 fn = hist_field_u16; 1744 break; 1745 case 1: 1746 if (field_is_signed) 1747 fn = hist_field_s8; 1748 else 1749 fn = hist_field_u8; 1750 break; 1751 } 1752 1753 return fn; 1754 } 1755 1756 static int parse_map_size(char *str) 1757 { 1758 unsigned long size, map_bits; 1759 int ret; 1760 1761 strsep(&str, "="); 1762 if (!str) { 1763 ret = -EINVAL; 1764 goto out; 1765 } 1766 1767 ret = kstrtoul(str, 0, &size); 1768 if (ret) 1769 goto out; 1770 1771 map_bits = ilog2(roundup_pow_of_two(size)); 1772 if (map_bits < TRACING_MAP_BITS_MIN || 1773 map_bits > TRACING_MAP_BITS_MAX) 1774 ret = -EINVAL; 1775 else 1776 ret = map_bits; 1777 out: 1778 return ret; 1779 } 1780 1781 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1782 { 1783 unsigned int i; 1784 1785 if (!attrs) 1786 return; 1787 1788 for (i = 0; i < attrs->n_assignments; i++) 1789 kfree(attrs->assignment_str[i]); 1790 1791 for (i = 0; i < attrs->n_actions; i++) 1792 kfree(attrs->action_str[i]); 1793 1794 kfree(attrs->name); 1795 kfree(attrs->sort_key_str); 1796 kfree(attrs->keys_str); 1797 kfree(attrs->vals_str); 1798 kfree(attrs->clock); 1799 kfree(attrs); 1800 } 1801 1802 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1803 { 1804 int ret = -EINVAL; 1805 1806 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1807 return ret; 1808 1809 if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) || 1810 (strncmp(str, "onmax(", strlen("onmax(")) == 0)) { 1811 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1812 if (!attrs->action_str[attrs->n_actions]) { 1813 ret = -ENOMEM; 1814 return ret; 1815 } 1816 attrs->n_actions++; 1817 ret = 0; 1818 } 1819 1820 return ret; 1821 } 1822 1823 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs) 1824 { 1825 int ret = 0; 1826 1827 if ((strncmp(str, "key=", strlen("key=")) == 0) || 1828 (strncmp(str, "keys=", strlen("keys=")) == 0)) { 1829 attrs->keys_str = kstrdup(str, GFP_KERNEL); 1830 if (!attrs->keys_str) { 1831 ret = -ENOMEM; 1832 goto out; 1833 } 1834 } else if ((strncmp(str, "val=", strlen("val=")) == 0) || 1835 (strncmp(str, "vals=", strlen("vals=")) == 0) || 1836 (strncmp(str, "values=", strlen("values=")) == 0)) { 1837 attrs->vals_str = kstrdup(str, GFP_KERNEL); 1838 if (!attrs->vals_str) { 1839 ret = -ENOMEM; 1840 goto out; 1841 } 1842 } else if (strncmp(str, "sort=", strlen("sort=")) == 0) { 1843 attrs->sort_key_str = kstrdup(str, GFP_KERNEL); 1844 if (!attrs->sort_key_str) { 1845 ret = -ENOMEM; 1846 goto out; 1847 } 1848 } else if (strncmp(str, "name=", strlen("name=")) == 0) { 1849 attrs->name = kstrdup(str, GFP_KERNEL); 1850 if (!attrs->name) { 1851 ret = -ENOMEM; 1852 goto out; 1853 } 1854 } else if (strncmp(str, "clock=", strlen("clock=")) == 0) { 1855 strsep(&str, "="); 1856 if (!str) { 1857 ret = -EINVAL; 1858 goto out; 1859 } 1860 1861 str = strstrip(str); 1862 attrs->clock = kstrdup(str, GFP_KERNEL); 1863 if (!attrs->clock) { 1864 ret = -ENOMEM; 1865 goto out; 1866 } 1867 } else if (strncmp(str, "size=", strlen("size=")) == 0) { 1868 int map_bits = parse_map_size(str); 1869 1870 if (map_bits < 0) { 1871 ret = map_bits; 1872 goto out; 1873 } 1874 attrs->map_bits = map_bits; 1875 } else { 1876 char *assignment; 1877 1878 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1879 hist_err("Too many variables defined: ", str); 1880 ret = -EINVAL; 1881 goto out; 1882 } 1883 1884 assignment = kstrdup(str, GFP_KERNEL); 1885 if (!assignment) { 1886 ret = -ENOMEM; 1887 goto out; 1888 } 1889 1890 attrs->assignment_str[attrs->n_assignments++] = assignment; 1891 } 1892 out: 1893 return ret; 1894 } 1895 1896 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str) 1897 { 1898 struct hist_trigger_attrs *attrs; 1899 int ret = 0; 1900 1901 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1902 if (!attrs) 1903 return ERR_PTR(-ENOMEM); 1904 1905 while (trigger_str) { 1906 char *str = strsep(&trigger_str, ":"); 1907 1908 if (strchr(str, '=')) { 1909 ret = parse_assignment(str, attrs); 1910 if (ret) 1911 goto free; 1912 } else if (strcmp(str, "pause") == 0) 1913 attrs->pause = true; 1914 else if ((strcmp(str, "cont") == 0) || 1915 (strcmp(str, "continue") == 0)) 1916 attrs->cont = true; 1917 else if (strcmp(str, "clear") == 0) 1918 attrs->clear = true; 1919 else { 1920 ret = parse_action(str, attrs); 1921 if (ret) 1922 goto free; 1923 } 1924 } 1925 1926 if (!attrs->keys_str) { 1927 ret = -EINVAL; 1928 goto free; 1929 } 1930 1931 if (!attrs->clock) { 1932 attrs->clock = kstrdup("global", GFP_KERNEL); 1933 if (!attrs->clock) { 1934 ret = -ENOMEM; 1935 goto free; 1936 } 1937 } 1938 1939 return attrs; 1940 free: 1941 destroy_hist_trigger_attrs(attrs); 1942 1943 return ERR_PTR(ret); 1944 } 1945 1946 static inline void save_comm(char *comm, struct task_struct *task) 1947 { 1948 if (!task->pid) { 1949 strcpy(comm, "<idle>"); 1950 return; 1951 } 1952 1953 if (WARN_ON_ONCE(task->pid < 0)) { 1954 strcpy(comm, "<XXX>"); 1955 return; 1956 } 1957 1958 memcpy(comm, task->comm, TASK_COMM_LEN); 1959 } 1960 1961 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1962 { 1963 unsigned int i; 1964 1965 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 1966 kfree(elt_data->field_var_str[i]); 1967 1968 kfree(elt_data->comm); 1969 kfree(elt_data); 1970 } 1971 1972 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1973 { 1974 struct hist_elt_data *elt_data = elt->private_data; 1975 1976 hist_elt_data_free(elt_data); 1977 } 1978 1979 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1980 { 1981 struct hist_trigger_data *hist_data = elt->map->private_data; 1982 unsigned int size = TASK_COMM_LEN; 1983 struct hist_elt_data *elt_data; 1984 struct hist_field *key_field; 1985 unsigned int i, n_str; 1986 1987 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1988 if (!elt_data) 1989 return -ENOMEM; 1990 1991 for_each_hist_key_field(i, hist_data) { 1992 key_field = hist_data->fields[i]; 1993 1994 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1995 elt_data->comm = kzalloc(size, GFP_KERNEL); 1996 if (!elt_data->comm) { 1997 kfree(elt_data); 1998 return -ENOMEM; 1999 } 2000 break; 2001 } 2002 } 2003 2004 n_str = hist_data->n_field_var_str + hist_data->n_max_var_str; 2005 2006 size = STR_VAR_LEN_MAX; 2007 2008 for (i = 0; i < n_str; i++) { 2009 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 2010 if (!elt_data->field_var_str[i]) { 2011 hist_elt_data_free(elt_data); 2012 return -ENOMEM; 2013 } 2014 } 2015 2016 elt->private_data = elt_data; 2017 2018 return 0; 2019 } 2020 2021 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 2022 { 2023 struct hist_elt_data *elt_data = elt->private_data; 2024 2025 if (elt_data->comm) 2026 save_comm(elt_data->comm, current); 2027 } 2028 2029 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 2030 .elt_alloc = hist_trigger_elt_data_alloc, 2031 .elt_free = hist_trigger_elt_data_free, 2032 .elt_init = hist_trigger_elt_data_init, 2033 }; 2034 2035 static const char *get_hist_field_flags(struct hist_field *hist_field) 2036 { 2037 const char *flags_str = NULL; 2038 2039 if (hist_field->flags & HIST_FIELD_FL_HEX) 2040 flags_str = "hex"; 2041 else if (hist_field->flags & HIST_FIELD_FL_SYM) 2042 flags_str = "sym"; 2043 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 2044 flags_str = "sym-offset"; 2045 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 2046 flags_str = "execname"; 2047 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 2048 flags_str = "syscall"; 2049 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 2050 flags_str = "log2"; 2051 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2052 flags_str = "usecs"; 2053 2054 return flags_str; 2055 } 2056 2057 static void expr_field_str(struct hist_field *field, char *expr) 2058 { 2059 if (field->flags & HIST_FIELD_FL_VAR_REF) 2060 strcat(expr, "$"); 2061 2062 strcat(expr, hist_field_name(field, 0)); 2063 2064 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 2065 const char *flags_str = get_hist_field_flags(field); 2066 2067 if (flags_str) { 2068 strcat(expr, "."); 2069 strcat(expr, flags_str); 2070 } 2071 } 2072 } 2073 2074 static char *expr_str(struct hist_field *field, unsigned int level) 2075 { 2076 char *expr; 2077 2078 if (level > 1) 2079 return NULL; 2080 2081 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2082 if (!expr) 2083 return NULL; 2084 2085 if (!field->operands[0]) { 2086 expr_field_str(field, expr); 2087 return expr; 2088 } 2089 2090 if (field->operator == FIELD_OP_UNARY_MINUS) { 2091 char *subexpr; 2092 2093 strcat(expr, "-("); 2094 subexpr = expr_str(field->operands[0], ++level); 2095 if (!subexpr) { 2096 kfree(expr); 2097 return NULL; 2098 } 2099 strcat(expr, subexpr); 2100 strcat(expr, ")"); 2101 2102 kfree(subexpr); 2103 2104 return expr; 2105 } 2106 2107 expr_field_str(field->operands[0], expr); 2108 2109 switch (field->operator) { 2110 case FIELD_OP_MINUS: 2111 strcat(expr, "-"); 2112 break; 2113 case FIELD_OP_PLUS: 2114 strcat(expr, "+"); 2115 break; 2116 default: 2117 kfree(expr); 2118 return NULL; 2119 } 2120 2121 expr_field_str(field->operands[1], expr); 2122 2123 return expr; 2124 } 2125 2126 static int contains_operator(char *str) 2127 { 2128 enum field_op_id field_op = FIELD_OP_NONE; 2129 char *op; 2130 2131 op = strpbrk(str, "+-"); 2132 if (!op) 2133 return FIELD_OP_NONE; 2134 2135 switch (*op) { 2136 case '-': 2137 if (*str == '-') 2138 field_op = FIELD_OP_UNARY_MINUS; 2139 else 2140 field_op = FIELD_OP_MINUS; 2141 break; 2142 case '+': 2143 field_op = FIELD_OP_PLUS; 2144 break; 2145 default: 2146 break; 2147 } 2148 2149 return field_op; 2150 } 2151 2152 static void destroy_hist_field(struct hist_field *hist_field, 2153 unsigned int level) 2154 { 2155 unsigned int i; 2156 2157 if (level > 3) 2158 return; 2159 2160 if (!hist_field) 2161 return; 2162 2163 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 2164 destroy_hist_field(hist_field->operands[i], level + 1); 2165 2166 kfree(hist_field->var.name); 2167 kfree(hist_field->name); 2168 kfree(hist_field->type); 2169 2170 kfree(hist_field); 2171 } 2172 2173 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 2174 struct ftrace_event_field *field, 2175 unsigned long flags, 2176 char *var_name) 2177 { 2178 struct hist_field *hist_field; 2179 2180 if (field && is_function_field(field)) 2181 return NULL; 2182 2183 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2184 if (!hist_field) 2185 return NULL; 2186 2187 hist_field->hist_data = hist_data; 2188 2189 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 2190 goto out; /* caller will populate */ 2191 2192 if (flags & HIST_FIELD_FL_VAR_REF) { 2193 hist_field->fn = hist_field_var_ref; 2194 goto out; 2195 } 2196 2197 if (flags & HIST_FIELD_FL_HITCOUNT) { 2198 hist_field->fn = hist_field_counter; 2199 hist_field->size = sizeof(u64); 2200 hist_field->type = kstrdup("u64", GFP_KERNEL); 2201 if (!hist_field->type) 2202 goto free; 2203 goto out; 2204 } 2205 2206 if (flags & HIST_FIELD_FL_STACKTRACE) { 2207 hist_field->fn = hist_field_none; 2208 goto out; 2209 } 2210 2211 if (flags & HIST_FIELD_FL_LOG2) { 2212 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 2213 hist_field->fn = hist_field_log2; 2214 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2215 hist_field->size = hist_field->operands[0]->size; 2216 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 2217 if (!hist_field->type) 2218 goto free; 2219 goto out; 2220 } 2221 2222 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2223 hist_field->fn = hist_field_timestamp; 2224 hist_field->size = sizeof(u64); 2225 hist_field->type = kstrdup("u64", GFP_KERNEL); 2226 if (!hist_field->type) 2227 goto free; 2228 goto out; 2229 } 2230 2231 if (flags & HIST_FIELD_FL_CPU) { 2232 hist_field->fn = hist_field_cpu; 2233 hist_field->size = sizeof(int); 2234 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 2235 if (!hist_field->type) 2236 goto free; 2237 goto out; 2238 } 2239 2240 if (WARN_ON_ONCE(!field)) 2241 goto out; 2242 2243 if (is_string_field(field)) { 2244 flags |= HIST_FIELD_FL_STRING; 2245 2246 hist_field->size = MAX_FILTER_STR_VAL; 2247 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2248 if (!hist_field->type) 2249 goto free; 2250 2251 if (field->filter_type == FILTER_STATIC_STRING) 2252 hist_field->fn = hist_field_string; 2253 else if (field->filter_type == FILTER_DYN_STRING) 2254 hist_field->fn = hist_field_dynstring; 2255 else 2256 hist_field->fn = hist_field_pstring; 2257 } else { 2258 hist_field->size = field->size; 2259 hist_field->is_signed = field->is_signed; 2260 hist_field->type = kstrdup(field->type, GFP_KERNEL); 2261 if (!hist_field->type) 2262 goto free; 2263 2264 hist_field->fn = select_value_fn(field->size, 2265 field->is_signed); 2266 if (!hist_field->fn) { 2267 destroy_hist_field(hist_field, 0); 2268 return NULL; 2269 } 2270 } 2271 out: 2272 hist_field->field = field; 2273 hist_field->flags = flags; 2274 2275 if (var_name) { 2276 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2277 if (!hist_field->var.name) 2278 goto free; 2279 } 2280 2281 return hist_field; 2282 free: 2283 destroy_hist_field(hist_field, 0); 2284 return NULL; 2285 } 2286 2287 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2288 { 2289 unsigned int i; 2290 2291 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2292 if (hist_data->fields[i]) { 2293 destroy_hist_field(hist_data->fields[i], 0); 2294 hist_data->fields[i] = NULL; 2295 } 2296 } 2297 } 2298 2299 static int init_var_ref(struct hist_field *ref_field, 2300 struct hist_field *var_field, 2301 char *system, char *event_name) 2302 { 2303 int err = 0; 2304 2305 ref_field->var.idx = var_field->var.idx; 2306 ref_field->var.hist_data = var_field->hist_data; 2307 ref_field->size = var_field->size; 2308 ref_field->is_signed = var_field->is_signed; 2309 ref_field->flags |= var_field->flags & 2310 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2311 2312 if (system) { 2313 ref_field->system = kstrdup(system, GFP_KERNEL); 2314 if (!ref_field->system) 2315 return -ENOMEM; 2316 } 2317 2318 if (event_name) { 2319 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2320 if (!ref_field->event_name) { 2321 err = -ENOMEM; 2322 goto free; 2323 } 2324 } 2325 2326 if (var_field->var.name) { 2327 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2328 if (!ref_field->name) { 2329 err = -ENOMEM; 2330 goto free; 2331 } 2332 } else if (var_field->name) { 2333 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2334 if (!ref_field->name) { 2335 err = -ENOMEM; 2336 goto free; 2337 } 2338 } 2339 2340 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 2341 if (!ref_field->type) { 2342 err = -ENOMEM; 2343 goto free; 2344 } 2345 out: 2346 return err; 2347 free: 2348 kfree(ref_field->system); 2349 kfree(ref_field->event_name); 2350 kfree(ref_field->name); 2351 2352 goto out; 2353 } 2354 2355 static struct hist_field *create_var_ref(struct hist_field *var_field, 2356 char *system, char *event_name) 2357 { 2358 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2359 struct hist_field *ref_field; 2360 2361 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2362 if (ref_field) { 2363 if (init_var_ref(ref_field, var_field, system, event_name)) { 2364 destroy_hist_field(ref_field, 0); 2365 return NULL; 2366 } 2367 } 2368 2369 return ref_field; 2370 } 2371 2372 static bool is_var_ref(char *var_name) 2373 { 2374 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2375 return false; 2376 2377 return true; 2378 } 2379 2380 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2381 char *var_name) 2382 { 2383 char *name, *field; 2384 unsigned int i; 2385 2386 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2387 name = hist_data->attrs->var_defs.name[i]; 2388 2389 if (strcmp(var_name, name) == 0) { 2390 field = hist_data->attrs->var_defs.expr[i]; 2391 if (contains_operator(field) || is_var_ref(field)) 2392 continue; 2393 return field; 2394 } 2395 } 2396 2397 return NULL; 2398 } 2399 2400 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2401 char *system, char *event_name, 2402 char *var_name) 2403 { 2404 struct trace_event_call *call; 2405 2406 if (system && event_name) { 2407 call = hist_data->event_file->event_call; 2408 2409 if (strcmp(system, call->class->system) != 0) 2410 return NULL; 2411 2412 if (strcmp(event_name, trace_event_name(call)) != 0) 2413 return NULL; 2414 } 2415 2416 if (!!system != !!event_name) 2417 return NULL; 2418 2419 if (!is_var_ref(var_name)) 2420 return NULL; 2421 2422 var_name++; 2423 2424 return field_name_from_var(hist_data, var_name); 2425 } 2426 2427 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2428 char *system, char *event_name, 2429 char *var_name) 2430 { 2431 struct hist_field *var_field = NULL, *ref_field = NULL; 2432 2433 if (!is_var_ref(var_name)) 2434 return NULL; 2435 2436 var_name++; 2437 2438 var_field = find_event_var(hist_data, system, event_name, var_name); 2439 if (var_field) 2440 ref_field = create_var_ref(var_field, system, event_name); 2441 2442 if (!ref_field) 2443 hist_err_event("Couldn't find variable: $", 2444 system, event_name, var_name); 2445 2446 return ref_field; 2447 } 2448 2449 static struct ftrace_event_field * 2450 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2451 char *field_str, unsigned long *flags) 2452 { 2453 struct ftrace_event_field *field = NULL; 2454 char *field_name, *modifier, *str; 2455 2456 modifier = str = kstrdup(field_str, GFP_KERNEL); 2457 if (!modifier) 2458 return ERR_PTR(-ENOMEM); 2459 2460 field_name = strsep(&modifier, "."); 2461 if (modifier) { 2462 if (strcmp(modifier, "hex") == 0) 2463 *flags |= HIST_FIELD_FL_HEX; 2464 else if (strcmp(modifier, "sym") == 0) 2465 *flags |= HIST_FIELD_FL_SYM; 2466 else if (strcmp(modifier, "sym-offset") == 0) 2467 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2468 else if ((strcmp(modifier, "execname") == 0) && 2469 (strcmp(field_name, "common_pid") == 0)) 2470 *flags |= HIST_FIELD_FL_EXECNAME; 2471 else if (strcmp(modifier, "syscall") == 0) 2472 *flags |= HIST_FIELD_FL_SYSCALL; 2473 else if (strcmp(modifier, "log2") == 0) 2474 *flags |= HIST_FIELD_FL_LOG2; 2475 else if (strcmp(modifier, "usecs") == 0) 2476 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2477 else { 2478 hist_err("Invalid field modifier: ", modifier); 2479 field = ERR_PTR(-EINVAL); 2480 goto out; 2481 } 2482 } 2483 2484 if (strcmp(field_name, "common_timestamp") == 0) { 2485 *flags |= HIST_FIELD_FL_TIMESTAMP; 2486 hist_data->enable_timestamps = true; 2487 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2488 hist_data->attrs->ts_in_usecs = true; 2489 } else if (strcmp(field_name, "cpu") == 0) 2490 *flags |= HIST_FIELD_FL_CPU; 2491 else { 2492 field = trace_find_event_field(file->event_call, field_name); 2493 if (!field || !field->size) { 2494 hist_err("Couldn't find field: ", field_name); 2495 field = ERR_PTR(-EINVAL); 2496 goto out; 2497 } 2498 } 2499 out: 2500 kfree(str); 2501 2502 return field; 2503 } 2504 2505 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2506 struct hist_field *var_ref, 2507 char *var_name) 2508 { 2509 struct hist_field *alias = NULL; 2510 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2511 2512 alias = create_hist_field(hist_data, NULL, flags, var_name); 2513 if (!alias) 2514 return NULL; 2515 2516 alias->fn = var_ref->fn; 2517 alias->operands[0] = var_ref; 2518 2519 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2520 destroy_hist_field(alias, 0); 2521 return NULL; 2522 } 2523 2524 return alias; 2525 } 2526 2527 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2528 struct trace_event_file *file, char *str, 2529 unsigned long *flags, char *var_name) 2530 { 2531 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2532 struct ftrace_event_field *field = NULL; 2533 struct hist_field *hist_field = NULL; 2534 int ret = 0; 2535 2536 s = strchr(str, '.'); 2537 if (s) { 2538 s = strchr(++s, '.'); 2539 if (s) { 2540 ref_system = strsep(&str, "."); 2541 if (!str) { 2542 ret = -EINVAL; 2543 goto out; 2544 } 2545 ref_event = strsep(&str, "."); 2546 if (!str) { 2547 ret = -EINVAL; 2548 goto out; 2549 } 2550 ref_var = str; 2551 } 2552 } 2553 2554 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2555 if (!s) { 2556 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var); 2557 if (hist_field) { 2558 hist_data->var_refs[hist_data->n_var_refs] = hist_field; 2559 hist_field->var_ref_idx = hist_data->n_var_refs++; 2560 if (var_name) { 2561 hist_field = create_alias(hist_data, hist_field, var_name); 2562 if (!hist_field) { 2563 ret = -ENOMEM; 2564 goto out; 2565 } 2566 } 2567 return hist_field; 2568 } 2569 } else 2570 str = s; 2571 2572 field = parse_field(hist_data, file, str, flags); 2573 if (IS_ERR(field)) { 2574 ret = PTR_ERR(field); 2575 goto out; 2576 } 2577 2578 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2579 if (!hist_field) { 2580 ret = -ENOMEM; 2581 goto out; 2582 } 2583 2584 return hist_field; 2585 out: 2586 return ERR_PTR(ret); 2587 } 2588 2589 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2590 struct trace_event_file *file, 2591 char *str, unsigned long flags, 2592 char *var_name, unsigned int level); 2593 2594 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2595 struct trace_event_file *file, 2596 char *str, unsigned long flags, 2597 char *var_name, unsigned int level) 2598 { 2599 struct hist_field *operand1, *expr = NULL; 2600 unsigned long operand_flags; 2601 int ret = 0; 2602 char *s; 2603 2604 /* we support only -(xxx) i.e. explicit parens required */ 2605 2606 if (level > 3) { 2607 hist_err("Too many subexpressions (3 max): ", str); 2608 ret = -EINVAL; 2609 goto free; 2610 } 2611 2612 str++; /* skip leading '-' */ 2613 2614 s = strchr(str, '('); 2615 if (s) 2616 str++; 2617 else { 2618 ret = -EINVAL; 2619 goto free; 2620 } 2621 2622 s = strrchr(str, ')'); 2623 if (s) 2624 *s = '\0'; 2625 else { 2626 ret = -EINVAL; /* no closing ')' */ 2627 goto free; 2628 } 2629 2630 flags |= HIST_FIELD_FL_EXPR; 2631 expr = create_hist_field(hist_data, NULL, flags, var_name); 2632 if (!expr) { 2633 ret = -ENOMEM; 2634 goto free; 2635 } 2636 2637 operand_flags = 0; 2638 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2639 if (IS_ERR(operand1)) { 2640 ret = PTR_ERR(operand1); 2641 goto free; 2642 } 2643 2644 expr->flags |= operand1->flags & 2645 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2646 expr->fn = hist_field_unary_minus; 2647 expr->operands[0] = operand1; 2648 expr->operator = FIELD_OP_UNARY_MINUS; 2649 expr->name = expr_str(expr, 0); 2650 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2651 if (!expr->type) { 2652 ret = -ENOMEM; 2653 goto free; 2654 } 2655 2656 return expr; 2657 free: 2658 destroy_hist_field(expr, 0); 2659 return ERR_PTR(ret); 2660 } 2661 2662 static int check_expr_operands(struct hist_field *operand1, 2663 struct hist_field *operand2) 2664 { 2665 unsigned long operand1_flags = operand1->flags; 2666 unsigned long operand2_flags = operand2->flags; 2667 2668 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2669 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2670 struct hist_field *var; 2671 2672 var = find_var_field(operand1->var.hist_data, operand1->name); 2673 if (!var) 2674 return -EINVAL; 2675 operand1_flags = var->flags; 2676 } 2677 2678 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2679 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2680 struct hist_field *var; 2681 2682 var = find_var_field(operand2->var.hist_data, operand2->name); 2683 if (!var) 2684 return -EINVAL; 2685 operand2_flags = var->flags; 2686 } 2687 2688 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2689 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2690 hist_err("Timestamp units in expression don't match", NULL); 2691 return -EINVAL; 2692 } 2693 2694 return 0; 2695 } 2696 2697 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2698 struct trace_event_file *file, 2699 char *str, unsigned long flags, 2700 char *var_name, unsigned int level) 2701 { 2702 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2703 unsigned long operand_flags; 2704 int field_op, ret = -EINVAL; 2705 char *sep, *operand1_str; 2706 2707 if (level > 3) { 2708 hist_err("Too many subexpressions (3 max): ", str); 2709 return ERR_PTR(-EINVAL); 2710 } 2711 2712 field_op = contains_operator(str); 2713 2714 if (field_op == FIELD_OP_NONE) 2715 return parse_atom(hist_data, file, str, &flags, var_name); 2716 2717 if (field_op == FIELD_OP_UNARY_MINUS) 2718 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2719 2720 switch (field_op) { 2721 case FIELD_OP_MINUS: 2722 sep = "-"; 2723 break; 2724 case FIELD_OP_PLUS: 2725 sep = "+"; 2726 break; 2727 default: 2728 goto free; 2729 } 2730 2731 operand1_str = strsep(&str, sep); 2732 if (!operand1_str || !str) 2733 goto free; 2734 2735 operand_flags = 0; 2736 operand1 = parse_atom(hist_data, file, operand1_str, 2737 &operand_flags, NULL); 2738 if (IS_ERR(operand1)) { 2739 ret = PTR_ERR(operand1); 2740 operand1 = NULL; 2741 goto free; 2742 } 2743 2744 /* rest of string could be another expression e.g. b+c in a+b+c */ 2745 operand_flags = 0; 2746 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2747 if (IS_ERR(operand2)) { 2748 ret = PTR_ERR(operand2); 2749 operand2 = NULL; 2750 goto free; 2751 } 2752 2753 ret = check_expr_operands(operand1, operand2); 2754 if (ret) 2755 goto free; 2756 2757 flags |= HIST_FIELD_FL_EXPR; 2758 2759 flags |= operand1->flags & 2760 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2761 2762 expr = create_hist_field(hist_data, NULL, flags, var_name); 2763 if (!expr) { 2764 ret = -ENOMEM; 2765 goto free; 2766 } 2767 2768 operand1->read_once = true; 2769 operand2->read_once = true; 2770 2771 expr->operands[0] = operand1; 2772 expr->operands[1] = operand2; 2773 expr->operator = field_op; 2774 expr->name = expr_str(expr, 0); 2775 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2776 if (!expr->type) { 2777 ret = -ENOMEM; 2778 goto free; 2779 } 2780 2781 switch (field_op) { 2782 case FIELD_OP_MINUS: 2783 expr->fn = hist_field_minus; 2784 break; 2785 case FIELD_OP_PLUS: 2786 expr->fn = hist_field_plus; 2787 break; 2788 default: 2789 ret = -EINVAL; 2790 goto free; 2791 } 2792 2793 return expr; 2794 free: 2795 destroy_hist_field(operand1, 0); 2796 destroy_hist_field(operand2, 0); 2797 destroy_hist_field(expr, 0); 2798 2799 return ERR_PTR(ret); 2800 } 2801 2802 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2803 struct trace_event_file *file) 2804 { 2805 struct event_trigger_data *test; 2806 2807 list_for_each_entry_rcu(test, &file->triggers, list) { 2808 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2809 if (test->private_data == hist_data) 2810 return test->filter_str; 2811 } 2812 } 2813 2814 return NULL; 2815 } 2816 2817 static struct event_command trigger_hist_cmd; 2818 static int event_hist_trigger_func(struct event_command *cmd_ops, 2819 struct trace_event_file *file, 2820 char *glob, char *cmd, char *param); 2821 2822 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2823 struct hist_trigger_data *hist_data, 2824 unsigned int n_keys) 2825 { 2826 struct hist_field *target_hist_field, *hist_field; 2827 unsigned int n, i, j; 2828 2829 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2830 return false; 2831 2832 i = hist_data->n_vals; 2833 j = target_hist_data->n_vals; 2834 2835 for (n = 0; n < n_keys; n++) { 2836 hist_field = hist_data->fields[i + n]; 2837 target_hist_field = target_hist_data->fields[j + n]; 2838 2839 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2840 return false; 2841 if (hist_field->size != target_hist_field->size) 2842 return false; 2843 if (hist_field->is_signed != target_hist_field->is_signed) 2844 return false; 2845 } 2846 2847 return true; 2848 } 2849 2850 static struct hist_trigger_data * 2851 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2852 struct trace_event_file *file) 2853 { 2854 struct hist_trigger_data *hist_data; 2855 struct event_trigger_data *test; 2856 unsigned int n_keys; 2857 2858 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2859 2860 list_for_each_entry_rcu(test, &file->triggers, list) { 2861 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2862 hist_data = test->private_data; 2863 2864 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2865 return hist_data; 2866 } 2867 } 2868 2869 return NULL; 2870 } 2871 2872 static struct trace_event_file *event_file(struct trace_array *tr, 2873 char *system, char *event_name) 2874 { 2875 struct trace_event_file *file; 2876 2877 file = __find_event_file(tr, system, event_name); 2878 if (!file) 2879 return ERR_PTR(-EINVAL); 2880 2881 return file; 2882 } 2883 2884 static struct hist_field * 2885 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2886 char *system, char *event_name, char *field_name) 2887 { 2888 struct hist_field *event_var; 2889 char *synthetic_name; 2890 2891 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2892 if (!synthetic_name) 2893 return ERR_PTR(-ENOMEM); 2894 2895 strcpy(synthetic_name, "synthetic_"); 2896 strcat(synthetic_name, field_name); 2897 2898 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2899 2900 kfree(synthetic_name); 2901 2902 return event_var; 2903 } 2904 2905 /** 2906 * create_field_var_hist - Automatically create a histogram and var for a field 2907 * @target_hist_data: The target hist trigger 2908 * @subsys_name: Optional subsystem name 2909 * @event_name: Optional event name 2910 * @field_name: The name of the field (and the resulting variable) 2911 * 2912 * Hist trigger actions fetch data from variables, not directly from 2913 * events. However, for convenience, users are allowed to directly 2914 * specify an event field in an action, which will be automatically 2915 * converted into a variable on their behalf. 2916 2917 * If a user specifies a field on an event that isn't the event the 2918 * histogram currently being defined (the target event histogram), the 2919 * only way that can be accomplished is if a new hist trigger is 2920 * created and the field variable defined on that. 2921 * 2922 * This function creates a new histogram compatible with the target 2923 * event (meaning a histogram with the same key as the target 2924 * histogram), and creates a variable for the specified field, but 2925 * with 'synthetic_' prepended to the variable name in order to avoid 2926 * collision with normal field variables. 2927 * 2928 * Return: The variable created for the field. 2929 */ 2930 static struct hist_field * 2931 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2932 char *subsys_name, char *event_name, char *field_name) 2933 { 2934 struct trace_array *tr = target_hist_data->event_file->tr; 2935 struct hist_field *event_var = ERR_PTR(-EINVAL); 2936 struct hist_trigger_data *hist_data; 2937 unsigned int i, n, first = true; 2938 struct field_var_hist *var_hist; 2939 struct trace_event_file *file; 2940 struct hist_field *key_field; 2941 char *saved_filter; 2942 char *cmd; 2943 int ret; 2944 2945 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2946 hist_err_event("onmatch: Too many field variables defined: ", 2947 subsys_name, event_name, field_name); 2948 return ERR_PTR(-EINVAL); 2949 } 2950 2951 file = event_file(tr, subsys_name, event_name); 2952 2953 if (IS_ERR(file)) { 2954 hist_err_event("onmatch: Event file not found: ", 2955 subsys_name, event_name, field_name); 2956 ret = PTR_ERR(file); 2957 return ERR_PTR(ret); 2958 } 2959 2960 /* 2961 * Look for a histogram compatible with target. We'll use the 2962 * found histogram specification to create a new matching 2963 * histogram with our variable on it. target_hist_data is not 2964 * yet a registered histogram so we can't use that. 2965 */ 2966 hist_data = find_compatible_hist(target_hist_data, file); 2967 if (!hist_data) { 2968 hist_err_event("onmatch: Matching event histogram not found: ", 2969 subsys_name, event_name, field_name); 2970 return ERR_PTR(-EINVAL); 2971 } 2972 2973 /* See if a synthetic field variable has already been created */ 2974 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2975 event_name, field_name); 2976 if (!IS_ERR_OR_NULL(event_var)) 2977 return event_var; 2978 2979 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 2980 if (!var_hist) 2981 return ERR_PTR(-ENOMEM); 2982 2983 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2984 if (!cmd) { 2985 kfree(var_hist); 2986 return ERR_PTR(-ENOMEM); 2987 } 2988 2989 /* Use the same keys as the compatible histogram */ 2990 strcat(cmd, "keys="); 2991 2992 for_each_hist_key_field(i, hist_data) { 2993 key_field = hist_data->fields[i]; 2994 if (!first) 2995 strcat(cmd, ","); 2996 strcat(cmd, key_field->field->name); 2997 first = false; 2998 } 2999 3000 /* Create the synthetic field variable specification */ 3001 strcat(cmd, ":synthetic_"); 3002 strcat(cmd, field_name); 3003 strcat(cmd, "="); 3004 strcat(cmd, field_name); 3005 3006 /* Use the same filter as the compatible histogram */ 3007 saved_filter = find_trigger_filter(hist_data, file); 3008 if (saved_filter) { 3009 strcat(cmd, " if "); 3010 strcat(cmd, saved_filter); 3011 } 3012 3013 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3014 if (!var_hist->cmd) { 3015 kfree(cmd); 3016 kfree(var_hist); 3017 return ERR_PTR(-ENOMEM); 3018 } 3019 3020 /* Save the compatible histogram information */ 3021 var_hist->hist_data = hist_data; 3022 3023 /* Create the new histogram with our variable */ 3024 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 3025 "", "hist", cmd); 3026 if (ret) { 3027 kfree(cmd); 3028 kfree(var_hist->cmd); 3029 kfree(var_hist); 3030 hist_err_event("onmatch: Couldn't create histogram for field: ", 3031 subsys_name, event_name, field_name); 3032 return ERR_PTR(ret); 3033 } 3034 3035 kfree(cmd); 3036 3037 /* If we can't find the variable, something went wrong */ 3038 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3039 event_name, field_name); 3040 if (IS_ERR_OR_NULL(event_var)) { 3041 kfree(var_hist->cmd); 3042 kfree(var_hist); 3043 hist_err_event("onmatch: Couldn't find synthetic variable: ", 3044 subsys_name, event_name, field_name); 3045 return ERR_PTR(-EINVAL); 3046 } 3047 3048 n = target_hist_data->n_field_var_hists; 3049 target_hist_data->field_var_hists[n] = var_hist; 3050 target_hist_data->n_field_var_hists++; 3051 3052 return event_var; 3053 } 3054 3055 static struct hist_field * 3056 find_target_event_var(struct hist_trigger_data *hist_data, 3057 char *subsys_name, char *event_name, char *var_name) 3058 { 3059 struct trace_event_file *file = hist_data->event_file; 3060 struct hist_field *hist_field = NULL; 3061 3062 if (subsys_name) { 3063 struct trace_event_call *call; 3064 3065 if (!event_name) 3066 return NULL; 3067 3068 call = file->event_call; 3069 3070 if (strcmp(subsys_name, call->class->system) != 0) 3071 return NULL; 3072 3073 if (strcmp(event_name, trace_event_name(call)) != 0) 3074 return NULL; 3075 } 3076 3077 hist_field = find_var_field(hist_data, var_name); 3078 3079 return hist_field; 3080 } 3081 3082 static inline void __update_field_vars(struct tracing_map_elt *elt, 3083 struct ring_buffer_event *rbe, 3084 void *rec, 3085 struct field_var **field_vars, 3086 unsigned int n_field_vars, 3087 unsigned int field_var_str_start) 3088 { 3089 struct hist_elt_data *elt_data = elt->private_data; 3090 unsigned int i, j, var_idx; 3091 u64 var_val; 3092 3093 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3094 struct field_var *field_var = field_vars[i]; 3095 struct hist_field *var = field_var->var; 3096 struct hist_field *val = field_var->val; 3097 3098 var_val = val->fn(val, elt, rbe, rec); 3099 var_idx = var->var.idx; 3100 3101 if (val->flags & HIST_FIELD_FL_STRING) { 3102 char *str = elt_data->field_var_str[j++]; 3103 char *val_str = (char *)(uintptr_t)var_val; 3104 3105 strscpy(str, val_str, STR_VAR_LEN_MAX); 3106 var_val = (u64)(uintptr_t)str; 3107 } 3108 tracing_map_set_var(elt, var_idx, var_val); 3109 } 3110 } 3111 3112 static void update_field_vars(struct hist_trigger_data *hist_data, 3113 struct tracing_map_elt *elt, 3114 struct ring_buffer_event *rbe, 3115 void *rec) 3116 { 3117 __update_field_vars(elt, rbe, rec, hist_data->field_vars, 3118 hist_data->n_field_vars, 0); 3119 } 3120 3121 static void update_max_vars(struct hist_trigger_data *hist_data, 3122 struct tracing_map_elt *elt, 3123 struct ring_buffer_event *rbe, 3124 void *rec) 3125 { 3126 __update_field_vars(elt, rbe, rec, hist_data->max_vars, 3127 hist_data->n_max_vars, hist_data->n_field_var_str); 3128 } 3129 3130 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3131 struct trace_event_file *file, 3132 char *name, int size, const char *type) 3133 { 3134 struct hist_field *var; 3135 int idx; 3136 3137 if (find_var(hist_data, file, name) && !hist_data->remove) { 3138 var = ERR_PTR(-EINVAL); 3139 goto out; 3140 } 3141 3142 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3143 if (!var) { 3144 var = ERR_PTR(-ENOMEM); 3145 goto out; 3146 } 3147 3148 idx = tracing_map_add_var(hist_data->map); 3149 if (idx < 0) { 3150 kfree(var); 3151 var = ERR_PTR(-EINVAL); 3152 goto out; 3153 } 3154 3155 var->flags = HIST_FIELD_FL_VAR; 3156 var->var.idx = idx; 3157 var->var.hist_data = var->hist_data = hist_data; 3158 var->size = size; 3159 var->var.name = kstrdup(name, GFP_KERNEL); 3160 var->type = kstrdup(type, GFP_KERNEL); 3161 if (!var->var.name || !var->type) { 3162 kfree(var->var.name); 3163 kfree(var->type); 3164 kfree(var); 3165 var = ERR_PTR(-ENOMEM); 3166 } 3167 out: 3168 return var; 3169 } 3170 3171 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3172 struct trace_event_file *file, 3173 char *field_name) 3174 { 3175 struct hist_field *val = NULL, *var = NULL; 3176 unsigned long flags = HIST_FIELD_FL_VAR; 3177 struct field_var *field_var; 3178 int ret = 0; 3179 3180 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3181 hist_err("Too many field variables defined: ", field_name); 3182 ret = -EINVAL; 3183 goto err; 3184 } 3185 3186 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3187 if (IS_ERR(val)) { 3188 hist_err("Couldn't parse field variable: ", field_name); 3189 ret = PTR_ERR(val); 3190 goto err; 3191 } 3192 3193 var = create_var(hist_data, file, field_name, val->size, val->type); 3194 if (IS_ERR(var)) { 3195 hist_err("Couldn't create or find variable: ", field_name); 3196 kfree(val); 3197 ret = PTR_ERR(var); 3198 goto err; 3199 } 3200 3201 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3202 if (!field_var) { 3203 kfree(val); 3204 kfree(var); 3205 ret = -ENOMEM; 3206 goto err; 3207 } 3208 3209 field_var->var = var; 3210 field_var->val = val; 3211 out: 3212 return field_var; 3213 err: 3214 field_var = ERR_PTR(ret); 3215 goto out; 3216 } 3217 3218 /** 3219 * create_target_field_var - Automatically create a variable for a field 3220 * @target_hist_data: The target hist trigger 3221 * @subsys_name: Optional subsystem name 3222 * @event_name: Optional event name 3223 * @var_name: The name of the field (and the resulting variable) 3224 * 3225 * Hist trigger actions fetch data from variables, not directly from 3226 * events. However, for convenience, users are allowed to directly 3227 * specify an event field in an action, which will be automatically 3228 * converted into a variable on their behalf. 3229 3230 * This function creates a field variable with the name var_name on 3231 * the hist trigger currently being defined on the target event. If 3232 * subsys_name and event_name are specified, this function simply 3233 * verifies that they do in fact match the target event subsystem and 3234 * event name. 3235 * 3236 * Return: The variable created for the field. 3237 */ 3238 static struct field_var * 3239 create_target_field_var(struct hist_trigger_data *target_hist_data, 3240 char *subsys_name, char *event_name, char *var_name) 3241 { 3242 struct trace_event_file *file = target_hist_data->event_file; 3243 3244 if (subsys_name) { 3245 struct trace_event_call *call; 3246 3247 if (!event_name) 3248 return NULL; 3249 3250 call = file->event_call; 3251 3252 if (strcmp(subsys_name, call->class->system) != 0) 3253 return NULL; 3254 3255 if (strcmp(event_name, trace_event_name(call)) != 0) 3256 return NULL; 3257 } 3258 3259 return create_field_var(target_hist_data, file, var_name); 3260 } 3261 3262 static void onmax_print(struct seq_file *m, 3263 struct hist_trigger_data *hist_data, 3264 struct tracing_map_elt *elt, 3265 struct action_data *data) 3266 { 3267 unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx; 3268 3269 seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx)); 3270 3271 for (i = 0; i < hist_data->n_max_vars; i++) { 3272 struct hist_field *save_val = hist_data->max_vars[i]->val; 3273 struct hist_field *save_var = hist_data->max_vars[i]->var; 3274 u64 val; 3275 3276 save_var_idx = save_var->var.idx; 3277 3278 val = tracing_map_read_var(elt, save_var_idx); 3279 3280 if (save_val->flags & HIST_FIELD_FL_STRING) { 3281 seq_printf(m, " %s: %-32s", save_var->var.name, 3282 (char *)(uintptr_t)(val)); 3283 } else 3284 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3285 } 3286 } 3287 3288 static void onmax_save(struct hist_trigger_data *hist_data, 3289 struct tracing_map_elt *elt, void *rec, 3290 struct ring_buffer_event *rbe, 3291 struct action_data *data, u64 *var_ref_vals) 3292 { 3293 unsigned int max_idx = data->onmax.max_var->var.idx; 3294 unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx; 3295 3296 u64 var_val, max_val; 3297 3298 var_val = var_ref_vals[max_var_ref_idx]; 3299 max_val = tracing_map_read_var(elt, max_idx); 3300 3301 if (var_val <= max_val) 3302 return; 3303 3304 tracing_map_set_var(elt, max_idx, var_val); 3305 3306 update_max_vars(hist_data, elt, rbe, rec); 3307 } 3308 3309 static void onmax_destroy(struct action_data *data) 3310 { 3311 unsigned int i; 3312 3313 destroy_hist_field(data->onmax.max_var, 0); 3314 destroy_hist_field(data->onmax.var, 0); 3315 3316 kfree(data->onmax.var_str); 3317 kfree(data->onmax.fn_name); 3318 3319 for (i = 0; i < data->n_params; i++) 3320 kfree(data->params[i]); 3321 3322 kfree(data); 3323 } 3324 3325 static int onmax_create(struct hist_trigger_data *hist_data, 3326 struct action_data *data) 3327 { 3328 struct trace_event_file *file = hist_data->event_file; 3329 struct hist_field *var_field, *ref_field, *max_var; 3330 unsigned int var_ref_idx = hist_data->n_var_refs; 3331 struct field_var *field_var; 3332 char *onmax_var_str, *param; 3333 unsigned long flags; 3334 unsigned int i; 3335 int ret = 0; 3336 3337 onmax_var_str = data->onmax.var_str; 3338 if (onmax_var_str[0] != '$') { 3339 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str); 3340 return -EINVAL; 3341 } 3342 onmax_var_str++; 3343 3344 var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str); 3345 if (!var_field) { 3346 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str); 3347 return -EINVAL; 3348 } 3349 3350 flags = HIST_FIELD_FL_VAR_REF; 3351 ref_field = create_hist_field(hist_data, NULL, flags, NULL); 3352 if (!ref_field) 3353 return -ENOMEM; 3354 3355 if (init_var_ref(ref_field, var_field, NULL, NULL)) { 3356 destroy_hist_field(ref_field, 0); 3357 ret = -ENOMEM; 3358 goto out; 3359 } 3360 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 3361 ref_field->var_ref_idx = hist_data->n_var_refs++; 3362 data->onmax.var = ref_field; 3363 3364 data->fn = onmax_save; 3365 data->onmax.max_var_ref_idx = var_ref_idx; 3366 max_var = create_var(hist_data, file, "max", sizeof(u64), "u64"); 3367 if (IS_ERR(max_var)) { 3368 hist_err("onmax: Couldn't create onmax variable: ", "max"); 3369 ret = PTR_ERR(max_var); 3370 goto out; 3371 } 3372 data->onmax.max_var = max_var; 3373 3374 for (i = 0; i < data->n_params; i++) { 3375 param = kstrdup(data->params[i], GFP_KERNEL); 3376 if (!param) { 3377 ret = -ENOMEM; 3378 goto out; 3379 } 3380 3381 field_var = create_target_field_var(hist_data, NULL, NULL, param); 3382 if (IS_ERR(field_var)) { 3383 hist_err("onmax: Couldn't create field variable: ", param); 3384 ret = PTR_ERR(field_var); 3385 kfree(param); 3386 goto out; 3387 } 3388 3389 hist_data->max_vars[hist_data->n_max_vars++] = field_var; 3390 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3391 hist_data->n_max_var_str++; 3392 3393 kfree(param); 3394 } 3395 out: 3396 return ret; 3397 } 3398 3399 static int parse_action_params(char *params, struct action_data *data) 3400 { 3401 char *param, *saved_param; 3402 int ret = 0; 3403 3404 while (params) { 3405 if (data->n_params >= SYNTH_FIELDS_MAX) 3406 goto out; 3407 3408 param = strsep(¶ms, ","); 3409 if (!param) { 3410 ret = -EINVAL; 3411 goto out; 3412 } 3413 3414 param = strstrip(param); 3415 if (strlen(param) < 2) { 3416 hist_err("Invalid action param: ", param); 3417 ret = -EINVAL; 3418 goto out; 3419 } 3420 3421 saved_param = kstrdup(param, GFP_KERNEL); 3422 if (!saved_param) { 3423 ret = -ENOMEM; 3424 goto out; 3425 } 3426 3427 data->params[data->n_params++] = saved_param; 3428 } 3429 out: 3430 return ret; 3431 } 3432 3433 static struct action_data *onmax_parse(char *str) 3434 { 3435 char *onmax_fn_name, *onmax_var_str; 3436 struct action_data *data; 3437 int ret = -EINVAL; 3438 3439 data = kzalloc(sizeof(*data), GFP_KERNEL); 3440 if (!data) 3441 return ERR_PTR(-ENOMEM); 3442 3443 onmax_var_str = strsep(&str, ")"); 3444 if (!onmax_var_str || !str) { 3445 ret = -EINVAL; 3446 goto free; 3447 } 3448 3449 data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL); 3450 if (!data->onmax.var_str) { 3451 ret = -ENOMEM; 3452 goto free; 3453 } 3454 3455 strsep(&str, "."); 3456 if (!str) 3457 goto free; 3458 3459 onmax_fn_name = strsep(&str, "("); 3460 if (!onmax_fn_name || !str) 3461 goto free; 3462 3463 if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) { 3464 char *params = strsep(&str, ")"); 3465 3466 if (!params) { 3467 ret = -EINVAL; 3468 goto free; 3469 } 3470 3471 ret = parse_action_params(params, data); 3472 if (ret) 3473 goto free; 3474 } else 3475 goto free; 3476 3477 data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL); 3478 if (!data->onmax.fn_name) { 3479 ret = -ENOMEM; 3480 goto free; 3481 } 3482 out: 3483 return data; 3484 free: 3485 onmax_destroy(data); 3486 data = ERR_PTR(ret); 3487 goto out; 3488 } 3489 3490 static void onmatch_destroy(struct action_data *data) 3491 { 3492 unsigned int i; 3493 3494 mutex_lock(&synth_event_mutex); 3495 3496 kfree(data->onmatch.match_event); 3497 kfree(data->onmatch.match_event_system); 3498 kfree(data->onmatch.synth_event_name); 3499 3500 for (i = 0; i < data->n_params; i++) 3501 kfree(data->params[i]); 3502 3503 if (data->onmatch.synth_event) 3504 data->onmatch.synth_event->ref--; 3505 3506 kfree(data); 3507 3508 mutex_unlock(&synth_event_mutex); 3509 } 3510 3511 static void destroy_field_var(struct field_var *field_var) 3512 { 3513 if (!field_var) 3514 return; 3515 3516 destroy_hist_field(field_var->var, 0); 3517 destroy_hist_field(field_var->val, 0); 3518 3519 kfree(field_var); 3520 } 3521 3522 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3523 { 3524 unsigned int i; 3525 3526 for (i = 0; i < hist_data->n_field_vars; i++) 3527 destroy_field_var(hist_data->field_vars[i]); 3528 } 3529 3530 static void save_field_var(struct hist_trigger_data *hist_data, 3531 struct field_var *field_var) 3532 { 3533 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3534 3535 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3536 hist_data->n_field_var_str++; 3537 } 3538 3539 3540 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data) 3541 { 3542 unsigned int i; 3543 3544 for (i = 0; i < hist_data->n_synth_var_refs; i++) 3545 destroy_hist_field(hist_data->synth_var_refs[i], 0); 3546 } 3547 3548 static void save_synth_var_ref(struct hist_trigger_data *hist_data, 3549 struct hist_field *var_ref) 3550 { 3551 hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref; 3552 3553 hist_data->var_refs[hist_data->n_var_refs] = var_ref; 3554 var_ref->var_ref_idx = hist_data->n_var_refs++; 3555 } 3556 3557 static int check_synth_field(struct synth_event *event, 3558 struct hist_field *hist_field, 3559 unsigned int field_pos) 3560 { 3561 struct synth_field *field; 3562 3563 if (field_pos >= event->n_fields) 3564 return -EINVAL; 3565 3566 field = event->fields[field_pos]; 3567 3568 if (strcmp(field->type, hist_field->type) != 0) 3569 return -EINVAL; 3570 3571 return 0; 3572 } 3573 3574 static struct hist_field * 3575 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data, 3576 char *system, char *event, char *var) 3577 { 3578 struct hist_field *hist_field; 3579 3580 var++; /* skip '$' */ 3581 3582 hist_field = find_target_event_var(hist_data, system, event, var); 3583 if (!hist_field) { 3584 if (!system) { 3585 system = data->onmatch.match_event_system; 3586 event = data->onmatch.match_event; 3587 } 3588 3589 hist_field = find_event_var(hist_data, system, event, var); 3590 } 3591 3592 if (!hist_field) 3593 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var); 3594 3595 return hist_field; 3596 } 3597 3598 static struct hist_field * 3599 onmatch_create_field_var(struct hist_trigger_data *hist_data, 3600 struct action_data *data, char *system, 3601 char *event, char *var) 3602 { 3603 struct hist_field *hist_field = NULL; 3604 struct field_var *field_var; 3605 3606 /* 3607 * First try to create a field var on the target event (the 3608 * currently being defined). This will create a variable for 3609 * unqualified fields on the target event, or if qualified, 3610 * target fields that have qualified names matching the target. 3611 */ 3612 field_var = create_target_field_var(hist_data, system, event, var); 3613 3614 if (field_var && !IS_ERR(field_var)) { 3615 save_field_var(hist_data, field_var); 3616 hist_field = field_var->var; 3617 } else { 3618 field_var = NULL; 3619 /* 3620 * If no explicit system.event is specfied, default to 3621 * looking for fields on the onmatch(system.event.xxx) 3622 * event. 3623 */ 3624 if (!system) { 3625 system = data->onmatch.match_event_system; 3626 event = data->onmatch.match_event; 3627 } 3628 3629 /* 3630 * At this point, we're looking at a field on another 3631 * event. Because we can't modify a hist trigger on 3632 * another event to add a variable for a field, we need 3633 * to create a new trigger on that event and create the 3634 * variable at the same time. 3635 */ 3636 hist_field = create_field_var_hist(hist_data, system, event, var); 3637 if (IS_ERR(hist_field)) 3638 goto free; 3639 } 3640 out: 3641 return hist_field; 3642 free: 3643 destroy_field_var(field_var); 3644 hist_field = NULL; 3645 goto out; 3646 } 3647 3648 static int onmatch_create(struct hist_trigger_data *hist_data, 3649 struct trace_event_file *file, 3650 struct action_data *data) 3651 { 3652 char *event_name, *param, *system = NULL; 3653 struct hist_field *hist_field, *var_ref; 3654 unsigned int i, var_ref_idx; 3655 unsigned int field_pos = 0; 3656 struct synth_event *event; 3657 int ret = 0; 3658 3659 mutex_lock(&synth_event_mutex); 3660 event = find_synth_event(data->onmatch.synth_event_name); 3661 if (!event) { 3662 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name); 3663 mutex_unlock(&synth_event_mutex); 3664 return -EINVAL; 3665 } 3666 event->ref++; 3667 mutex_unlock(&synth_event_mutex); 3668 3669 var_ref_idx = hist_data->n_var_refs; 3670 3671 for (i = 0; i < data->n_params; i++) { 3672 char *p; 3673 3674 p = param = kstrdup(data->params[i], GFP_KERNEL); 3675 if (!param) { 3676 ret = -ENOMEM; 3677 goto err; 3678 } 3679 3680 system = strsep(¶m, "."); 3681 if (!param) { 3682 param = (char *)system; 3683 system = event_name = NULL; 3684 } else { 3685 event_name = strsep(¶m, "."); 3686 if (!param) { 3687 kfree(p); 3688 ret = -EINVAL; 3689 goto err; 3690 } 3691 } 3692 3693 if (param[0] == '$') 3694 hist_field = onmatch_find_var(hist_data, data, system, 3695 event_name, param); 3696 else 3697 hist_field = onmatch_create_field_var(hist_data, data, 3698 system, 3699 event_name, 3700 param); 3701 3702 if (!hist_field) { 3703 kfree(p); 3704 ret = -EINVAL; 3705 goto err; 3706 } 3707 3708 if (check_synth_field(event, hist_field, field_pos) == 0) { 3709 var_ref = create_var_ref(hist_field, system, event_name); 3710 if (!var_ref) { 3711 kfree(p); 3712 ret = -ENOMEM; 3713 goto err; 3714 } 3715 3716 save_synth_var_ref(hist_data, var_ref); 3717 field_pos++; 3718 kfree(p); 3719 continue; 3720 } 3721 3722 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ", 3723 system, event_name, param); 3724 kfree(p); 3725 ret = -EINVAL; 3726 goto err; 3727 } 3728 3729 if (field_pos != event->n_fields) { 3730 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name); 3731 ret = -EINVAL; 3732 goto err; 3733 } 3734 3735 data->fn = action_trace; 3736 data->onmatch.synth_event = event; 3737 data->onmatch.var_ref_idx = var_ref_idx; 3738 out: 3739 return ret; 3740 err: 3741 mutex_lock(&synth_event_mutex); 3742 event->ref--; 3743 mutex_unlock(&synth_event_mutex); 3744 3745 goto out; 3746 } 3747 3748 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 3749 { 3750 char *match_event, *match_event_system; 3751 char *synth_event_name, *params; 3752 struct action_data *data; 3753 int ret = -EINVAL; 3754 3755 data = kzalloc(sizeof(*data), GFP_KERNEL); 3756 if (!data) 3757 return ERR_PTR(-ENOMEM); 3758 3759 match_event = strsep(&str, ")"); 3760 if (!match_event || !str) { 3761 hist_err("onmatch: Missing closing paren: ", match_event); 3762 goto free; 3763 } 3764 3765 match_event_system = strsep(&match_event, "."); 3766 if (!match_event) { 3767 hist_err("onmatch: Missing subsystem for match event: ", match_event_system); 3768 goto free; 3769 } 3770 3771 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 3772 hist_err_event("onmatch: Invalid subsystem or event name: ", 3773 match_event_system, match_event, NULL); 3774 goto free; 3775 } 3776 3777 data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL); 3778 if (!data->onmatch.match_event) { 3779 ret = -ENOMEM; 3780 goto free; 3781 } 3782 3783 data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL); 3784 if (!data->onmatch.match_event_system) { 3785 ret = -ENOMEM; 3786 goto free; 3787 } 3788 3789 strsep(&str, "."); 3790 if (!str) { 3791 hist_err("onmatch: Missing . after onmatch(): ", str); 3792 goto free; 3793 } 3794 3795 synth_event_name = strsep(&str, "("); 3796 if (!synth_event_name || !str) { 3797 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name); 3798 goto free; 3799 } 3800 3801 data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL); 3802 if (!data->onmatch.synth_event_name) { 3803 ret = -ENOMEM; 3804 goto free; 3805 } 3806 3807 params = strsep(&str, ")"); 3808 if (!params || !str || (str && strlen(str))) { 3809 hist_err("onmatch: Missing closing paramlist paren: ", params); 3810 goto free; 3811 } 3812 3813 ret = parse_action_params(params, data); 3814 if (ret) 3815 goto free; 3816 out: 3817 return data; 3818 free: 3819 onmatch_destroy(data); 3820 data = ERR_PTR(ret); 3821 goto out; 3822 } 3823 3824 static int create_hitcount_val(struct hist_trigger_data *hist_data) 3825 { 3826 hist_data->fields[HITCOUNT_IDX] = 3827 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 3828 if (!hist_data->fields[HITCOUNT_IDX]) 3829 return -ENOMEM; 3830 3831 hist_data->n_vals++; 3832 hist_data->n_fields++; 3833 3834 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 3835 return -EINVAL; 3836 3837 return 0; 3838 } 3839 3840 static int __create_val_field(struct hist_trigger_data *hist_data, 3841 unsigned int val_idx, 3842 struct trace_event_file *file, 3843 char *var_name, char *field_str, 3844 unsigned long flags) 3845 { 3846 struct hist_field *hist_field; 3847 int ret = 0; 3848 3849 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 3850 if (IS_ERR(hist_field)) { 3851 ret = PTR_ERR(hist_field); 3852 goto out; 3853 } 3854 3855 hist_data->fields[val_idx] = hist_field; 3856 3857 ++hist_data->n_vals; 3858 ++hist_data->n_fields; 3859 3860 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3861 ret = -EINVAL; 3862 out: 3863 return ret; 3864 } 3865 3866 static int create_val_field(struct hist_trigger_data *hist_data, 3867 unsigned int val_idx, 3868 struct trace_event_file *file, 3869 char *field_str) 3870 { 3871 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 3872 return -EINVAL; 3873 3874 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 3875 } 3876 3877 static int create_var_field(struct hist_trigger_data *hist_data, 3878 unsigned int val_idx, 3879 struct trace_event_file *file, 3880 char *var_name, char *expr_str) 3881 { 3882 unsigned long flags = 0; 3883 3884 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3885 return -EINVAL; 3886 3887 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 3888 hist_err("Variable already defined: ", var_name); 3889 return -EINVAL; 3890 } 3891 3892 flags |= HIST_FIELD_FL_VAR; 3893 hist_data->n_vars++; 3894 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 3895 return -EINVAL; 3896 3897 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 3898 } 3899 3900 static int create_val_fields(struct hist_trigger_data *hist_data, 3901 struct trace_event_file *file) 3902 { 3903 char *fields_str, *field_str; 3904 unsigned int i, j = 1; 3905 int ret; 3906 3907 ret = create_hitcount_val(hist_data); 3908 if (ret) 3909 goto out; 3910 3911 fields_str = hist_data->attrs->vals_str; 3912 if (!fields_str) 3913 goto out; 3914 3915 strsep(&fields_str, "="); 3916 if (!fields_str) 3917 goto out; 3918 3919 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 3920 j < TRACING_MAP_VALS_MAX; i++) { 3921 field_str = strsep(&fields_str, ","); 3922 if (!field_str) 3923 break; 3924 3925 if (strcmp(field_str, "hitcount") == 0) 3926 continue; 3927 3928 ret = create_val_field(hist_data, j++, file, field_str); 3929 if (ret) 3930 goto out; 3931 } 3932 3933 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 3934 ret = -EINVAL; 3935 out: 3936 return ret; 3937 } 3938 3939 static int create_key_field(struct hist_trigger_data *hist_data, 3940 unsigned int key_idx, 3941 unsigned int key_offset, 3942 struct trace_event_file *file, 3943 char *field_str) 3944 { 3945 struct hist_field *hist_field = NULL; 3946 3947 unsigned long flags = 0; 3948 unsigned int key_size; 3949 int ret = 0; 3950 3951 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 3952 return -EINVAL; 3953 3954 flags |= HIST_FIELD_FL_KEY; 3955 3956 if (strcmp(field_str, "stacktrace") == 0) { 3957 flags |= HIST_FIELD_FL_STACKTRACE; 3958 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 3959 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 3960 } else { 3961 hist_field = parse_expr(hist_data, file, field_str, flags, 3962 NULL, 0); 3963 if (IS_ERR(hist_field)) { 3964 ret = PTR_ERR(hist_field); 3965 goto out; 3966 } 3967 3968 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) { 3969 hist_err("Using variable references as keys not supported: ", field_str); 3970 destroy_hist_field(hist_field, 0); 3971 ret = -EINVAL; 3972 goto out; 3973 } 3974 3975 key_size = hist_field->size; 3976 } 3977 3978 hist_data->fields[key_idx] = hist_field; 3979 3980 key_size = ALIGN(key_size, sizeof(u64)); 3981 hist_data->fields[key_idx]->size = key_size; 3982 hist_data->fields[key_idx]->offset = key_offset; 3983 3984 hist_data->key_size += key_size; 3985 3986 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 3987 ret = -EINVAL; 3988 goto out; 3989 } 3990 3991 hist_data->n_keys++; 3992 hist_data->n_fields++; 3993 3994 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 3995 return -EINVAL; 3996 3997 ret = key_size; 3998 out: 3999 return ret; 4000 } 4001 4002 static int create_key_fields(struct hist_trigger_data *hist_data, 4003 struct trace_event_file *file) 4004 { 4005 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4006 char *fields_str, *field_str; 4007 int ret = -EINVAL; 4008 4009 fields_str = hist_data->attrs->keys_str; 4010 if (!fields_str) 4011 goto out; 4012 4013 strsep(&fields_str, "="); 4014 if (!fields_str) 4015 goto out; 4016 4017 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4018 field_str = strsep(&fields_str, ","); 4019 if (!field_str) 4020 break; 4021 ret = create_key_field(hist_data, i, key_offset, 4022 file, field_str); 4023 if (ret < 0) 4024 goto out; 4025 key_offset += ret; 4026 } 4027 if (fields_str) { 4028 ret = -EINVAL; 4029 goto out; 4030 } 4031 ret = 0; 4032 out: 4033 return ret; 4034 } 4035 4036 static int create_var_fields(struct hist_trigger_data *hist_data, 4037 struct trace_event_file *file) 4038 { 4039 unsigned int i, j = hist_data->n_vals; 4040 int ret = 0; 4041 4042 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4043 4044 for (i = 0; i < n_vars; i++) { 4045 char *var_name = hist_data->attrs->var_defs.name[i]; 4046 char *expr = hist_data->attrs->var_defs.expr[i]; 4047 4048 ret = create_var_field(hist_data, j++, file, var_name, expr); 4049 if (ret) 4050 goto out; 4051 } 4052 out: 4053 return ret; 4054 } 4055 4056 static void free_var_defs(struct hist_trigger_data *hist_data) 4057 { 4058 unsigned int i; 4059 4060 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4061 kfree(hist_data->attrs->var_defs.name[i]); 4062 kfree(hist_data->attrs->var_defs.expr[i]); 4063 } 4064 4065 hist_data->attrs->var_defs.n_vars = 0; 4066 } 4067 4068 static int parse_var_defs(struct hist_trigger_data *hist_data) 4069 { 4070 char *s, *str, *var_name, *field_str; 4071 unsigned int i, j, n_vars = 0; 4072 int ret = 0; 4073 4074 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4075 str = hist_data->attrs->assignment_str[i]; 4076 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4077 field_str = strsep(&str, ","); 4078 if (!field_str) 4079 break; 4080 4081 var_name = strsep(&field_str, "="); 4082 if (!var_name || !field_str) { 4083 hist_err("Malformed assignment: ", var_name); 4084 ret = -EINVAL; 4085 goto free; 4086 } 4087 4088 if (n_vars == TRACING_MAP_VARS_MAX) { 4089 hist_err("Too many variables defined: ", var_name); 4090 ret = -EINVAL; 4091 goto free; 4092 } 4093 4094 s = kstrdup(var_name, GFP_KERNEL); 4095 if (!s) { 4096 ret = -ENOMEM; 4097 goto free; 4098 } 4099 hist_data->attrs->var_defs.name[n_vars] = s; 4100 4101 s = kstrdup(field_str, GFP_KERNEL); 4102 if (!s) { 4103 kfree(hist_data->attrs->var_defs.name[n_vars]); 4104 ret = -ENOMEM; 4105 goto free; 4106 } 4107 hist_data->attrs->var_defs.expr[n_vars++] = s; 4108 4109 hist_data->attrs->var_defs.n_vars = n_vars; 4110 } 4111 } 4112 4113 return ret; 4114 free: 4115 free_var_defs(hist_data); 4116 4117 return ret; 4118 } 4119 4120 static int create_hist_fields(struct hist_trigger_data *hist_data, 4121 struct trace_event_file *file) 4122 { 4123 int ret; 4124 4125 ret = parse_var_defs(hist_data); 4126 if (ret) 4127 goto out; 4128 4129 ret = create_val_fields(hist_data, file); 4130 if (ret) 4131 goto out; 4132 4133 ret = create_var_fields(hist_data, file); 4134 if (ret) 4135 goto out; 4136 4137 ret = create_key_fields(hist_data, file); 4138 if (ret) 4139 goto out; 4140 out: 4141 free_var_defs(hist_data); 4142 4143 return ret; 4144 } 4145 4146 static int is_descending(const char *str) 4147 { 4148 if (!str) 4149 return 0; 4150 4151 if (strcmp(str, "descending") == 0) 4152 return 1; 4153 4154 if (strcmp(str, "ascending") == 0) 4155 return 0; 4156 4157 return -EINVAL; 4158 } 4159 4160 static int create_sort_keys(struct hist_trigger_data *hist_data) 4161 { 4162 char *fields_str = hist_data->attrs->sort_key_str; 4163 struct tracing_map_sort_key *sort_key; 4164 int descending, ret = 0; 4165 unsigned int i, j, k; 4166 4167 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4168 4169 if (!fields_str) 4170 goto out; 4171 4172 strsep(&fields_str, "="); 4173 if (!fields_str) { 4174 ret = -EINVAL; 4175 goto out; 4176 } 4177 4178 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4179 struct hist_field *hist_field; 4180 char *field_str, *field_name; 4181 const char *test_name; 4182 4183 sort_key = &hist_data->sort_keys[i]; 4184 4185 field_str = strsep(&fields_str, ","); 4186 if (!field_str) { 4187 if (i == 0) 4188 ret = -EINVAL; 4189 break; 4190 } 4191 4192 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4193 ret = -EINVAL; 4194 break; 4195 } 4196 4197 field_name = strsep(&field_str, "."); 4198 if (!field_name) { 4199 ret = -EINVAL; 4200 break; 4201 } 4202 4203 if (strcmp(field_name, "hitcount") == 0) { 4204 descending = is_descending(field_str); 4205 if (descending < 0) { 4206 ret = descending; 4207 break; 4208 } 4209 sort_key->descending = descending; 4210 continue; 4211 } 4212 4213 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4214 unsigned int idx; 4215 4216 hist_field = hist_data->fields[j]; 4217 if (hist_field->flags & HIST_FIELD_FL_VAR) 4218 continue; 4219 4220 idx = k++; 4221 4222 test_name = hist_field_name(hist_field, 0); 4223 4224 if (strcmp(field_name, test_name) == 0) { 4225 sort_key->field_idx = idx; 4226 descending = is_descending(field_str); 4227 if (descending < 0) { 4228 ret = descending; 4229 goto out; 4230 } 4231 sort_key->descending = descending; 4232 break; 4233 } 4234 } 4235 if (j == hist_data->n_fields) { 4236 ret = -EINVAL; 4237 break; 4238 } 4239 } 4240 4241 hist_data->n_sort_keys = i; 4242 out: 4243 return ret; 4244 } 4245 4246 static void destroy_actions(struct hist_trigger_data *hist_data) 4247 { 4248 unsigned int i; 4249 4250 for (i = 0; i < hist_data->n_actions; i++) { 4251 struct action_data *data = hist_data->actions[i]; 4252 4253 if (data->fn == action_trace) 4254 onmatch_destroy(data); 4255 else if (data->fn == onmax_save) 4256 onmax_destroy(data); 4257 else 4258 kfree(data); 4259 } 4260 } 4261 4262 static int parse_actions(struct hist_trigger_data *hist_data) 4263 { 4264 struct trace_array *tr = hist_data->event_file->tr; 4265 struct action_data *data; 4266 unsigned int i; 4267 int ret = 0; 4268 char *str; 4269 4270 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4271 str = hist_data->attrs->action_str[i]; 4272 4273 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) { 4274 char *action_str = str + strlen("onmatch("); 4275 4276 data = onmatch_parse(tr, action_str); 4277 if (IS_ERR(data)) { 4278 ret = PTR_ERR(data); 4279 break; 4280 } 4281 data->fn = action_trace; 4282 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) { 4283 char *action_str = str + strlen("onmax("); 4284 4285 data = onmax_parse(action_str); 4286 if (IS_ERR(data)) { 4287 ret = PTR_ERR(data); 4288 break; 4289 } 4290 data->fn = onmax_save; 4291 } else { 4292 ret = -EINVAL; 4293 break; 4294 } 4295 4296 hist_data->actions[hist_data->n_actions++] = data; 4297 } 4298 4299 return ret; 4300 } 4301 4302 static int create_actions(struct hist_trigger_data *hist_data, 4303 struct trace_event_file *file) 4304 { 4305 struct action_data *data; 4306 unsigned int i; 4307 int ret = 0; 4308 4309 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4310 data = hist_data->actions[i]; 4311 4312 if (data->fn == action_trace) { 4313 ret = onmatch_create(hist_data, file, data); 4314 if (ret) 4315 return ret; 4316 } else if (data->fn == onmax_save) { 4317 ret = onmax_create(hist_data, data); 4318 if (ret) 4319 return ret; 4320 } 4321 } 4322 4323 return ret; 4324 } 4325 4326 static void print_actions(struct seq_file *m, 4327 struct hist_trigger_data *hist_data, 4328 struct tracing_map_elt *elt) 4329 { 4330 unsigned int i; 4331 4332 for (i = 0; i < hist_data->n_actions; i++) { 4333 struct action_data *data = hist_data->actions[i]; 4334 4335 if (data->fn == onmax_save) 4336 onmax_print(m, hist_data, elt, data); 4337 } 4338 } 4339 4340 static void print_onmax_spec(struct seq_file *m, 4341 struct hist_trigger_data *hist_data, 4342 struct action_data *data) 4343 { 4344 unsigned int i; 4345 4346 seq_puts(m, ":onmax("); 4347 seq_printf(m, "%s", data->onmax.var_str); 4348 seq_printf(m, ").%s(", data->onmax.fn_name); 4349 4350 for (i = 0; i < hist_data->n_max_vars; i++) { 4351 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name); 4352 if (i < hist_data->n_max_vars - 1) 4353 seq_puts(m, ","); 4354 } 4355 seq_puts(m, ")"); 4356 } 4357 4358 static void print_onmatch_spec(struct seq_file *m, 4359 struct hist_trigger_data *hist_data, 4360 struct action_data *data) 4361 { 4362 unsigned int i; 4363 4364 seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system, 4365 data->onmatch.match_event); 4366 4367 seq_printf(m, "%s(", data->onmatch.synth_event->name); 4368 4369 for (i = 0; i < data->n_params; i++) { 4370 if (i) 4371 seq_puts(m, ","); 4372 seq_printf(m, "%s", data->params[i]); 4373 } 4374 4375 seq_puts(m, ")"); 4376 } 4377 4378 static bool actions_match(struct hist_trigger_data *hist_data, 4379 struct hist_trigger_data *hist_data_test) 4380 { 4381 unsigned int i, j; 4382 4383 if (hist_data->n_actions != hist_data_test->n_actions) 4384 return false; 4385 4386 for (i = 0; i < hist_data->n_actions; i++) { 4387 struct action_data *data = hist_data->actions[i]; 4388 struct action_data *data_test = hist_data_test->actions[i]; 4389 4390 if (data->fn != data_test->fn) 4391 return false; 4392 4393 if (data->n_params != data_test->n_params) 4394 return false; 4395 4396 for (j = 0; j < data->n_params; j++) { 4397 if (strcmp(data->params[j], data_test->params[j]) != 0) 4398 return false; 4399 } 4400 4401 if (data->fn == action_trace) { 4402 if (strcmp(data->onmatch.synth_event_name, 4403 data_test->onmatch.synth_event_name) != 0) 4404 return false; 4405 if (strcmp(data->onmatch.match_event_system, 4406 data_test->onmatch.match_event_system) != 0) 4407 return false; 4408 if (strcmp(data->onmatch.match_event, 4409 data_test->onmatch.match_event) != 0) 4410 return false; 4411 } else if (data->fn == onmax_save) { 4412 if (strcmp(data->onmax.var_str, 4413 data_test->onmax.var_str) != 0) 4414 return false; 4415 if (strcmp(data->onmax.fn_name, 4416 data_test->onmax.fn_name) != 0) 4417 return false; 4418 } 4419 } 4420 4421 return true; 4422 } 4423 4424 4425 static void print_actions_spec(struct seq_file *m, 4426 struct hist_trigger_data *hist_data) 4427 { 4428 unsigned int i; 4429 4430 for (i = 0; i < hist_data->n_actions; i++) { 4431 struct action_data *data = hist_data->actions[i]; 4432 4433 if (data->fn == action_trace) 4434 print_onmatch_spec(m, hist_data, data); 4435 else if (data->fn == onmax_save) 4436 print_onmax_spec(m, hist_data, data); 4437 } 4438 } 4439 4440 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 4441 { 4442 unsigned int i; 4443 4444 for (i = 0; i < hist_data->n_field_var_hists; i++) { 4445 kfree(hist_data->field_var_hists[i]->cmd); 4446 kfree(hist_data->field_var_hists[i]); 4447 } 4448 } 4449 4450 static void destroy_hist_data(struct hist_trigger_data *hist_data) 4451 { 4452 if (!hist_data) 4453 return; 4454 4455 destroy_hist_trigger_attrs(hist_data->attrs); 4456 destroy_hist_fields(hist_data); 4457 tracing_map_destroy(hist_data->map); 4458 4459 destroy_actions(hist_data); 4460 destroy_field_vars(hist_data); 4461 destroy_field_var_hists(hist_data); 4462 destroy_synth_var_refs(hist_data); 4463 4464 kfree(hist_data); 4465 } 4466 4467 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 4468 { 4469 struct tracing_map *map = hist_data->map; 4470 struct ftrace_event_field *field; 4471 struct hist_field *hist_field; 4472 int i, idx = 0; 4473 4474 for_each_hist_field(i, hist_data) { 4475 hist_field = hist_data->fields[i]; 4476 if (hist_field->flags & HIST_FIELD_FL_KEY) { 4477 tracing_map_cmp_fn_t cmp_fn; 4478 4479 field = hist_field->field; 4480 4481 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 4482 cmp_fn = tracing_map_cmp_none; 4483 else if (!field) 4484 cmp_fn = tracing_map_cmp_num(hist_field->size, 4485 hist_field->is_signed); 4486 else if (is_string_field(field)) 4487 cmp_fn = tracing_map_cmp_string; 4488 else 4489 cmp_fn = tracing_map_cmp_num(field->size, 4490 field->is_signed); 4491 idx = tracing_map_add_key_field(map, 4492 hist_field->offset, 4493 cmp_fn); 4494 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 4495 idx = tracing_map_add_sum_field(map); 4496 4497 if (idx < 0) 4498 return idx; 4499 4500 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4501 idx = tracing_map_add_var(map); 4502 if (idx < 0) 4503 return idx; 4504 hist_field->var.idx = idx; 4505 hist_field->var.hist_data = hist_data; 4506 } 4507 } 4508 4509 return 0; 4510 } 4511 4512 static struct hist_trigger_data * 4513 create_hist_data(unsigned int map_bits, 4514 struct hist_trigger_attrs *attrs, 4515 struct trace_event_file *file, 4516 bool remove) 4517 { 4518 const struct tracing_map_ops *map_ops = NULL; 4519 struct hist_trigger_data *hist_data; 4520 int ret = 0; 4521 4522 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 4523 if (!hist_data) 4524 return ERR_PTR(-ENOMEM); 4525 4526 hist_data->attrs = attrs; 4527 hist_data->remove = remove; 4528 hist_data->event_file = file; 4529 4530 ret = parse_actions(hist_data); 4531 if (ret) 4532 goto free; 4533 4534 ret = create_hist_fields(hist_data, file); 4535 if (ret) 4536 goto free; 4537 4538 ret = create_sort_keys(hist_data); 4539 if (ret) 4540 goto free; 4541 4542 map_ops = &hist_trigger_elt_data_ops; 4543 4544 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 4545 map_ops, hist_data); 4546 if (IS_ERR(hist_data->map)) { 4547 ret = PTR_ERR(hist_data->map); 4548 hist_data->map = NULL; 4549 goto free; 4550 } 4551 4552 ret = create_tracing_map_fields(hist_data); 4553 if (ret) 4554 goto free; 4555 out: 4556 return hist_data; 4557 free: 4558 hist_data->attrs = NULL; 4559 4560 destroy_hist_data(hist_data); 4561 4562 hist_data = ERR_PTR(ret); 4563 4564 goto out; 4565 } 4566 4567 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 4568 struct tracing_map_elt *elt, void *rec, 4569 struct ring_buffer_event *rbe, 4570 u64 *var_ref_vals) 4571 { 4572 struct hist_elt_data *elt_data; 4573 struct hist_field *hist_field; 4574 unsigned int i, var_idx; 4575 u64 hist_val; 4576 4577 elt_data = elt->private_data; 4578 elt_data->var_ref_vals = var_ref_vals; 4579 4580 for_each_hist_val_field(i, hist_data) { 4581 hist_field = hist_data->fields[i]; 4582 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4583 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4584 var_idx = hist_field->var.idx; 4585 tracing_map_set_var(elt, var_idx, hist_val); 4586 continue; 4587 } 4588 tracing_map_update_sum(elt, i, hist_val); 4589 } 4590 4591 for_each_hist_key_field(i, hist_data) { 4592 hist_field = hist_data->fields[i]; 4593 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4594 hist_val = hist_field->fn(hist_field, elt, rbe, rec); 4595 var_idx = hist_field->var.idx; 4596 tracing_map_set_var(elt, var_idx, hist_val); 4597 } 4598 } 4599 4600 update_field_vars(hist_data, elt, rbe, rec); 4601 } 4602 4603 static inline void add_to_key(char *compound_key, void *key, 4604 struct hist_field *key_field, void *rec) 4605 { 4606 size_t size = key_field->size; 4607 4608 if (key_field->flags & HIST_FIELD_FL_STRING) { 4609 struct ftrace_event_field *field; 4610 4611 field = key_field->field; 4612 if (field->filter_type == FILTER_DYN_STRING) 4613 size = *(u32 *)(rec + field->offset) >> 16; 4614 else if (field->filter_type == FILTER_PTR_STRING) 4615 size = strlen(key); 4616 else if (field->filter_type == FILTER_STATIC_STRING) 4617 size = field->size; 4618 4619 /* ensure NULL-termination */ 4620 if (size > key_field->size - 1) 4621 size = key_field->size - 1; 4622 } 4623 4624 memcpy(compound_key + key_field->offset, key, size); 4625 } 4626 4627 static void 4628 hist_trigger_actions(struct hist_trigger_data *hist_data, 4629 struct tracing_map_elt *elt, void *rec, 4630 struct ring_buffer_event *rbe, u64 *var_ref_vals) 4631 { 4632 struct action_data *data; 4633 unsigned int i; 4634 4635 for (i = 0; i < hist_data->n_actions; i++) { 4636 data = hist_data->actions[i]; 4637 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals); 4638 } 4639 } 4640 4641 static void event_hist_trigger(struct event_trigger_data *data, void *rec, 4642 struct ring_buffer_event *rbe) 4643 { 4644 struct hist_trigger_data *hist_data = data->private_data; 4645 bool use_compound_key = (hist_data->n_keys > 1); 4646 unsigned long entries[HIST_STACKTRACE_DEPTH]; 4647 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 4648 char compound_key[HIST_KEY_SIZE_MAX]; 4649 struct tracing_map_elt *elt = NULL; 4650 struct stack_trace stacktrace; 4651 struct hist_field *key_field; 4652 u64 field_contents; 4653 void *key = NULL; 4654 unsigned int i; 4655 4656 memset(compound_key, 0, hist_data->key_size); 4657 4658 for_each_hist_key_field(i, hist_data) { 4659 key_field = hist_data->fields[i]; 4660 4661 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4662 stacktrace.max_entries = HIST_STACKTRACE_DEPTH; 4663 stacktrace.entries = entries; 4664 stacktrace.nr_entries = 0; 4665 stacktrace.skip = HIST_STACKTRACE_SKIP; 4666 4667 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE); 4668 save_stack_trace(&stacktrace); 4669 4670 key = entries; 4671 } else { 4672 field_contents = key_field->fn(key_field, elt, rbe, rec); 4673 if (key_field->flags & HIST_FIELD_FL_STRING) { 4674 key = (void *)(unsigned long)field_contents; 4675 use_compound_key = true; 4676 } else 4677 key = (void *)&field_contents; 4678 } 4679 4680 if (use_compound_key) 4681 add_to_key(compound_key, key, key_field, rec); 4682 } 4683 4684 if (use_compound_key) 4685 key = compound_key; 4686 4687 if (hist_data->n_var_refs && 4688 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 4689 return; 4690 4691 elt = tracing_map_insert(hist_data->map, key); 4692 if (!elt) 4693 return; 4694 4695 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals); 4696 4697 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 4698 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals); 4699 } 4700 4701 static void hist_trigger_stacktrace_print(struct seq_file *m, 4702 unsigned long *stacktrace_entries, 4703 unsigned int max_entries) 4704 { 4705 char str[KSYM_SYMBOL_LEN]; 4706 unsigned int spaces = 8; 4707 unsigned int i; 4708 4709 for (i = 0; i < max_entries; i++) { 4710 if (stacktrace_entries[i] == ULONG_MAX) 4711 return; 4712 4713 seq_printf(m, "%*c", 1 + spaces, ' '); 4714 sprint_symbol(str, stacktrace_entries[i]); 4715 seq_printf(m, "%s\n", str); 4716 } 4717 } 4718 4719 static void 4720 hist_trigger_entry_print(struct seq_file *m, 4721 struct hist_trigger_data *hist_data, void *key, 4722 struct tracing_map_elt *elt) 4723 { 4724 struct hist_field *key_field; 4725 char str[KSYM_SYMBOL_LEN]; 4726 bool multiline = false; 4727 const char *field_name; 4728 unsigned int i; 4729 u64 uval; 4730 4731 seq_puts(m, "{ "); 4732 4733 for_each_hist_key_field(i, hist_data) { 4734 key_field = hist_data->fields[i]; 4735 4736 if (i > hist_data->n_vals) 4737 seq_puts(m, ", "); 4738 4739 field_name = hist_field_name(key_field, 0); 4740 4741 if (key_field->flags & HIST_FIELD_FL_HEX) { 4742 uval = *(u64 *)(key + key_field->offset); 4743 seq_printf(m, "%s: %llx", field_name, uval); 4744 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 4745 uval = *(u64 *)(key + key_field->offset); 4746 sprint_symbol_no_offset(str, uval); 4747 seq_printf(m, "%s: [%llx] %-45s", field_name, 4748 uval, str); 4749 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 4750 uval = *(u64 *)(key + key_field->offset); 4751 sprint_symbol(str, uval); 4752 seq_printf(m, "%s: [%llx] %-55s", field_name, 4753 uval, str); 4754 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 4755 struct hist_elt_data *elt_data = elt->private_data; 4756 char *comm; 4757 4758 if (WARN_ON_ONCE(!elt_data)) 4759 return; 4760 4761 comm = elt_data->comm; 4762 4763 uval = *(u64 *)(key + key_field->offset); 4764 seq_printf(m, "%s: %-16s[%10llu]", field_name, 4765 comm, uval); 4766 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 4767 const char *syscall_name; 4768 4769 uval = *(u64 *)(key + key_field->offset); 4770 syscall_name = get_syscall_name(uval); 4771 if (!syscall_name) 4772 syscall_name = "unknown_syscall"; 4773 4774 seq_printf(m, "%s: %-30s[%3llu]", field_name, 4775 syscall_name, uval); 4776 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4777 seq_puts(m, "stacktrace:\n"); 4778 hist_trigger_stacktrace_print(m, 4779 key + key_field->offset, 4780 HIST_STACKTRACE_DEPTH); 4781 multiline = true; 4782 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 4783 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 4784 *(u64 *)(key + key_field->offset)); 4785 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 4786 seq_printf(m, "%s: %-50s", field_name, 4787 (char *)(key + key_field->offset)); 4788 } else { 4789 uval = *(u64 *)(key + key_field->offset); 4790 seq_printf(m, "%s: %10llu", field_name, uval); 4791 } 4792 } 4793 4794 if (!multiline) 4795 seq_puts(m, " "); 4796 4797 seq_puts(m, "}"); 4798 4799 seq_printf(m, " hitcount: %10llu", 4800 tracing_map_read_sum(elt, HITCOUNT_IDX)); 4801 4802 for (i = 1; i < hist_data->n_vals; i++) { 4803 field_name = hist_field_name(hist_data->fields[i], 0); 4804 4805 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 4806 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 4807 continue; 4808 4809 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 4810 seq_printf(m, " %s: %10llx", field_name, 4811 tracing_map_read_sum(elt, i)); 4812 } else { 4813 seq_printf(m, " %s: %10llu", field_name, 4814 tracing_map_read_sum(elt, i)); 4815 } 4816 } 4817 4818 print_actions(m, hist_data, elt); 4819 4820 seq_puts(m, "\n"); 4821 } 4822 4823 static int print_entries(struct seq_file *m, 4824 struct hist_trigger_data *hist_data) 4825 { 4826 struct tracing_map_sort_entry **sort_entries = NULL; 4827 struct tracing_map *map = hist_data->map; 4828 int i, n_entries; 4829 4830 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 4831 hist_data->n_sort_keys, 4832 &sort_entries); 4833 if (n_entries < 0) 4834 return n_entries; 4835 4836 for (i = 0; i < n_entries; i++) 4837 hist_trigger_entry_print(m, hist_data, 4838 sort_entries[i]->key, 4839 sort_entries[i]->elt); 4840 4841 tracing_map_destroy_sort_entries(sort_entries, n_entries); 4842 4843 return n_entries; 4844 } 4845 4846 static void hist_trigger_show(struct seq_file *m, 4847 struct event_trigger_data *data, int n) 4848 { 4849 struct hist_trigger_data *hist_data; 4850 int n_entries; 4851 4852 if (n > 0) 4853 seq_puts(m, "\n\n"); 4854 4855 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4856 data->ops->print(m, data->ops, data); 4857 seq_puts(m, "#\n\n"); 4858 4859 hist_data = data->private_data; 4860 n_entries = print_entries(m, hist_data); 4861 if (n_entries < 0) 4862 n_entries = 0; 4863 4864 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 4865 (u64)atomic64_read(&hist_data->map->hits), 4866 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 4867 } 4868 4869 static int hist_show(struct seq_file *m, void *v) 4870 { 4871 struct event_trigger_data *data; 4872 struct trace_event_file *event_file; 4873 int n = 0, ret = 0; 4874 4875 mutex_lock(&event_mutex); 4876 4877 event_file = event_file_data(m->private); 4878 if (unlikely(!event_file)) { 4879 ret = -ENODEV; 4880 goto out_unlock; 4881 } 4882 4883 list_for_each_entry_rcu(data, &event_file->triggers, list) { 4884 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4885 hist_trigger_show(m, data, n++); 4886 } 4887 4888 if (have_hist_err()) { 4889 seq_printf(m, "\nERROR: %s\n", hist_err_str); 4890 seq_printf(m, " Last command: %s\n", last_hist_cmd); 4891 } 4892 4893 out_unlock: 4894 mutex_unlock(&event_mutex); 4895 4896 return ret; 4897 } 4898 4899 static int event_hist_open(struct inode *inode, struct file *file) 4900 { 4901 return single_open(file, hist_show, file); 4902 } 4903 4904 const struct file_operations event_hist_fops = { 4905 .open = event_hist_open, 4906 .read = seq_read, 4907 .llseek = seq_lseek, 4908 .release = single_release, 4909 }; 4910 4911 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 4912 { 4913 const char *field_name = hist_field_name(hist_field, 0); 4914 4915 if (hist_field->var.name) 4916 seq_printf(m, "%s=", hist_field->var.name); 4917 4918 if (hist_field->flags & HIST_FIELD_FL_CPU) 4919 seq_puts(m, "cpu"); 4920 else if (field_name) { 4921 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 4922 hist_field->flags & HIST_FIELD_FL_ALIAS) 4923 seq_putc(m, '$'); 4924 seq_printf(m, "%s", field_name); 4925 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 4926 seq_puts(m, "common_timestamp"); 4927 4928 if (hist_field->flags) { 4929 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 4930 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 4931 const char *flags = get_hist_field_flags(hist_field); 4932 4933 if (flags) 4934 seq_printf(m, ".%s", flags); 4935 } 4936 } 4937 } 4938 4939 static int event_hist_trigger_print(struct seq_file *m, 4940 struct event_trigger_ops *ops, 4941 struct event_trigger_data *data) 4942 { 4943 struct hist_trigger_data *hist_data = data->private_data; 4944 struct hist_field *field; 4945 bool have_var = false; 4946 unsigned int i; 4947 4948 seq_puts(m, "hist:"); 4949 4950 if (data->name) 4951 seq_printf(m, "%s:", data->name); 4952 4953 seq_puts(m, "keys="); 4954 4955 for_each_hist_key_field(i, hist_data) { 4956 field = hist_data->fields[i]; 4957 4958 if (i > hist_data->n_vals) 4959 seq_puts(m, ","); 4960 4961 if (field->flags & HIST_FIELD_FL_STACKTRACE) 4962 seq_puts(m, "stacktrace"); 4963 else 4964 hist_field_print(m, field); 4965 } 4966 4967 seq_puts(m, ":vals="); 4968 4969 for_each_hist_val_field(i, hist_data) { 4970 field = hist_data->fields[i]; 4971 if (field->flags & HIST_FIELD_FL_VAR) { 4972 have_var = true; 4973 continue; 4974 } 4975 4976 if (i == HITCOUNT_IDX) 4977 seq_puts(m, "hitcount"); 4978 else { 4979 seq_puts(m, ","); 4980 hist_field_print(m, field); 4981 } 4982 } 4983 4984 if (have_var) { 4985 unsigned int n = 0; 4986 4987 seq_puts(m, ":"); 4988 4989 for_each_hist_val_field(i, hist_data) { 4990 field = hist_data->fields[i]; 4991 4992 if (field->flags & HIST_FIELD_FL_VAR) { 4993 if (n++) 4994 seq_puts(m, ","); 4995 hist_field_print(m, field); 4996 } 4997 } 4998 } 4999 5000 seq_puts(m, ":sort="); 5001 5002 for (i = 0; i < hist_data->n_sort_keys; i++) { 5003 struct tracing_map_sort_key *sort_key; 5004 unsigned int idx, first_key_idx; 5005 5006 /* skip VAR vals */ 5007 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5008 5009 sort_key = &hist_data->sort_keys[i]; 5010 idx = sort_key->field_idx; 5011 5012 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5013 return -EINVAL; 5014 5015 if (i > 0) 5016 seq_puts(m, ","); 5017 5018 if (idx == HITCOUNT_IDX) 5019 seq_puts(m, "hitcount"); 5020 else { 5021 if (idx >= first_key_idx) 5022 idx += hist_data->n_vars; 5023 hist_field_print(m, hist_data->fields[idx]); 5024 } 5025 5026 if (sort_key->descending) 5027 seq_puts(m, ".descending"); 5028 } 5029 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5030 if (hist_data->enable_timestamps) 5031 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5032 5033 print_actions_spec(m, hist_data); 5034 5035 if (data->filter_str) 5036 seq_printf(m, " if %s", data->filter_str); 5037 5038 if (data->paused) 5039 seq_puts(m, " [paused]"); 5040 else 5041 seq_puts(m, " [active]"); 5042 5043 seq_putc(m, '\n'); 5044 5045 return 0; 5046 } 5047 5048 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5049 struct event_trigger_data *data) 5050 { 5051 struct hist_trigger_data *hist_data = data->private_data; 5052 5053 if (!data->ref && hist_data->attrs->name) 5054 save_named_trigger(hist_data->attrs->name, data); 5055 5056 data->ref++; 5057 5058 return 0; 5059 } 5060 5061 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5062 { 5063 struct trace_event_file *file; 5064 unsigned int i; 5065 char *cmd; 5066 int ret; 5067 5068 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5069 file = hist_data->field_var_hists[i]->hist_data->event_file; 5070 cmd = hist_data->field_var_hists[i]->cmd; 5071 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5072 "!hist", "hist", cmd); 5073 } 5074 } 5075 5076 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5077 struct event_trigger_data *data) 5078 { 5079 struct hist_trigger_data *hist_data = data->private_data; 5080 5081 if (WARN_ON_ONCE(data->ref <= 0)) 5082 return; 5083 5084 data->ref--; 5085 if (!data->ref) { 5086 if (data->name) 5087 del_named_trigger(data); 5088 5089 trigger_data_free(data); 5090 5091 remove_hist_vars(hist_data); 5092 5093 unregister_field_var_hists(hist_data); 5094 5095 destroy_hist_data(hist_data); 5096 } 5097 } 5098 5099 static struct event_trigger_ops event_hist_trigger_ops = { 5100 .func = event_hist_trigger, 5101 .print = event_hist_trigger_print, 5102 .init = event_hist_trigger_init, 5103 .free = event_hist_trigger_free, 5104 }; 5105 5106 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5107 struct event_trigger_data *data) 5108 { 5109 data->ref++; 5110 5111 save_named_trigger(data->named_data->name, data); 5112 5113 event_hist_trigger_init(ops, data->named_data); 5114 5115 return 0; 5116 } 5117 5118 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5119 struct event_trigger_data *data) 5120 { 5121 if (WARN_ON_ONCE(data->ref <= 0)) 5122 return; 5123 5124 event_hist_trigger_free(ops, data->named_data); 5125 5126 data->ref--; 5127 if (!data->ref) { 5128 del_named_trigger(data); 5129 trigger_data_free(data); 5130 } 5131 } 5132 5133 static struct event_trigger_ops event_hist_trigger_named_ops = { 5134 .func = event_hist_trigger, 5135 .print = event_hist_trigger_print, 5136 .init = event_hist_trigger_named_init, 5137 .free = event_hist_trigger_named_free, 5138 }; 5139 5140 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5141 char *param) 5142 { 5143 return &event_hist_trigger_ops; 5144 } 5145 5146 static void hist_clear(struct event_trigger_data *data) 5147 { 5148 struct hist_trigger_data *hist_data = data->private_data; 5149 5150 if (data->name) 5151 pause_named_trigger(data); 5152 5153 tracepoint_synchronize_unregister(); 5154 5155 tracing_map_clear(hist_data->map); 5156 5157 if (data->name) 5158 unpause_named_trigger(data); 5159 } 5160 5161 static bool compatible_field(struct ftrace_event_field *field, 5162 struct ftrace_event_field *test_field) 5163 { 5164 if (field == test_field) 5165 return true; 5166 if (field == NULL || test_field == NULL) 5167 return false; 5168 if (strcmp(field->name, test_field->name) != 0) 5169 return false; 5170 if (strcmp(field->type, test_field->type) != 0) 5171 return false; 5172 if (field->size != test_field->size) 5173 return false; 5174 if (field->is_signed != test_field->is_signed) 5175 return false; 5176 5177 return true; 5178 } 5179 5180 static bool hist_trigger_match(struct event_trigger_data *data, 5181 struct event_trigger_data *data_test, 5182 struct event_trigger_data *named_data, 5183 bool ignore_filter) 5184 { 5185 struct tracing_map_sort_key *sort_key, *sort_key_test; 5186 struct hist_trigger_data *hist_data, *hist_data_test; 5187 struct hist_field *key_field, *key_field_test; 5188 unsigned int i; 5189 5190 if (named_data && (named_data != data_test) && 5191 (named_data != data_test->named_data)) 5192 return false; 5193 5194 if (!named_data && is_named_trigger(data_test)) 5195 return false; 5196 5197 hist_data = data->private_data; 5198 hist_data_test = data_test->private_data; 5199 5200 if (hist_data->n_vals != hist_data_test->n_vals || 5201 hist_data->n_fields != hist_data_test->n_fields || 5202 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5203 return false; 5204 5205 if (!ignore_filter) { 5206 if ((data->filter_str && !data_test->filter_str) || 5207 (!data->filter_str && data_test->filter_str)) 5208 return false; 5209 } 5210 5211 for_each_hist_field(i, hist_data) { 5212 key_field = hist_data->fields[i]; 5213 key_field_test = hist_data_test->fields[i]; 5214 5215 if (key_field->flags != key_field_test->flags) 5216 return false; 5217 if (!compatible_field(key_field->field, key_field_test->field)) 5218 return false; 5219 if (key_field->offset != key_field_test->offset) 5220 return false; 5221 if (key_field->size != key_field_test->size) 5222 return false; 5223 if (key_field->is_signed != key_field_test->is_signed) 5224 return false; 5225 if (!!key_field->var.name != !!key_field_test->var.name) 5226 return false; 5227 if (key_field->var.name && 5228 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5229 return false; 5230 } 5231 5232 for (i = 0; i < hist_data->n_sort_keys; i++) { 5233 sort_key = &hist_data->sort_keys[i]; 5234 sort_key_test = &hist_data_test->sort_keys[i]; 5235 5236 if (sort_key->field_idx != sort_key_test->field_idx || 5237 sort_key->descending != sort_key_test->descending) 5238 return false; 5239 } 5240 5241 if (!ignore_filter && data->filter_str && 5242 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5243 return false; 5244 5245 if (!actions_match(hist_data, hist_data_test)) 5246 return false; 5247 5248 return true; 5249 } 5250 5251 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5252 struct event_trigger_data *data, 5253 struct trace_event_file *file) 5254 { 5255 struct hist_trigger_data *hist_data = data->private_data; 5256 struct event_trigger_data *test, *named_data = NULL; 5257 int ret = 0; 5258 5259 if (hist_data->attrs->name) { 5260 named_data = find_named_trigger(hist_data->attrs->name); 5261 if (named_data) { 5262 if (!hist_trigger_match(data, named_data, named_data, 5263 true)) { 5264 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name); 5265 ret = -EINVAL; 5266 goto out; 5267 } 5268 } 5269 } 5270 5271 if (hist_data->attrs->name && !named_data) 5272 goto new; 5273 5274 list_for_each_entry_rcu(test, &file->triggers, list) { 5275 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5276 if (!hist_trigger_match(data, test, named_data, false)) 5277 continue; 5278 if (hist_data->attrs->pause) 5279 test->paused = true; 5280 else if (hist_data->attrs->cont) 5281 test->paused = false; 5282 else if (hist_data->attrs->clear) 5283 hist_clear(test); 5284 else { 5285 hist_err("Hist trigger already exists", NULL); 5286 ret = -EEXIST; 5287 } 5288 goto out; 5289 } 5290 } 5291 new: 5292 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5293 hist_err("Can't clear or continue a nonexistent hist trigger", NULL); 5294 ret = -ENOENT; 5295 goto out; 5296 } 5297 5298 if (hist_data->attrs->pause) 5299 data->paused = true; 5300 5301 if (named_data) { 5302 data->private_data = named_data->private_data; 5303 set_named_trigger_data(data, named_data); 5304 data->ops = &event_hist_trigger_named_ops; 5305 } 5306 5307 if (data->ops->init) { 5308 ret = data->ops->init(data->ops, data); 5309 if (ret < 0) 5310 goto out; 5311 } 5312 5313 if (hist_data->enable_timestamps) { 5314 char *clock = hist_data->attrs->clock; 5315 5316 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5317 if (ret) { 5318 hist_err("Couldn't set trace_clock: ", clock); 5319 goto out; 5320 } 5321 5322 tracing_set_time_stamp_abs(file->tr, true); 5323 } 5324 5325 if (named_data) 5326 destroy_hist_data(hist_data); 5327 5328 ret++; 5329 out: 5330 return ret; 5331 } 5332 5333 static int hist_trigger_enable(struct event_trigger_data *data, 5334 struct trace_event_file *file) 5335 { 5336 int ret = 0; 5337 5338 list_add_tail_rcu(&data->list, &file->triggers); 5339 5340 update_cond_flag(file); 5341 5342 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5343 list_del_rcu(&data->list); 5344 update_cond_flag(file); 5345 ret--; 5346 } 5347 5348 return ret; 5349 } 5350 5351 static bool have_hist_trigger_match(struct event_trigger_data *data, 5352 struct trace_event_file *file) 5353 { 5354 struct hist_trigger_data *hist_data = data->private_data; 5355 struct event_trigger_data *test, *named_data = NULL; 5356 bool match = false; 5357 5358 if (hist_data->attrs->name) 5359 named_data = find_named_trigger(hist_data->attrs->name); 5360 5361 list_for_each_entry_rcu(test, &file->triggers, list) { 5362 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5363 if (hist_trigger_match(data, test, named_data, false)) { 5364 match = true; 5365 break; 5366 } 5367 } 5368 } 5369 5370 return match; 5371 } 5372 5373 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5374 struct trace_event_file *file) 5375 { 5376 struct hist_trigger_data *hist_data = data->private_data; 5377 struct event_trigger_data *test, *named_data = NULL; 5378 5379 if (hist_data->attrs->name) 5380 named_data = find_named_trigger(hist_data->attrs->name); 5381 5382 list_for_each_entry_rcu(test, &file->triggers, list) { 5383 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5384 if (!hist_trigger_match(data, test, named_data, false)) 5385 continue; 5386 hist_data = test->private_data; 5387 if (check_var_refs(hist_data)) 5388 return true; 5389 break; 5390 } 5391 } 5392 5393 return false; 5394 } 5395 5396 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5397 struct event_trigger_data *data, 5398 struct trace_event_file *file) 5399 { 5400 struct hist_trigger_data *hist_data = data->private_data; 5401 struct event_trigger_data *test, *named_data = NULL; 5402 bool unregistered = false; 5403 5404 if (hist_data->attrs->name) 5405 named_data = find_named_trigger(hist_data->attrs->name); 5406 5407 list_for_each_entry_rcu(test, &file->triggers, list) { 5408 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5409 if (!hist_trigger_match(data, test, named_data, false)) 5410 continue; 5411 unregistered = true; 5412 list_del_rcu(&test->list); 5413 trace_event_trigger_enable_disable(file, 0); 5414 update_cond_flag(file); 5415 break; 5416 } 5417 } 5418 5419 if (unregistered && test->ops->free) 5420 test->ops->free(test->ops, test); 5421 5422 if (hist_data->enable_timestamps) { 5423 if (!hist_data->remove || unregistered) 5424 tracing_set_time_stamp_abs(file->tr, false); 5425 } 5426 } 5427 5428 static bool hist_file_check_refs(struct trace_event_file *file) 5429 { 5430 struct hist_trigger_data *hist_data; 5431 struct event_trigger_data *test; 5432 5433 list_for_each_entry_rcu(test, &file->triggers, list) { 5434 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5435 hist_data = test->private_data; 5436 if (check_var_refs(hist_data)) 5437 return true; 5438 } 5439 } 5440 5441 return false; 5442 } 5443 5444 static void hist_unreg_all(struct trace_event_file *file) 5445 { 5446 struct event_trigger_data *test, *n; 5447 struct hist_trigger_data *hist_data; 5448 struct synth_event *se; 5449 const char *se_name; 5450 5451 if (hist_file_check_refs(file)) 5452 return; 5453 5454 list_for_each_entry_safe(test, n, &file->triggers, list) { 5455 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5456 hist_data = test->private_data; 5457 list_del_rcu(&test->list); 5458 trace_event_trigger_enable_disable(file, 0); 5459 5460 mutex_lock(&synth_event_mutex); 5461 se_name = trace_event_name(file->event_call); 5462 se = find_synth_event(se_name); 5463 if (se) 5464 se->ref--; 5465 mutex_unlock(&synth_event_mutex); 5466 5467 update_cond_flag(file); 5468 if (hist_data->enable_timestamps) 5469 tracing_set_time_stamp_abs(file->tr, false); 5470 if (test->ops->free) 5471 test->ops->free(test->ops, test); 5472 } 5473 } 5474 } 5475 5476 static int event_hist_trigger_func(struct event_command *cmd_ops, 5477 struct trace_event_file *file, 5478 char *glob, char *cmd, char *param) 5479 { 5480 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 5481 struct event_trigger_data *trigger_data; 5482 struct hist_trigger_attrs *attrs; 5483 struct event_trigger_ops *trigger_ops; 5484 struct hist_trigger_data *hist_data; 5485 struct synth_event *se; 5486 const char *se_name; 5487 bool remove = false; 5488 char *trigger, *p; 5489 int ret = 0; 5490 5491 if (glob && strlen(glob)) { 5492 last_cmd_set(param); 5493 hist_err_clear(); 5494 } 5495 5496 if (!param) 5497 return -EINVAL; 5498 5499 if (glob[0] == '!') 5500 remove = true; 5501 5502 /* 5503 * separate the trigger from the filter (k:v [if filter]) 5504 * allowing for whitespace in the trigger 5505 */ 5506 p = trigger = param; 5507 do { 5508 p = strstr(p, "if"); 5509 if (!p) 5510 break; 5511 if (p == param) 5512 return -EINVAL; 5513 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 5514 p++; 5515 continue; 5516 } 5517 if (p >= param + strlen(param) - strlen("if") - 1) 5518 return -EINVAL; 5519 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') { 5520 p++; 5521 continue; 5522 } 5523 break; 5524 } while (p); 5525 5526 if (!p) 5527 param = NULL; 5528 else { 5529 *(p - 1) = '\0'; 5530 param = strstrip(p); 5531 trigger = strstrip(trigger); 5532 } 5533 5534 attrs = parse_hist_trigger_attrs(trigger); 5535 if (IS_ERR(attrs)) 5536 return PTR_ERR(attrs); 5537 5538 if (attrs->map_bits) 5539 hist_trigger_bits = attrs->map_bits; 5540 5541 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 5542 if (IS_ERR(hist_data)) { 5543 destroy_hist_trigger_attrs(attrs); 5544 return PTR_ERR(hist_data); 5545 } 5546 5547 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 5548 5549 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 5550 if (!trigger_data) { 5551 ret = -ENOMEM; 5552 goto out_free; 5553 } 5554 5555 trigger_data->count = -1; 5556 trigger_data->ops = trigger_ops; 5557 trigger_data->cmd_ops = cmd_ops; 5558 5559 INIT_LIST_HEAD(&trigger_data->list); 5560 RCU_INIT_POINTER(trigger_data->filter, NULL); 5561 5562 trigger_data->private_data = hist_data; 5563 5564 /* if param is non-empty, it's supposed to be a filter */ 5565 if (param && cmd_ops->set_filter) { 5566 ret = cmd_ops->set_filter(param, trigger_data, file); 5567 if (ret < 0) 5568 goto out_free; 5569 } 5570 5571 if (remove) { 5572 if (!have_hist_trigger_match(trigger_data, file)) 5573 goto out_free; 5574 5575 if (hist_trigger_check_refs(trigger_data, file)) { 5576 ret = -EBUSY; 5577 goto out_free; 5578 } 5579 5580 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5581 5582 mutex_lock(&synth_event_mutex); 5583 se_name = trace_event_name(file->event_call); 5584 se = find_synth_event(se_name); 5585 if (se) 5586 se->ref--; 5587 mutex_unlock(&synth_event_mutex); 5588 5589 ret = 0; 5590 goto out_free; 5591 } 5592 5593 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 5594 /* 5595 * The above returns on success the # of triggers registered, 5596 * but if it didn't register any it returns zero. Consider no 5597 * triggers registered a failure too. 5598 */ 5599 if (!ret) { 5600 if (!(attrs->pause || attrs->cont || attrs->clear)) 5601 ret = -ENOENT; 5602 goto out_free; 5603 } else if (ret < 0) 5604 goto out_free; 5605 5606 if (get_named_trigger_data(trigger_data)) 5607 goto enable; 5608 5609 if (has_hist_vars(hist_data)) 5610 save_hist_vars(hist_data); 5611 5612 ret = create_actions(hist_data, file); 5613 if (ret) 5614 goto out_unreg; 5615 5616 ret = tracing_map_init(hist_data->map); 5617 if (ret) 5618 goto out_unreg; 5619 enable: 5620 ret = hist_trigger_enable(trigger_data, file); 5621 if (ret) 5622 goto out_unreg; 5623 5624 mutex_lock(&synth_event_mutex); 5625 se_name = trace_event_name(file->event_call); 5626 se = find_synth_event(se_name); 5627 if (se) 5628 se->ref++; 5629 mutex_unlock(&synth_event_mutex); 5630 5631 /* Just return zero, not the number of registered triggers */ 5632 ret = 0; 5633 out: 5634 if (ret == 0) 5635 hist_err_clear(); 5636 5637 return ret; 5638 out_unreg: 5639 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5640 out_free: 5641 if (cmd_ops->set_filter) 5642 cmd_ops->set_filter(NULL, trigger_data, NULL); 5643 5644 remove_hist_vars(hist_data); 5645 5646 kfree(trigger_data); 5647 5648 destroy_hist_data(hist_data); 5649 goto out; 5650 } 5651 5652 static struct event_command trigger_hist_cmd = { 5653 .name = "hist", 5654 .trigger_type = ETT_EVENT_HIST, 5655 .flags = EVENT_CMD_FL_NEEDS_REC, 5656 .func = event_hist_trigger_func, 5657 .reg = hist_register_trigger, 5658 .unreg = hist_unregister_trigger, 5659 .unreg_all = hist_unreg_all, 5660 .get_trigger_ops = event_hist_get_trigger_ops, 5661 .set_filter = set_trigger_filter, 5662 }; 5663 5664 __init int register_trigger_hist_cmd(void) 5665 { 5666 int ret; 5667 5668 ret = register_event_command(&trigger_hist_cmd); 5669 WARN_ON(ret < 0); 5670 5671 return ret; 5672 } 5673 5674 static void 5675 hist_enable_trigger(struct event_trigger_data *data, void *rec, 5676 struct ring_buffer_event *event) 5677 { 5678 struct enable_trigger_data *enable_data = data->private_data; 5679 struct event_trigger_data *test; 5680 5681 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) { 5682 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5683 if (enable_data->enable) 5684 test->paused = false; 5685 else 5686 test->paused = true; 5687 } 5688 } 5689 } 5690 5691 static void 5692 hist_enable_count_trigger(struct event_trigger_data *data, void *rec, 5693 struct ring_buffer_event *event) 5694 { 5695 if (!data->count) 5696 return; 5697 5698 if (data->count != -1) 5699 (data->count)--; 5700 5701 hist_enable_trigger(data, rec, event); 5702 } 5703 5704 static struct event_trigger_ops hist_enable_trigger_ops = { 5705 .func = hist_enable_trigger, 5706 .print = event_enable_trigger_print, 5707 .init = event_trigger_init, 5708 .free = event_enable_trigger_free, 5709 }; 5710 5711 static struct event_trigger_ops hist_enable_count_trigger_ops = { 5712 .func = hist_enable_count_trigger, 5713 .print = event_enable_trigger_print, 5714 .init = event_trigger_init, 5715 .free = event_enable_trigger_free, 5716 }; 5717 5718 static struct event_trigger_ops hist_disable_trigger_ops = { 5719 .func = hist_enable_trigger, 5720 .print = event_enable_trigger_print, 5721 .init = event_trigger_init, 5722 .free = event_enable_trigger_free, 5723 }; 5724 5725 static struct event_trigger_ops hist_disable_count_trigger_ops = { 5726 .func = hist_enable_count_trigger, 5727 .print = event_enable_trigger_print, 5728 .init = event_trigger_init, 5729 .free = event_enable_trigger_free, 5730 }; 5731 5732 static struct event_trigger_ops * 5733 hist_enable_get_trigger_ops(char *cmd, char *param) 5734 { 5735 struct event_trigger_ops *ops; 5736 bool enable; 5737 5738 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 5739 5740 if (enable) 5741 ops = param ? &hist_enable_count_trigger_ops : 5742 &hist_enable_trigger_ops; 5743 else 5744 ops = param ? &hist_disable_count_trigger_ops : 5745 &hist_disable_trigger_ops; 5746 5747 return ops; 5748 } 5749 5750 static void hist_enable_unreg_all(struct trace_event_file *file) 5751 { 5752 struct event_trigger_data *test, *n; 5753 5754 list_for_each_entry_safe(test, n, &file->triggers, list) { 5755 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 5756 list_del_rcu(&test->list); 5757 update_cond_flag(file); 5758 trace_event_trigger_enable_disable(file, 0); 5759 if (test->ops->free) 5760 test->ops->free(test->ops, test); 5761 } 5762 } 5763 } 5764 5765 static struct event_command trigger_hist_enable_cmd = { 5766 .name = ENABLE_HIST_STR, 5767 .trigger_type = ETT_HIST_ENABLE, 5768 .func = event_enable_trigger_func, 5769 .reg = event_enable_register_trigger, 5770 .unreg = event_enable_unregister_trigger, 5771 .unreg_all = hist_enable_unreg_all, 5772 .get_trigger_ops = hist_enable_get_trigger_ops, 5773 .set_filter = set_trigger_filter, 5774 }; 5775 5776 static struct event_command trigger_hist_disable_cmd = { 5777 .name = DISABLE_HIST_STR, 5778 .trigger_type = ETT_HIST_ENABLE, 5779 .func = event_enable_trigger_func, 5780 .reg = event_enable_register_trigger, 5781 .unreg = event_enable_unregister_trigger, 5782 .unreg_all = hist_enable_unreg_all, 5783 .get_trigger_ops = hist_enable_get_trigger_ops, 5784 .set_filter = set_trigger_filter, 5785 }; 5786 5787 static __init void unregister_trigger_hist_enable_disable_cmds(void) 5788 { 5789 unregister_event_command(&trigger_hist_enable_cmd); 5790 unregister_event_command(&trigger_hist_disable_cmd); 5791 } 5792 5793 __init int register_trigger_hist_enable_disable_cmds(void) 5794 { 5795 int ret; 5796 5797 ret = register_event_command(&trigger_hist_enable_cmd); 5798 if (WARN_ON(ret < 0)) 5799 return ret; 5800 ret = register_event_command(&trigger_hist_disable_cmd); 5801 if (WARN_ON(ret < 0)) 5802 unregister_trigger_hist_enable_disable_cmds(); 5803 5804 return ret; 5805 } 5806 5807 static __init int trace_events_hist_init(void) 5808 { 5809 struct dentry *entry = NULL; 5810 struct dentry *d_tracer; 5811 int err = 0; 5812 5813 d_tracer = tracing_init_dentry(); 5814 if (IS_ERR(d_tracer)) { 5815 err = PTR_ERR(d_tracer); 5816 goto err; 5817 } 5818 5819 entry = tracefs_create_file("synthetic_events", 0644, d_tracer, 5820 NULL, &synth_events_fops); 5821 if (!entry) { 5822 err = -ENODEV; 5823 goto err; 5824 } 5825 5826 return err; 5827 err: 5828 pr_warn("Could not create tracefs 'synthetic_events' entry\n"); 5829 5830 return err; 5831 } 5832 5833 fs_initcall(trace_events_hist_init); 5834