1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/mutex.h> 12 #include <linux/slab.h> 13 #include <linux/stacktrace.h> 14 #include <linux/rculist.h> 15 #include <linux/tracefs.h> 16 17 /* for gfp flag names */ 18 #include <linux/trace_events.h> 19 #include <trace/events/mmflags.h> 20 21 #include "tracing_map.h" 22 #include "trace_synth.h" 23 24 #define ERRORS \ 25 C(NONE, "No error"), \ 26 C(DUPLICATE_VAR, "Variable already defined"), \ 27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 28 C(TOO_MANY_VARS, "Too many variables defined"), \ 29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 51 C(TOO_MANY_PARAMS, "Too many action params"), \ 52 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 53 C(INVALID_PARAM, "Invalid action param"), \ 54 C(ACTION_NOT_FOUND, "No action found"), \ 55 C(NO_SAVE_PARAMS, "No params found for save()"), \ 56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 57 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 58 C(NO_CLOSING_PAREN, "No closing paren found"), \ 59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 62 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 63 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 66 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), 69 70 #undef C 71 #define C(a, b) HIST_ERR_##a 72 73 enum { ERRORS }; 74 75 #undef C 76 #define C(a, b) b 77 78 static const char *err_text[] = { ERRORS }; 79 80 struct hist_field; 81 82 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 83 struct tracing_map_elt *elt, 84 struct trace_buffer *buffer, 85 struct ring_buffer_event *rbe, 86 void *event); 87 88 #define HIST_FIELD_OPERANDS_MAX 2 89 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 90 #define HIST_ACTIONS_MAX 8 91 92 enum field_op_id { 93 FIELD_OP_NONE, 94 FIELD_OP_PLUS, 95 FIELD_OP_MINUS, 96 FIELD_OP_UNARY_MINUS, 97 }; 98 99 /* 100 * A hist_var (histogram variable) contains variable information for 101 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 102 * flag set. A hist_var has a variable name e.g. ts0, and is 103 * associated with a given histogram trigger, as specified by 104 * hist_data. The hist_var idx is the unique index assigned to the 105 * variable by the hist trigger's tracing_map. The idx is what is 106 * used to set a variable's value and, by a variable reference, to 107 * retrieve it. 108 */ 109 struct hist_var { 110 char *name; 111 struct hist_trigger_data *hist_data; 112 unsigned int idx; 113 }; 114 115 struct hist_field { 116 struct ftrace_event_field *field; 117 unsigned long flags; 118 hist_field_fn_t fn; 119 unsigned int ref; 120 unsigned int size; 121 unsigned int offset; 122 unsigned int is_signed; 123 const char *type; 124 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 125 struct hist_trigger_data *hist_data; 126 127 /* 128 * Variable fields contain variable-specific info in var. 129 */ 130 struct hist_var var; 131 enum field_op_id operator; 132 char *system; 133 char *event_name; 134 135 /* 136 * The name field is used for EXPR and VAR_REF fields. VAR 137 * fields contain the variable name in var.name. 138 */ 139 char *name; 140 141 /* 142 * When a histogram trigger is hit, if it has any references 143 * to variables, the values of those variables are collected 144 * into a var_ref_vals array by resolve_var_refs(). The 145 * current value of each variable is read from the tracing_map 146 * using the hist field's hist_var.idx and entered into the 147 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 148 */ 149 unsigned int var_ref_idx; 150 bool read_once; 151 152 unsigned int var_str_idx; 153 }; 154 155 static u64 hist_field_none(struct hist_field *field, 156 struct tracing_map_elt *elt, 157 struct trace_buffer *buffer, 158 struct ring_buffer_event *rbe, 159 void *event) 160 { 161 return 0; 162 } 163 164 static u64 hist_field_counter(struct hist_field *field, 165 struct tracing_map_elt *elt, 166 struct trace_buffer *buffer, 167 struct ring_buffer_event *rbe, 168 void *event) 169 { 170 return 1; 171 } 172 173 static u64 hist_field_string(struct hist_field *hist_field, 174 struct tracing_map_elt *elt, 175 struct trace_buffer *buffer, 176 struct ring_buffer_event *rbe, 177 void *event) 178 { 179 char *addr = (char *)(event + hist_field->field->offset); 180 181 return (u64)(unsigned long)addr; 182 } 183 184 static u64 hist_field_dynstring(struct hist_field *hist_field, 185 struct tracing_map_elt *elt, 186 struct trace_buffer *buffer, 187 struct ring_buffer_event *rbe, 188 void *event) 189 { 190 u32 str_item = *(u32 *)(event + hist_field->field->offset); 191 int str_loc = str_item & 0xffff; 192 char *addr = (char *)(event + str_loc); 193 194 return (u64)(unsigned long)addr; 195 } 196 197 static u64 hist_field_pstring(struct hist_field *hist_field, 198 struct tracing_map_elt *elt, 199 struct trace_buffer *buffer, 200 struct ring_buffer_event *rbe, 201 void *event) 202 { 203 char **addr = (char **)(event + hist_field->field->offset); 204 205 return (u64)(unsigned long)*addr; 206 } 207 208 static u64 hist_field_log2(struct hist_field *hist_field, 209 struct tracing_map_elt *elt, 210 struct trace_buffer *buffer, 211 struct ring_buffer_event *rbe, 212 void *event) 213 { 214 struct hist_field *operand = hist_field->operands[0]; 215 216 u64 val = operand->fn(operand, elt, buffer, rbe, event); 217 218 return (u64) ilog2(roundup_pow_of_two(val)); 219 } 220 221 static u64 hist_field_plus(struct hist_field *hist_field, 222 struct tracing_map_elt *elt, 223 struct trace_buffer *buffer, 224 struct ring_buffer_event *rbe, 225 void *event) 226 { 227 struct hist_field *operand1 = hist_field->operands[0]; 228 struct hist_field *operand2 = hist_field->operands[1]; 229 230 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); 231 u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event); 232 233 return val1 + val2; 234 } 235 236 static u64 hist_field_minus(struct hist_field *hist_field, 237 struct tracing_map_elt *elt, 238 struct trace_buffer *buffer, 239 struct ring_buffer_event *rbe, 240 void *event) 241 { 242 struct hist_field *operand1 = hist_field->operands[0]; 243 struct hist_field *operand2 = hist_field->operands[1]; 244 245 u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); 246 u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event); 247 248 return val1 - val2; 249 } 250 251 static u64 hist_field_unary_minus(struct hist_field *hist_field, 252 struct tracing_map_elt *elt, 253 struct trace_buffer *buffer, 254 struct ring_buffer_event *rbe, 255 void *event) 256 { 257 struct hist_field *operand = hist_field->operands[0]; 258 259 s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event); 260 u64 val = (u64)-sval; 261 262 return val; 263 } 264 265 #define DEFINE_HIST_FIELD_FN(type) \ 266 static u64 hist_field_##type(struct hist_field *hist_field, \ 267 struct tracing_map_elt *elt, \ 268 struct trace_buffer *buffer, \ 269 struct ring_buffer_event *rbe, \ 270 void *event) \ 271 { \ 272 type *addr = (type *)(event + hist_field->field->offset); \ 273 \ 274 return (u64)(unsigned long)*addr; \ 275 } 276 277 DEFINE_HIST_FIELD_FN(s64); 278 DEFINE_HIST_FIELD_FN(u64); 279 DEFINE_HIST_FIELD_FN(s32); 280 DEFINE_HIST_FIELD_FN(u32); 281 DEFINE_HIST_FIELD_FN(s16); 282 DEFINE_HIST_FIELD_FN(u16); 283 DEFINE_HIST_FIELD_FN(s8); 284 DEFINE_HIST_FIELD_FN(u8); 285 286 #define for_each_hist_field(i, hist_data) \ 287 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 288 289 #define for_each_hist_val_field(i, hist_data) \ 290 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 291 292 #define for_each_hist_key_field(i, hist_data) \ 293 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 294 295 #define HIST_STACKTRACE_DEPTH 16 296 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long)) 297 #define HIST_STACKTRACE_SKIP 5 298 299 #define HITCOUNT_IDX 0 300 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 301 302 enum hist_field_flags { 303 HIST_FIELD_FL_HITCOUNT = 1 << 0, 304 HIST_FIELD_FL_KEY = 1 << 1, 305 HIST_FIELD_FL_STRING = 1 << 2, 306 HIST_FIELD_FL_HEX = 1 << 3, 307 HIST_FIELD_FL_SYM = 1 << 4, 308 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 309 HIST_FIELD_FL_EXECNAME = 1 << 6, 310 HIST_FIELD_FL_SYSCALL = 1 << 7, 311 HIST_FIELD_FL_STACKTRACE = 1 << 8, 312 HIST_FIELD_FL_LOG2 = 1 << 9, 313 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 314 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 315 HIST_FIELD_FL_VAR = 1 << 12, 316 HIST_FIELD_FL_EXPR = 1 << 13, 317 HIST_FIELD_FL_VAR_REF = 1 << 14, 318 HIST_FIELD_FL_CPU = 1 << 15, 319 HIST_FIELD_FL_ALIAS = 1 << 16, 320 }; 321 322 struct var_defs { 323 unsigned int n_vars; 324 char *name[TRACING_MAP_VARS_MAX]; 325 char *expr[TRACING_MAP_VARS_MAX]; 326 }; 327 328 struct hist_trigger_attrs { 329 char *keys_str; 330 char *vals_str; 331 char *sort_key_str; 332 char *name; 333 char *clock; 334 bool pause; 335 bool cont; 336 bool clear; 337 bool ts_in_usecs; 338 unsigned int map_bits; 339 340 char *assignment_str[TRACING_MAP_VARS_MAX]; 341 unsigned int n_assignments; 342 343 char *action_str[HIST_ACTIONS_MAX]; 344 unsigned int n_actions; 345 346 struct var_defs var_defs; 347 }; 348 349 struct field_var { 350 struct hist_field *var; 351 struct hist_field *val; 352 }; 353 354 struct field_var_hist { 355 struct hist_trigger_data *hist_data; 356 char *cmd; 357 }; 358 359 struct hist_trigger_data { 360 struct hist_field *fields[HIST_FIELDS_MAX]; 361 unsigned int n_vals; 362 unsigned int n_keys; 363 unsigned int n_fields; 364 unsigned int n_vars; 365 unsigned int n_var_str; 366 unsigned int key_size; 367 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 368 unsigned int n_sort_keys; 369 struct trace_event_file *event_file; 370 struct hist_trigger_attrs *attrs; 371 struct tracing_map *map; 372 bool enable_timestamps; 373 bool remove; 374 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 375 unsigned int n_var_refs; 376 377 struct action_data *actions[HIST_ACTIONS_MAX]; 378 unsigned int n_actions; 379 380 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 381 unsigned int n_field_vars; 382 unsigned int n_field_var_str; 383 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 384 unsigned int n_field_var_hists; 385 386 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 387 unsigned int n_save_vars; 388 unsigned int n_save_var_str; 389 }; 390 391 struct action_data; 392 393 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 394 struct tracing_map_elt *elt, 395 struct trace_buffer *buffer, void *rec, 396 struct ring_buffer_event *rbe, void *key, 397 struct action_data *data, u64 *var_ref_vals); 398 399 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 400 401 enum handler_id { 402 HANDLER_ONMATCH = 1, 403 HANDLER_ONMAX, 404 HANDLER_ONCHANGE, 405 }; 406 407 enum action_id { 408 ACTION_SAVE = 1, 409 ACTION_TRACE, 410 ACTION_SNAPSHOT, 411 }; 412 413 struct action_data { 414 enum handler_id handler; 415 enum action_id action; 416 char *action_name; 417 action_fn_t fn; 418 419 unsigned int n_params; 420 char *params[SYNTH_FIELDS_MAX]; 421 422 /* 423 * When a histogram trigger is hit, the values of any 424 * references to variables, including variables being passed 425 * as parameters to synthetic events, are collected into a 426 * var_ref_vals array. This var_ref_idx array is an array of 427 * indices into the var_ref_vals array, one for each synthetic 428 * event param, and is passed to the synthetic event 429 * invocation. 430 */ 431 unsigned int var_ref_idx[TRACING_MAP_VARS_MAX]; 432 struct synth_event *synth_event; 433 bool use_trace_keyword; 434 char *synth_event_name; 435 436 union { 437 struct { 438 char *event; 439 char *event_system; 440 } match_data; 441 442 struct { 443 /* 444 * var_str contains the $-unstripped variable 445 * name referenced by var_ref, and used when 446 * printing the action. Because var_ref 447 * creation is deferred to create_actions(), 448 * we need a per-action way to save it until 449 * then, thus var_str. 450 */ 451 char *var_str; 452 453 /* 454 * var_ref refers to the variable being 455 * tracked e.g onmax($var). 456 */ 457 struct hist_field *var_ref; 458 459 /* 460 * track_var contains the 'invisible' tracking 461 * variable created to keep the current 462 * e.g. max value. 463 */ 464 struct hist_field *track_var; 465 466 check_track_val_fn_t check_val; 467 action_fn_t save_data; 468 } track_data; 469 }; 470 }; 471 472 struct track_data { 473 u64 track_val; 474 bool updated; 475 476 unsigned int key_len; 477 void *key; 478 struct tracing_map_elt elt; 479 480 struct action_data *action_data; 481 struct hist_trigger_data *hist_data; 482 }; 483 484 struct hist_elt_data { 485 char *comm; 486 u64 *var_ref_vals; 487 char *field_var_str[SYNTH_FIELDS_MAX]; 488 }; 489 490 struct snapshot_context { 491 struct tracing_map_elt *elt; 492 void *key; 493 }; 494 495 static void track_data_free(struct track_data *track_data) 496 { 497 struct hist_elt_data *elt_data; 498 499 if (!track_data) 500 return; 501 502 kfree(track_data->key); 503 504 elt_data = track_data->elt.private_data; 505 if (elt_data) { 506 kfree(elt_data->comm); 507 kfree(elt_data); 508 } 509 510 kfree(track_data); 511 } 512 513 static struct track_data *track_data_alloc(unsigned int key_len, 514 struct action_data *action_data, 515 struct hist_trigger_data *hist_data) 516 { 517 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 518 struct hist_elt_data *elt_data; 519 520 if (!data) 521 return ERR_PTR(-ENOMEM); 522 523 data->key = kzalloc(key_len, GFP_KERNEL); 524 if (!data->key) { 525 track_data_free(data); 526 return ERR_PTR(-ENOMEM); 527 } 528 529 data->key_len = key_len; 530 data->action_data = action_data; 531 data->hist_data = hist_data; 532 533 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 534 if (!elt_data) { 535 track_data_free(data); 536 return ERR_PTR(-ENOMEM); 537 } 538 539 data->elt.private_data = elt_data; 540 541 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 542 if (!elt_data->comm) { 543 track_data_free(data); 544 return ERR_PTR(-ENOMEM); 545 } 546 547 return data; 548 } 549 550 static char last_cmd[MAX_FILTER_STR_VAL]; 551 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 552 553 static int errpos(char *str) 554 { 555 return err_pos(last_cmd, str); 556 } 557 558 static void last_cmd_set(struct trace_event_file *file, char *str) 559 { 560 const char *system = NULL, *name = NULL; 561 struct trace_event_call *call; 562 563 if (!str) 564 return; 565 566 strcpy(last_cmd, "hist:"); 567 strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:")); 568 569 if (file) { 570 call = file->event_call; 571 system = call->class->system; 572 if (system) { 573 name = trace_event_name(call); 574 if (!name) 575 system = NULL; 576 } 577 } 578 579 if (system) 580 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name); 581 } 582 583 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos) 584 { 585 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 586 err_type, err_pos); 587 } 588 589 static void hist_err_clear(void) 590 { 591 last_cmd[0] = '\0'; 592 last_cmd_loc[0] = '\0'; 593 } 594 595 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 596 unsigned int *var_ref_idx); 597 598 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 599 unsigned int *var_ref_idx) 600 { 601 struct tracepoint *tp = event->tp; 602 603 if (unlikely(atomic_read(&tp->key.enabled) > 0)) { 604 struct tracepoint_func *probe_func_ptr; 605 synth_probe_func_t probe_func; 606 void *__data; 607 608 if (!(cpu_online(raw_smp_processor_id()))) 609 return; 610 611 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 612 if (probe_func_ptr) { 613 do { 614 probe_func = probe_func_ptr->func; 615 __data = probe_func_ptr->data; 616 probe_func(__data, var_ref_vals, var_ref_idx); 617 } while ((++probe_func_ptr)->func); 618 } 619 } 620 } 621 622 static void action_trace(struct hist_trigger_data *hist_data, 623 struct tracing_map_elt *elt, 624 struct trace_buffer *buffer, void *rec, 625 struct ring_buffer_event *rbe, void *key, 626 struct action_data *data, u64 *var_ref_vals) 627 { 628 struct synth_event *event = data->synth_event; 629 630 trace_synth(event, var_ref_vals, data->var_ref_idx); 631 } 632 633 struct hist_var_data { 634 struct list_head list; 635 struct hist_trigger_data *hist_data; 636 }; 637 638 static u64 hist_field_timestamp(struct hist_field *hist_field, 639 struct tracing_map_elt *elt, 640 struct trace_buffer *buffer, 641 struct ring_buffer_event *rbe, 642 void *event) 643 { 644 struct hist_trigger_data *hist_data = hist_field->hist_data; 645 struct trace_array *tr = hist_data->event_file->tr; 646 647 u64 ts = ring_buffer_event_time_stamp(buffer, rbe); 648 649 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 650 ts = ns2usecs(ts); 651 652 return ts; 653 } 654 655 static u64 hist_field_cpu(struct hist_field *hist_field, 656 struct tracing_map_elt *elt, 657 struct trace_buffer *buffer, 658 struct ring_buffer_event *rbe, 659 void *event) 660 { 661 int cpu = smp_processor_id(); 662 663 return cpu; 664 } 665 666 /** 667 * check_field_for_var_ref - Check if a VAR_REF field references a variable 668 * @hist_field: The VAR_REF field to check 669 * @var_data: The hist trigger that owns the variable 670 * @var_idx: The trigger variable identifier 671 * 672 * Check the given VAR_REF field to see whether or not it references 673 * the given variable associated with the given trigger. 674 * 675 * Return: The VAR_REF field if it does reference the variable, NULL if not 676 */ 677 static struct hist_field * 678 check_field_for_var_ref(struct hist_field *hist_field, 679 struct hist_trigger_data *var_data, 680 unsigned int var_idx) 681 { 682 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 683 684 if (hist_field && hist_field->var.idx == var_idx && 685 hist_field->var.hist_data == var_data) 686 return hist_field; 687 688 return NULL; 689 } 690 691 /** 692 * find_var_ref - Check if a trigger has a reference to a trigger variable 693 * @hist_data: The hist trigger that might have a reference to the variable 694 * @var_data: The hist trigger that owns the variable 695 * @var_idx: The trigger variable identifier 696 * 697 * Check the list of var_refs[] on the first hist trigger to see 698 * whether any of them are references to the variable on the second 699 * trigger. 700 * 701 * Return: The VAR_REF field referencing the variable if so, NULL if not 702 */ 703 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 704 struct hist_trigger_data *var_data, 705 unsigned int var_idx) 706 { 707 struct hist_field *hist_field; 708 unsigned int i; 709 710 for (i = 0; i < hist_data->n_var_refs; i++) { 711 hist_field = hist_data->var_refs[i]; 712 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 713 return hist_field; 714 } 715 716 return NULL; 717 } 718 719 /** 720 * find_any_var_ref - Check if there is a reference to a given trigger variable 721 * @hist_data: The hist trigger 722 * @var_idx: The trigger variable identifier 723 * 724 * Check to see whether the given variable is currently referenced by 725 * any other trigger. 726 * 727 * The trigger the variable is defined on is explicitly excluded - the 728 * assumption being that a self-reference doesn't prevent a trigger 729 * from being removed. 730 * 731 * Return: The VAR_REF field referencing the variable if so, NULL if not 732 */ 733 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 734 unsigned int var_idx) 735 { 736 struct trace_array *tr = hist_data->event_file->tr; 737 struct hist_field *found = NULL; 738 struct hist_var_data *var_data; 739 740 list_for_each_entry(var_data, &tr->hist_vars, list) { 741 if (var_data->hist_data == hist_data) 742 continue; 743 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 744 if (found) 745 break; 746 } 747 748 return found; 749 } 750 751 /** 752 * check_var_refs - Check if there is a reference to any of trigger's variables 753 * @hist_data: The hist trigger 754 * 755 * A trigger can define one or more variables. If any one of them is 756 * currently referenced by any other trigger, this function will 757 * determine that. 758 759 * Typically used to determine whether or not a trigger can be removed 760 * - if there are any references to a trigger's variables, it cannot. 761 * 762 * Return: True if there is a reference to any of trigger's variables 763 */ 764 static bool check_var_refs(struct hist_trigger_data *hist_data) 765 { 766 struct hist_field *field; 767 bool found = false; 768 int i; 769 770 for_each_hist_field(i, hist_data) { 771 field = hist_data->fields[i]; 772 if (field && field->flags & HIST_FIELD_FL_VAR) { 773 if (find_any_var_ref(hist_data, field->var.idx)) { 774 found = true; 775 break; 776 } 777 } 778 } 779 780 return found; 781 } 782 783 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 784 { 785 struct trace_array *tr = hist_data->event_file->tr; 786 struct hist_var_data *var_data, *found = NULL; 787 788 list_for_each_entry(var_data, &tr->hist_vars, list) { 789 if (var_data->hist_data == hist_data) { 790 found = var_data; 791 break; 792 } 793 } 794 795 return found; 796 } 797 798 static bool field_has_hist_vars(struct hist_field *hist_field, 799 unsigned int level) 800 { 801 int i; 802 803 if (level > 3) 804 return false; 805 806 if (!hist_field) 807 return false; 808 809 if (hist_field->flags & HIST_FIELD_FL_VAR || 810 hist_field->flags & HIST_FIELD_FL_VAR_REF) 811 return true; 812 813 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 814 struct hist_field *operand; 815 816 operand = hist_field->operands[i]; 817 if (field_has_hist_vars(operand, level + 1)) 818 return true; 819 } 820 821 return false; 822 } 823 824 static bool has_hist_vars(struct hist_trigger_data *hist_data) 825 { 826 struct hist_field *hist_field; 827 int i; 828 829 for_each_hist_field(i, hist_data) { 830 hist_field = hist_data->fields[i]; 831 if (field_has_hist_vars(hist_field, 0)) 832 return true; 833 } 834 835 return false; 836 } 837 838 static int save_hist_vars(struct hist_trigger_data *hist_data) 839 { 840 struct trace_array *tr = hist_data->event_file->tr; 841 struct hist_var_data *var_data; 842 843 var_data = find_hist_vars(hist_data); 844 if (var_data) 845 return 0; 846 847 if (tracing_check_open_get_tr(tr)) 848 return -ENODEV; 849 850 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 851 if (!var_data) { 852 trace_array_put(tr); 853 return -ENOMEM; 854 } 855 856 var_data->hist_data = hist_data; 857 list_add(&var_data->list, &tr->hist_vars); 858 859 return 0; 860 } 861 862 static void remove_hist_vars(struct hist_trigger_data *hist_data) 863 { 864 struct trace_array *tr = hist_data->event_file->tr; 865 struct hist_var_data *var_data; 866 867 var_data = find_hist_vars(hist_data); 868 if (!var_data) 869 return; 870 871 if (WARN_ON(check_var_refs(hist_data))) 872 return; 873 874 list_del(&var_data->list); 875 876 kfree(var_data); 877 878 trace_array_put(tr); 879 } 880 881 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 882 const char *var_name) 883 { 884 struct hist_field *hist_field, *found = NULL; 885 int i; 886 887 for_each_hist_field(i, hist_data) { 888 hist_field = hist_data->fields[i]; 889 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 890 strcmp(hist_field->var.name, var_name) == 0) { 891 found = hist_field; 892 break; 893 } 894 } 895 896 return found; 897 } 898 899 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 900 struct trace_event_file *file, 901 const char *var_name) 902 { 903 struct hist_trigger_data *test_data; 904 struct event_trigger_data *test; 905 struct hist_field *hist_field; 906 907 lockdep_assert_held(&event_mutex); 908 909 hist_field = find_var_field(hist_data, var_name); 910 if (hist_field) 911 return hist_field; 912 913 list_for_each_entry(test, &file->triggers, list) { 914 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 915 test_data = test->private_data; 916 hist_field = find_var_field(test_data, var_name); 917 if (hist_field) 918 return hist_field; 919 } 920 } 921 922 return NULL; 923 } 924 925 static struct trace_event_file *find_var_file(struct trace_array *tr, 926 char *system, 927 char *event_name, 928 char *var_name) 929 { 930 struct hist_trigger_data *var_hist_data; 931 struct hist_var_data *var_data; 932 struct trace_event_file *file, *found = NULL; 933 934 if (system) 935 return find_event_file(tr, system, event_name); 936 937 list_for_each_entry(var_data, &tr->hist_vars, list) { 938 var_hist_data = var_data->hist_data; 939 file = var_hist_data->event_file; 940 if (file == found) 941 continue; 942 943 if (find_var_field(var_hist_data, var_name)) { 944 if (found) { 945 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 946 return NULL; 947 } 948 949 found = file; 950 } 951 } 952 953 return found; 954 } 955 956 static struct hist_field *find_file_var(struct trace_event_file *file, 957 const char *var_name) 958 { 959 struct hist_trigger_data *test_data; 960 struct event_trigger_data *test; 961 struct hist_field *hist_field; 962 963 lockdep_assert_held(&event_mutex); 964 965 list_for_each_entry(test, &file->triggers, list) { 966 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 967 test_data = test->private_data; 968 hist_field = find_var_field(test_data, var_name); 969 if (hist_field) 970 return hist_field; 971 } 972 } 973 974 return NULL; 975 } 976 977 static struct hist_field * 978 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 979 { 980 struct trace_array *tr = hist_data->event_file->tr; 981 struct hist_field *hist_field, *found = NULL; 982 struct trace_event_file *file; 983 unsigned int i; 984 985 for (i = 0; i < hist_data->n_actions; i++) { 986 struct action_data *data = hist_data->actions[i]; 987 988 if (data->handler == HANDLER_ONMATCH) { 989 char *system = data->match_data.event_system; 990 char *event_name = data->match_data.event; 991 992 file = find_var_file(tr, system, event_name, var_name); 993 if (!file) 994 continue; 995 hist_field = find_file_var(file, var_name); 996 if (hist_field) { 997 if (found) { 998 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 999 errpos(var_name)); 1000 return ERR_PTR(-EINVAL); 1001 } 1002 1003 found = hist_field; 1004 } 1005 } 1006 } 1007 return found; 1008 } 1009 1010 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1011 char *system, 1012 char *event_name, 1013 char *var_name) 1014 { 1015 struct trace_array *tr = hist_data->event_file->tr; 1016 struct hist_field *hist_field = NULL; 1017 struct trace_event_file *file; 1018 1019 if (!system || !event_name) { 1020 hist_field = find_match_var(hist_data, var_name); 1021 if (IS_ERR(hist_field)) 1022 return NULL; 1023 if (hist_field) 1024 return hist_field; 1025 } 1026 1027 file = find_var_file(tr, system, event_name, var_name); 1028 if (!file) 1029 return NULL; 1030 1031 hist_field = find_file_var(file, var_name); 1032 1033 return hist_field; 1034 } 1035 1036 static u64 hist_field_var_ref(struct hist_field *hist_field, 1037 struct tracing_map_elt *elt, 1038 struct trace_buffer *buffer, 1039 struct ring_buffer_event *rbe, 1040 void *event) 1041 { 1042 struct hist_elt_data *elt_data; 1043 u64 var_val = 0; 1044 1045 if (WARN_ON_ONCE(!elt)) 1046 return var_val; 1047 1048 elt_data = elt->private_data; 1049 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1050 1051 return var_val; 1052 } 1053 1054 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1055 u64 *var_ref_vals, bool self) 1056 { 1057 struct hist_trigger_data *var_data; 1058 struct tracing_map_elt *var_elt; 1059 struct hist_field *hist_field; 1060 unsigned int i, var_idx; 1061 bool resolved = true; 1062 u64 var_val = 0; 1063 1064 for (i = 0; i < hist_data->n_var_refs; i++) { 1065 hist_field = hist_data->var_refs[i]; 1066 var_idx = hist_field->var.idx; 1067 var_data = hist_field->var.hist_data; 1068 1069 if (var_data == NULL) { 1070 resolved = false; 1071 break; 1072 } 1073 1074 if ((self && var_data != hist_data) || 1075 (!self && var_data == hist_data)) 1076 continue; 1077 1078 var_elt = tracing_map_lookup(var_data->map, key); 1079 if (!var_elt) { 1080 resolved = false; 1081 break; 1082 } 1083 1084 if (!tracing_map_var_set(var_elt, var_idx)) { 1085 resolved = false; 1086 break; 1087 } 1088 1089 if (self || !hist_field->read_once) 1090 var_val = tracing_map_read_var(var_elt, var_idx); 1091 else 1092 var_val = tracing_map_read_var_once(var_elt, var_idx); 1093 1094 var_ref_vals[i] = var_val; 1095 } 1096 1097 return resolved; 1098 } 1099 1100 static const char *hist_field_name(struct hist_field *field, 1101 unsigned int level) 1102 { 1103 const char *field_name = ""; 1104 1105 if (level > 1) 1106 return field_name; 1107 1108 if (field->field) 1109 field_name = field->field->name; 1110 else if (field->flags & HIST_FIELD_FL_LOG2 || 1111 field->flags & HIST_FIELD_FL_ALIAS) 1112 field_name = hist_field_name(field->operands[0], ++level); 1113 else if (field->flags & HIST_FIELD_FL_CPU) 1114 field_name = "cpu"; 1115 else if (field->flags & HIST_FIELD_FL_EXPR || 1116 field->flags & HIST_FIELD_FL_VAR_REF) { 1117 if (field->system) { 1118 static char full_name[MAX_FILTER_STR_VAL]; 1119 1120 strcat(full_name, field->system); 1121 strcat(full_name, "."); 1122 strcat(full_name, field->event_name); 1123 strcat(full_name, "."); 1124 strcat(full_name, field->name); 1125 field_name = full_name; 1126 } else 1127 field_name = field->name; 1128 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1129 field_name = "common_timestamp"; 1130 1131 if (field_name == NULL) 1132 field_name = ""; 1133 1134 return field_name; 1135 } 1136 1137 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) 1138 { 1139 hist_field_fn_t fn = NULL; 1140 1141 switch (field_size) { 1142 case 8: 1143 if (field_is_signed) 1144 fn = hist_field_s64; 1145 else 1146 fn = hist_field_u64; 1147 break; 1148 case 4: 1149 if (field_is_signed) 1150 fn = hist_field_s32; 1151 else 1152 fn = hist_field_u32; 1153 break; 1154 case 2: 1155 if (field_is_signed) 1156 fn = hist_field_s16; 1157 else 1158 fn = hist_field_u16; 1159 break; 1160 case 1: 1161 if (field_is_signed) 1162 fn = hist_field_s8; 1163 else 1164 fn = hist_field_u8; 1165 break; 1166 } 1167 1168 return fn; 1169 } 1170 1171 static int parse_map_size(char *str) 1172 { 1173 unsigned long size, map_bits; 1174 int ret; 1175 1176 ret = kstrtoul(str, 0, &size); 1177 if (ret) 1178 goto out; 1179 1180 map_bits = ilog2(roundup_pow_of_two(size)); 1181 if (map_bits < TRACING_MAP_BITS_MIN || 1182 map_bits > TRACING_MAP_BITS_MAX) 1183 ret = -EINVAL; 1184 else 1185 ret = map_bits; 1186 out: 1187 return ret; 1188 } 1189 1190 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1191 { 1192 unsigned int i; 1193 1194 if (!attrs) 1195 return; 1196 1197 for (i = 0; i < attrs->n_assignments; i++) 1198 kfree(attrs->assignment_str[i]); 1199 1200 for (i = 0; i < attrs->n_actions; i++) 1201 kfree(attrs->action_str[i]); 1202 1203 kfree(attrs->name); 1204 kfree(attrs->sort_key_str); 1205 kfree(attrs->keys_str); 1206 kfree(attrs->vals_str); 1207 kfree(attrs->clock); 1208 kfree(attrs); 1209 } 1210 1211 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1212 { 1213 int ret = -EINVAL; 1214 1215 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1216 return ret; 1217 1218 if ((str_has_prefix(str, "onmatch(")) || 1219 (str_has_prefix(str, "onmax(")) || 1220 (str_has_prefix(str, "onchange("))) { 1221 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1222 if (!attrs->action_str[attrs->n_actions]) { 1223 ret = -ENOMEM; 1224 return ret; 1225 } 1226 attrs->n_actions++; 1227 ret = 0; 1228 } 1229 return ret; 1230 } 1231 1232 static int parse_assignment(struct trace_array *tr, 1233 char *str, struct hist_trigger_attrs *attrs) 1234 { 1235 int len, ret = 0; 1236 1237 if ((len = str_has_prefix(str, "key=")) || 1238 (len = str_has_prefix(str, "keys="))) { 1239 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1240 if (!attrs->keys_str) { 1241 ret = -ENOMEM; 1242 goto out; 1243 } 1244 } else if ((len = str_has_prefix(str, "val=")) || 1245 (len = str_has_prefix(str, "vals=")) || 1246 (len = str_has_prefix(str, "values="))) { 1247 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1248 if (!attrs->vals_str) { 1249 ret = -ENOMEM; 1250 goto out; 1251 } 1252 } else if ((len = str_has_prefix(str, "sort="))) { 1253 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1254 if (!attrs->sort_key_str) { 1255 ret = -ENOMEM; 1256 goto out; 1257 } 1258 } else if (str_has_prefix(str, "name=")) { 1259 attrs->name = kstrdup(str, GFP_KERNEL); 1260 if (!attrs->name) { 1261 ret = -ENOMEM; 1262 goto out; 1263 } 1264 } else if ((len = str_has_prefix(str, "clock="))) { 1265 str += len; 1266 1267 str = strstrip(str); 1268 attrs->clock = kstrdup(str, GFP_KERNEL); 1269 if (!attrs->clock) { 1270 ret = -ENOMEM; 1271 goto out; 1272 } 1273 } else if ((len = str_has_prefix(str, "size="))) { 1274 int map_bits = parse_map_size(str + len); 1275 1276 if (map_bits < 0) { 1277 ret = map_bits; 1278 goto out; 1279 } 1280 attrs->map_bits = map_bits; 1281 } else { 1282 char *assignment; 1283 1284 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1285 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1286 ret = -EINVAL; 1287 goto out; 1288 } 1289 1290 assignment = kstrdup(str, GFP_KERNEL); 1291 if (!assignment) { 1292 ret = -ENOMEM; 1293 goto out; 1294 } 1295 1296 attrs->assignment_str[attrs->n_assignments++] = assignment; 1297 } 1298 out: 1299 return ret; 1300 } 1301 1302 static struct hist_trigger_attrs * 1303 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1304 { 1305 struct hist_trigger_attrs *attrs; 1306 int ret = 0; 1307 1308 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1309 if (!attrs) 1310 return ERR_PTR(-ENOMEM); 1311 1312 while (trigger_str) { 1313 char *str = strsep(&trigger_str, ":"); 1314 char *rhs; 1315 1316 rhs = strchr(str, '='); 1317 if (rhs) { 1318 if (!strlen(++rhs)) { 1319 ret = -EINVAL; 1320 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1321 goto free; 1322 } 1323 ret = parse_assignment(tr, str, attrs); 1324 if (ret) 1325 goto free; 1326 } else if (strcmp(str, "pause") == 0) 1327 attrs->pause = true; 1328 else if ((strcmp(str, "cont") == 0) || 1329 (strcmp(str, "continue") == 0)) 1330 attrs->cont = true; 1331 else if (strcmp(str, "clear") == 0) 1332 attrs->clear = true; 1333 else { 1334 ret = parse_action(str, attrs); 1335 if (ret) 1336 goto free; 1337 } 1338 } 1339 1340 if (!attrs->keys_str) { 1341 ret = -EINVAL; 1342 goto free; 1343 } 1344 1345 if (!attrs->clock) { 1346 attrs->clock = kstrdup("global", GFP_KERNEL); 1347 if (!attrs->clock) { 1348 ret = -ENOMEM; 1349 goto free; 1350 } 1351 } 1352 1353 return attrs; 1354 free: 1355 destroy_hist_trigger_attrs(attrs); 1356 1357 return ERR_PTR(ret); 1358 } 1359 1360 static inline void save_comm(char *comm, struct task_struct *task) 1361 { 1362 if (!task->pid) { 1363 strcpy(comm, "<idle>"); 1364 return; 1365 } 1366 1367 if (WARN_ON_ONCE(task->pid < 0)) { 1368 strcpy(comm, "<XXX>"); 1369 return; 1370 } 1371 1372 strncpy(comm, task->comm, TASK_COMM_LEN); 1373 } 1374 1375 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1376 { 1377 unsigned int i; 1378 1379 for (i = 0; i < SYNTH_FIELDS_MAX; i++) 1380 kfree(elt_data->field_var_str[i]); 1381 1382 kfree(elt_data->comm); 1383 kfree(elt_data); 1384 } 1385 1386 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1387 { 1388 struct hist_elt_data *elt_data = elt->private_data; 1389 1390 hist_elt_data_free(elt_data); 1391 } 1392 1393 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1394 { 1395 struct hist_trigger_data *hist_data = elt->map->private_data; 1396 unsigned int size = TASK_COMM_LEN; 1397 struct hist_elt_data *elt_data; 1398 struct hist_field *key_field; 1399 unsigned int i, n_str; 1400 1401 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1402 if (!elt_data) 1403 return -ENOMEM; 1404 1405 for_each_hist_key_field(i, hist_data) { 1406 key_field = hist_data->fields[i]; 1407 1408 if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 1409 elt_data->comm = kzalloc(size, GFP_KERNEL); 1410 if (!elt_data->comm) { 1411 kfree(elt_data); 1412 return -ENOMEM; 1413 } 1414 break; 1415 } 1416 } 1417 1418 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + 1419 hist_data->n_var_str; 1420 if (n_str > SYNTH_FIELDS_MAX) { 1421 hist_elt_data_free(elt_data); 1422 return -EINVAL; 1423 } 1424 1425 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); 1426 1427 size = STR_VAR_LEN_MAX; 1428 1429 for (i = 0; i < n_str; i++) { 1430 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1431 if (!elt_data->field_var_str[i]) { 1432 hist_elt_data_free(elt_data); 1433 return -ENOMEM; 1434 } 1435 } 1436 1437 elt->private_data = elt_data; 1438 1439 return 0; 1440 } 1441 1442 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1443 { 1444 struct hist_elt_data *elt_data = elt->private_data; 1445 1446 if (elt_data->comm) 1447 save_comm(elt_data->comm, current); 1448 } 1449 1450 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1451 .elt_alloc = hist_trigger_elt_data_alloc, 1452 .elt_free = hist_trigger_elt_data_free, 1453 .elt_init = hist_trigger_elt_data_init, 1454 }; 1455 1456 static const char *get_hist_field_flags(struct hist_field *hist_field) 1457 { 1458 const char *flags_str = NULL; 1459 1460 if (hist_field->flags & HIST_FIELD_FL_HEX) 1461 flags_str = "hex"; 1462 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1463 flags_str = "sym"; 1464 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1465 flags_str = "sym-offset"; 1466 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1467 flags_str = "execname"; 1468 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1469 flags_str = "syscall"; 1470 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1471 flags_str = "log2"; 1472 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1473 flags_str = "usecs"; 1474 1475 return flags_str; 1476 } 1477 1478 static void expr_field_str(struct hist_field *field, char *expr) 1479 { 1480 if (field->flags & HIST_FIELD_FL_VAR_REF) 1481 strcat(expr, "$"); 1482 1483 strcat(expr, hist_field_name(field, 0)); 1484 1485 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1486 const char *flags_str = get_hist_field_flags(field); 1487 1488 if (flags_str) { 1489 strcat(expr, "."); 1490 strcat(expr, flags_str); 1491 } 1492 } 1493 } 1494 1495 static char *expr_str(struct hist_field *field, unsigned int level) 1496 { 1497 char *expr; 1498 1499 if (level > 1) 1500 return NULL; 1501 1502 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1503 if (!expr) 1504 return NULL; 1505 1506 if (!field->operands[0]) { 1507 expr_field_str(field, expr); 1508 return expr; 1509 } 1510 1511 if (field->operator == FIELD_OP_UNARY_MINUS) { 1512 char *subexpr; 1513 1514 strcat(expr, "-("); 1515 subexpr = expr_str(field->operands[0], ++level); 1516 if (!subexpr) { 1517 kfree(expr); 1518 return NULL; 1519 } 1520 strcat(expr, subexpr); 1521 strcat(expr, ")"); 1522 1523 kfree(subexpr); 1524 1525 return expr; 1526 } 1527 1528 expr_field_str(field->operands[0], expr); 1529 1530 switch (field->operator) { 1531 case FIELD_OP_MINUS: 1532 strcat(expr, "-"); 1533 break; 1534 case FIELD_OP_PLUS: 1535 strcat(expr, "+"); 1536 break; 1537 default: 1538 kfree(expr); 1539 return NULL; 1540 } 1541 1542 expr_field_str(field->operands[1], expr); 1543 1544 return expr; 1545 } 1546 1547 static int contains_operator(char *str) 1548 { 1549 enum field_op_id field_op = FIELD_OP_NONE; 1550 char *op; 1551 1552 op = strpbrk(str, "+-"); 1553 if (!op) 1554 return FIELD_OP_NONE; 1555 1556 switch (*op) { 1557 case '-': 1558 if (*str == '-') 1559 field_op = FIELD_OP_UNARY_MINUS; 1560 else 1561 field_op = FIELD_OP_MINUS; 1562 break; 1563 case '+': 1564 field_op = FIELD_OP_PLUS; 1565 break; 1566 default: 1567 break; 1568 } 1569 1570 return field_op; 1571 } 1572 1573 static void get_hist_field(struct hist_field *hist_field) 1574 { 1575 hist_field->ref++; 1576 } 1577 1578 static void __destroy_hist_field(struct hist_field *hist_field) 1579 { 1580 if (--hist_field->ref > 1) 1581 return; 1582 1583 kfree(hist_field->var.name); 1584 kfree(hist_field->name); 1585 kfree(hist_field->type); 1586 1587 kfree(hist_field->system); 1588 kfree(hist_field->event_name); 1589 1590 kfree(hist_field); 1591 } 1592 1593 static void destroy_hist_field(struct hist_field *hist_field, 1594 unsigned int level) 1595 { 1596 unsigned int i; 1597 1598 if (level > 3) 1599 return; 1600 1601 if (!hist_field) 1602 return; 1603 1604 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1605 return; /* var refs will be destroyed separately */ 1606 1607 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1608 destroy_hist_field(hist_field->operands[i], level + 1); 1609 1610 __destroy_hist_field(hist_field); 1611 } 1612 1613 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1614 struct ftrace_event_field *field, 1615 unsigned long flags, 1616 char *var_name) 1617 { 1618 struct hist_field *hist_field; 1619 1620 if (field && is_function_field(field)) 1621 return NULL; 1622 1623 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 1624 if (!hist_field) 1625 return NULL; 1626 1627 hist_field->ref = 1; 1628 1629 hist_field->hist_data = hist_data; 1630 1631 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1632 goto out; /* caller will populate */ 1633 1634 if (flags & HIST_FIELD_FL_VAR_REF) { 1635 hist_field->fn = hist_field_var_ref; 1636 goto out; 1637 } 1638 1639 if (flags & HIST_FIELD_FL_HITCOUNT) { 1640 hist_field->fn = hist_field_counter; 1641 hist_field->size = sizeof(u64); 1642 hist_field->type = kstrdup("u64", GFP_KERNEL); 1643 if (!hist_field->type) 1644 goto free; 1645 goto out; 1646 } 1647 1648 if (flags & HIST_FIELD_FL_STACKTRACE) { 1649 hist_field->fn = hist_field_none; 1650 goto out; 1651 } 1652 1653 if (flags & HIST_FIELD_FL_LOG2) { 1654 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2; 1655 hist_field->fn = hist_field_log2; 1656 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 1657 hist_field->size = hist_field->operands[0]->size; 1658 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL); 1659 if (!hist_field->type) 1660 goto free; 1661 goto out; 1662 } 1663 1664 if (flags & HIST_FIELD_FL_TIMESTAMP) { 1665 hist_field->fn = hist_field_timestamp; 1666 hist_field->size = sizeof(u64); 1667 hist_field->type = kstrdup("u64", GFP_KERNEL); 1668 if (!hist_field->type) 1669 goto free; 1670 goto out; 1671 } 1672 1673 if (flags & HIST_FIELD_FL_CPU) { 1674 hist_field->fn = hist_field_cpu; 1675 hist_field->size = sizeof(int); 1676 hist_field->type = kstrdup("unsigned int", GFP_KERNEL); 1677 if (!hist_field->type) 1678 goto free; 1679 goto out; 1680 } 1681 1682 if (WARN_ON_ONCE(!field)) 1683 goto out; 1684 1685 if (is_string_field(field)) { 1686 flags |= HIST_FIELD_FL_STRING; 1687 1688 hist_field->size = MAX_FILTER_STR_VAL; 1689 hist_field->type = kstrdup(field->type, GFP_KERNEL); 1690 if (!hist_field->type) 1691 goto free; 1692 1693 if (field->filter_type == FILTER_STATIC_STRING) 1694 hist_field->fn = hist_field_string; 1695 else if (field->filter_type == FILTER_DYN_STRING) 1696 hist_field->fn = hist_field_dynstring; 1697 else 1698 hist_field->fn = hist_field_pstring; 1699 } else { 1700 hist_field->size = field->size; 1701 hist_field->is_signed = field->is_signed; 1702 hist_field->type = kstrdup(field->type, GFP_KERNEL); 1703 if (!hist_field->type) 1704 goto free; 1705 1706 hist_field->fn = select_value_fn(field->size, 1707 field->is_signed); 1708 if (!hist_field->fn) { 1709 destroy_hist_field(hist_field, 0); 1710 return NULL; 1711 } 1712 } 1713 out: 1714 hist_field->field = field; 1715 hist_field->flags = flags; 1716 1717 if (var_name) { 1718 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 1719 if (!hist_field->var.name) 1720 goto free; 1721 } 1722 1723 return hist_field; 1724 free: 1725 destroy_hist_field(hist_field, 0); 1726 return NULL; 1727 } 1728 1729 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 1730 { 1731 unsigned int i; 1732 1733 for (i = 0; i < HIST_FIELDS_MAX; i++) { 1734 if (hist_data->fields[i]) { 1735 destroy_hist_field(hist_data->fields[i], 0); 1736 hist_data->fields[i] = NULL; 1737 } 1738 } 1739 1740 for (i = 0; i < hist_data->n_var_refs; i++) { 1741 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 1742 __destroy_hist_field(hist_data->var_refs[i]); 1743 hist_data->var_refs[i] = NULL; 1744 } 1745 } 1746 1747 static int init_var_ref(struct hist_field *ref_field, 1748 struct hist_field *var_field, 1749 char *system, char *event_name) 1750 { 1751 int err = 0; 1752 1753 ref_field->var.idx = var_field->var.idx; 1754 ref_field->var.hist_data = var_field->hist_data; 1755 ref_field->size = var_field->size; 1756 ref_field->is_signed = var_field->is_signed; 1757 ref_field->flags |= var_field->flags & 1758 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 1759 1760 if (system) { 1761 ref_field->system = kstrdup(system, GFP_KERNEL); 1762 if (!ref_field->system) 1763 return -ENOMEM; 1764 } 1765 1766 if (event_name) { 1767 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 1768 if (!ref_field->event_name) { 1769 err = -ENOMEM; 1770 goto free; 1771 } 1772 } 1773 1774 if (var_field->var.name) { 1775 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 1776 if (!ref_field->name) { 1777 err = -ENOMEM; 1778 goto free; 1779 } 1780 } else if (var_field->name) { 1781 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 1782 if (!ref_field->name) { 1783 err = -ENOMEM; 1784 goto free; 1785 } 1786 } 1787 1788 ref_field->type = kstrdup(var_field->type, GFP_KERNEL); 1789 if (!ref_field->type) { 1790 err = -ENOMEM; 1791 goto free; 1792 } 1793 out: 1794 return err; 1795 free: 1796 kfree(ref_field->system); 1797 kfree(ref_field->event_name); 1798 kfree(ref_field->name); 1799 1800 goto out; 1801 } 1802 1803 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 1804 struct hist_field *var_field) 1805 { 1806 struct hist_field *ref_field; 1807 int i; 1808 1809 for (i = 0; i < hist_data->n_var_refs; i++) { 1810 ref_field = hist_data->var_refs[i]; 1811 if (ref_field->var.idx == var_field->var.idx && 1812 ref_field->var.hist_data == var_field->hist_data) 1813 return i; 1814 } 1815 1816 return -ENOENT; 1817 } 1818 1819 /** 1820 * create_var_ref - Create a variable reference and attach it to trigger 1821 * @hist_data: The trigger that will be referencing the variable 1822 * @var_field: The VAR field to create a reference to 1823 * @system: The optional system string 1824 * @event_name: The optional event_name string 1825 * 1826 * Given a variable hist_field, create a VAR_REF hist_field that 1827 * represents a reference to it. 1828 * 1829 * This function also adds the reference to the trigger that 1830 * now references the variable. 1831 * 1832 * Return: The VAR_REF field if successful, NULL if not 1833 */ 1834 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 1835 struct hist_field *var_field, 1836 char *system, char *event_name) 1837 { 1838 unsigned long flags = HIST_FIELD_FL_VAR_REF; 1839 struct hist_field *ref_field; 1840 int i; 1841 1842 /* Check if the variable already exists */ 1843 for (i = 0; i < hist_data->n_var_refs; i++) { 1844 ref_field = hist_data->var_refs[i]; 1845 if (ref_field->var.idx == var_field->var.idx && 1846 ref_field->var.hist_data == var_field->hist_data) { 1847 get_hist_field(ref_field); 1848 return ref_field; 1849 } 1850 } 1851 1852 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 1853 if (ref_field) { 1854 if (init_var_ref(ref_field, var_field, system, event_name)) { 1855 destroy_hist_field(ref_field, 0); 1856 return NULL; 1857 } 1858 1859 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 1860 ref_field->var_ref_idx = hist_data->n_var_refs++; 1861 } 1862 1863 return ref_field; 1864 } 1865 1866 static bool is_var_ref(char *var_name) 1867 { 1868 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 1869 return false; 1870 1871 return true; 1872 } 1873 1874 static char *field_name_from_var(struct hist_trigger_data *hist_data, 1875 char *var_name) 1876 { 1877 char *name, *field; 1878 unsigned int i; 1879 1880 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 1881 name = hist_data->attrs->var_defs.name[i]; 1882 1883 if (strcmp(var_name, name) == 0) { 1884 field = hist_data->attrs->var_defs.expr[i]; 1885 if (contains_operator(field) || is_var_ref(field)) 1886 continue; 1887 return field; 1888 } 1889 } 1890 1891 return NULL; 1892 } 1893 1894 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 1895 char *system, char *event_name, 1896 char *var_name) 1897 { 1898 struct trace_event_call *call; 1899 1900 if (system && event_name) { 1901 call = hist_data->event_file->event_call; 1902 1903 if (strcmp(system, call->class->system) != 0) 1904 return NULL; 1905 1906 if (strcmp(event_name, trace_event_name(call)) != 0) 1907 return NULL; 1908 } 1909 1910 if (!!system != !!event_name) 1911 return NULL; 1912 1913 if (!is_var_ref(var_name)) 1914 return NULL; 1915 1916 var_name++; 1917 1918 return field_name_from_var(hist_data, var_name); 1919 } 1920 1921 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 1922 char *system, char *event_name, 1923 char *var_name) 1924 { 1925 struct hist_field *var_field = NULL, *ref_field = NULL; 1926 struct trace_array *tr = hist_data->event_file->tr; 1927 1928 if (!is_var_ref(var_name)) 1929 return NULL; 1930 1931 var_name++; 1932 1933 var_field = find_event_var(hist_data, system, event_name, var_name); 1934 if (var_field) 1935 ref_field = create_var_ref(hist_data, var_field, 1936 system, event_name); 1937 1938 if (!ref_field) 1939 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 1940 1941 return ref_field; 1942 } 1943 1944 static struct ftrace_event_field * 1945 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 1946 char *field_str, unsigned long *flags) 1947 { 1948 struct ftrace_event_field *field = NULL; 1949 char *field_name, *modifier, *str; 1950 struct trace_array *tr = file->tr; 1951 1952 modifier = str = kstrdup(field_str, GFP_KERNEL); 1953 if (!modifier) 1954 return ERR_PTR(-ENOMEM); 1955 1956 field_name = strsep(&modifier, "."); 1957 if (modifier) { 1958 if (strcmp(modifier, "hex") == 0) 1959 *flags |= HIST_FIELD_FL_HEX; 1960 else if (strcmp(modifier, "sym") == 0) 1961 *flags |= HIST_FIELD_FL_SYM; 1962 else if (strcmp(modifier, "sym-offset") == 0) 1963 *flags |= HIST_FIELD_FL_SYM_OFFSET; 1964 else if ((strcmp(modifier, "execname") == 0) && 1965 (strcmp(field_name, "common_pid") == 0)) 1966 *flags |= HIST_FIELD_FL_EXECNAME; 1967 else if (strcmp(modifier, "syscall") == 0) 1968 *flags |= HIST_FIELD_FL_SYSCALL; 1969 else if (strcmp(modifier, "log2") == 0) 1970 *flags |= HIST_FIELD_FL_LOG2; 1971 else if (strcmp(modifier, "usecs") == 0) 1972 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 1973 else { 1974 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 1975 field = ERR_PTR(-EINVAL); 1976 goto out; 1977 } 1978 } 1979 1980 if (strcmp(field_name, "common_timestamp") == 0) { 1981 *flags |= HIST_FIELD_FL_TIMESTAMP; 1982 hist_data->enable_timestamps = true; 1983 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1984 hist_data->attrs->ts_in_usecs = true; 1985 } else if (strcmp(field_name, "cpu") == 0) 1986 *flags |= HIST_FIELD_FL_CPU; 1987 else { 1988 field = trace_find_event_field(file->event_call, field_name); 1989 if (!field || !field->size) { 1990 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name)); 1991 field = ERR_PTR(-EINVAL); 1992 goto out; 1993 } 1994 } 1995 out: 1996 kfree(str); 1997 1998 return field; 1999 } 2000 2001 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2002 struct hist_field *var_ref, 2003 char *var_name) 2004 { 2005 struct hist_field *alias = NULL; 2006 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2007 2008 alias = create_hist_field(hist_data, NULL, flags, var_name); 2009 if (!alias) 2010 return NULL; 2011 2012 alias->fn = var_ref->fn; 2013 alias->operands[0] = var_ref; 2014 2015 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2016 destroy_hist_field(alias, 0); 2017 return NULL; 2018 } 2019 2020 alias->var_ref_idx = var_ref->var_ref_idx; 2021 2022 return alias; 2023 } 2024 2025 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2026 struct trace_event_file *file, char *str, 2027 unsigned long *flags, char *var_name) 2028 { 2029 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2030 struct ftrace_event_field *field = NULL; 2031 struct hist_field *hist_field = NULL; 2032 int ret = 0; 2033 2034 s = strchr(str, '.'); 2035 if (s) { 2036 s = strchr(++s, '.'); 2037 if (s) { 2038 ref_system = strsep(&str, "."); 2039 if (!str) { 2040 ret = -EINVAL; 2041 goto out; 2042 } 2043 ref_event = strsep(&str, "."); 2044 if (!str) { 2045 ret = -EINVAL; 2046 goto out; 2047 } 2048 ref_var = str; 2049 } 2050 } 2051 2052 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2053 if (!s) { 2054 hist_field = parse_var_ref(hist_data, ref_system, 2055 ref_event, ref_var); 2056 if (hist_field) { 2057 if (var_name) { 2058 hist_field = create_alias(hist_data, hist_field, var_name); 2059 if (!hist_field) { 2060 ret = -ENOMEM; 2061 goto out; 2062 } 2063 } 2064 return hist_field; 2065 } 2066 } else 2067 str = s; 2068 2069 field = parse_field(hist_data, file, str, flags); 2070 if (IS_ERR(field)) { 2071 ret = PTR_ERR(field); 2072 goto out; 2073 } 2074 2075 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2076 if (!hist_field) { 2077 ret = -ENOMEM; 2078 goto out; 2079 } 2080 2081 return hist_field; 2082 out: 2083 return ERR_PTR(ret); 2084 } 2085 2086 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2087 struct trace_event_file *file, 2088 char *str, unsigned long flags, 2089 char *var_name, unsigned int level); 2090 2091 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2092 struct trace_event_file *file, 2093 char *str, unsigned long flags, 2094 char *var_name, unsigned int level) 2095 { 2096 struct hist_field *operand1, *expr = NULL; 2097 unsigned long operand_flags; 2098 int ret = 0; 2099 char *s; 2100 2101 /* we support only -(xxx) i.e. explicit parens required */ 2102 2103 if (level > 3) { 2104 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2105 ret = -EINVAL; 2106 goto free; 2107 } 2108 2109 str++; /* skip leading '-' */ 2110 2111 s = strchr(str, '('); 2112 if (s) 2113 str++; 2114 else { 2115 ret = -EINVAL; 2116 goto free; 2117 } 2118 2119 s = strrchr(str, ')'); 2120 if (s) 2121 *s = '\0'; 2122 else { 2123 ret = -EINVAL; /* no closing ')' */ 2124 goto free; 2125 } 2126 2127 flags |= HIST_FIELD_FL_EXPR; 2128 expr = create_hist_field(hist_data, NULL, flags, var_name); 2129 if (!expr) { 2130 ret = -ENOMEM; 2131 goto free; 2132 } 2133 2134 operand_flags = 0; 2135 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2136 if (IS_ERR(operand1)) { 2137 ret = PTR_ERR(operand1); 2138 goto free; 2139 } 2140 2141 expr->flags |= operand1->flags & 2142 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2143 expr->fn = hist_field_unary_minus; 2144 expr->operands[0] = operand1; 2145 expr->operator = FIELD_OP_UNARY_MINUS; 2146 expr->name = expr_str(expr, 0); 2147 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2148 if (!expr->type) { 2149 ret = -ENOMEM; 2150 goto free; 2151 } 2152 2153 return expr; 2154 free: 2155 destroy_hist_field(expr, 0); 2156 return ERR_PTR(ret); 2157 } 2158 2159 static int check_expr_operands(struct trace_array *tr, 2160 struct hist_field *operand1, 2161 struct hist_field *operand2) 2162 { 2163 unsigned long operand1_flags = operand1->flags; 2164 unsigned long operand2_flags = operand2->flags; 2165 2166 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2167 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2168 struct hist_field *var; 2169 2170 var = find_var_field(operand1->var.hist_data, operand1->name); 2171 if (!var) 2172 return -EINVAL; 2173 operand1_flags = var->flags; 2174 } 2175 2176 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2177 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2178 struct hist_field *var; 2179 2180 var = find_var_field(operand2->var.hist_data, operand2->name); 2181 if (!var) 2182 return -EINVAL; 2183 operand2_flags = var->flags; 2184 } 2185 2186 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2187 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2188 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2189 return -EINVAL; 2190 } 2191 2192 return 0; 2193 } 2194 2195 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2196 struct trace_event_file *file, 2197 char *str, unsigned long flags, 2198 char *var_name, unsigned int level) 2199 { 2200 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2201 unsigned long operand_flags; 2202 int field_op, ret = -EINVAL; 2203 char *sep, *operand1_str; 2204 2205 if (level > 3) { 2206 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2207 return ERR_PTR(-EINVAL); 2208 } 2209 2210 field_op = contains_operator(str); 2211 2212 if (field_op == FIELD_OP_NONE) 2213 return parse_atom(hist_data, file, str, &flags, var_name); 2214 2215 if (field_op == FIELD_OP_UNARY_MINUS) 2216 return parse_unary(hist_data, file, str, flags, var_name, ++level); 2217 2218 switch (field_op) { 2219 case FIELD_OP_MINUS: 2220 sep = "-"; 2221 break; 2222 case FIELD_OP_PLUS: 2223 sep = "+"; 2224 break; 2225 default: 2226 goto free; 2227 } 2228 2229 operand1_str = strsep(&str, sep); 2230 if (!operand1_str || !str) 2231 goto free; 2232 2233 operand_flags = 0; 2234 operand1 = parse_atom(hist_data, file, operand1_str, 2235 &operand_flags, NULL); 2236 if (IS_ERR(operand1)) { 2237 ret = PTR_ERR(operand1); 2238 operand1 = NULL; 2239 goto free; 2240 } 2241 2242 /* rest of string could be another expression e.g. b+c in a+b+c */ 2243 operand_flags = 0; 2244 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level); 2245 if (IS_ERR(operand2)) { 2246 ret = PTR_ERR(operand2); 2247 operand2 = NULL; 2248 goto free; 2249 } 2250 2251 ret = check_expr_operands(file->tr, operand1, operand2); 2252 if (ret) 2253 goto free; 2254 2255 flags |= HIST_FIELD_FL_EXPR; 2256 2257 flags |= operand1->flags & 2258 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2259 2260 expr = create_hist_field(hist_data, NULL, flags, var_name); 2261 if (!expr) { 2262 ret = -ENOMEM; 2263 goto free; 2264 } 2265 2266 operand1->read_once = true; 2267 operand2->read_once = true; 2268 2269 expr->operands[0] = operand1; 2270 expr->operands[1] = operand2; 2271 expr->operator = field_op; 2272 expr->name = expr_str(expr, 0); 2273 expr->type = kstrdup(operand1->type, GFP_KERNEL); 2274 if (!expr->type) { 2275 ret = -ENOMEM; 2276 goto free; 2277 } 2278 2279 switch (field_op) { 2280 case FIELD_OP_MINUS: 2281 expr->fn = hist_field_minus; 2282 break; 2283 case FIELD_OP_PLUS: 2284 expr->fn = hist_field_plus; 2285 break; 2286 default: 2287 ret = -EINVAL; 2288 goto free; 2289 } 2290 2291 return expr; 2292 free: 2293 destroy_hist_field(operand1, 0); 2294 destroy_hist_field(operand2, 0); 2295 destroy_hist_field(expr, 0); 2296 2297 return ERR_PTR(ret); 2298 } 2299 2300 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2301 struct trace_event_file *file) 2302 { 2303 struct event_trigger_data *test; 2304 2305 lockdep_assert_held(&event_mutex); 2306 2307 list_for_each_entry(test, &file->triggers, list) { 2308 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2309 if (test->private_data == hist_data) 2310 return test->filter_str; 2311 } 2312 } 2313 2314 return NULL; 2315 } 2316 2317 static struct event_command trigger_hist_cmd; 2318 static int event_hist_trigger_func(struct event_command *cmd_ops, 2319 struct trace_event_file *file, 2320 char *glob, char *cmd, char *param); 2321 2322 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2323 struct hist_trigger_data *hist_data, 2324 unsigned int n_keys) 2325 { 2326 struct hist_field *target_hist_field, *hist_field; 2327 unsigned int n, i, j; 2328 2329 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2330 return false; 2331 2332 i = hist_data->n_vals; 2333 j = target_hist_data->n_vals; 2334 2335 for (n = 0; n < n_keys; n++) { 2336 hist_field = hist_data->fields[i + n]; 2337 target_hist_field = target_hist_data->fields[j + n]; 2338 2339 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2340 return false; 2341 if (hist_field->size != target_hist_field->size) 2342 return false; 2343 if (hist_field->is_signed != target_hist_field->is_signed) 2344 return false; 2345 } 2346 2347 return true; 2348 } 2349 2350 static struct hist_trigger_data * 2351 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2352 struct trace_event_file *file) 2353 { 2354 struct hist_trigger_data *hist_data; 2355 struct event_trigger_data *test; 2356 unsigned int n_keys; 2357 2358 lockdep_assert_held(&event_mutex); 2359 2360 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2361 2362 list_for_each_entry(test, &file->triggers, list) { 2363 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2364 hist_data = test->private_data; 2365 2366 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2367 return hist_data; 2368 } 2369 } 2370 2371 return NULL; 2372 } 2373 2374 static struct trace_event_file *event_file(struct trace_array *tr, 2375 char *system, char *event_name) 2376 { 2377 struct trace_event_file *file; 2378 2379 file = __find_event_file(tr, system, event_name); 2380 if (!file) 2381 return ERR_PTR(-EINVAL); 2382 2383 return file; 2384 } 2385 2386 static struct hist_field * 2387 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2388 char *system, char *event_name, char *field_name) 2389 { 2390 struct hist_field *event_var; 2391 char *synthetic_name; 2392 2393 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2394 if (!synthetic_name) 2395 return ERR_PTR(-ENOMEM); 2396 2397 strcpy(synthetic_name, "synthetic_"); 2398 strcat(synthetic_name, field_name); 2399 2400 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2401 2402 kfree(synthetic_name); 2403 2404 return event_var; 2405 } 2406 2407 /** 2408 * create_field_var_hist - Automatically create a histogram and var for a field 2409 * @target_hist_data: The target hist trigger 2410 * @subsys_name: Optional subsystem name 2411 * @event_name: Optional event name 2412 * @field_name: The name of the field (and the resulting variable) 2413 * 2414 * Hist trigger actions fetch data from variables, not directly from 2415 * events. However, for convenience, users are allowed to directly 2416 * specify an event field in an action, which will be automatically 2417 * converted into a variable on their behalf. 2418 2419 * If a user specifies a field on an event that isn't the event the 2420 * histogram currently being defined (the target event histogram), the 2421 * only way that can be accomplished is if a new hist trigger is 2422 * created and the field variable defined on that. 2423 * 2424 * This function creates a new histogram compatible with the target 2425 * event (meaning a histogram with the same key as the target 2426 * histogram), and creates a variable for the specified field, but 2427 * with 'synthetic_' prepended to the variable name in order to avoid 2428 * collision with normal field variables. 2429 * 2430 * Return: The variable created for the field. 2431 */ 2432 static struct hist_field * 2433 create_field_var_hist(struct hist_trigger_data *target_hist_data, 2434 char *subsys_name, char *event_name, char *field_name) 2435 { 2436 struct trace_array *tr = target_hist_data->event_file->tr; 2437 struct hist_trigger_data *hist_data; 2438 unsigned int i, n, first = true; 2439 struct field_var_hist *var_hist; 2440 struct trace_event_file *file; 2441 struct hist_field *key_field; 2442 struct hist_field *event_var; 2443 char *saved_filter; 2444 char *cmd; 2445 int ret; 2446 2447 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 2448 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2449 return ERR_PTR(-EINVAL); 2450 } 2451 2452 file = event_file(tr, subsys_name, event_name); 2453 2454 if (IS_ERR(file)) { 2455 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 2456 ret = PTR_ERR(file); 2457 return ERR_PTR(ret); 2458 } 2459 2460 /* 2461 * Look for a histogram compatible with target. We'll use the 2462 * found histogram specification to create a new matching 2463 * histogram with our variable on it. target_hist_data is not 2464 * yet a registered histogram so we can't use that. 2465 */ 2466 hist_data = find_compatible_hist(target_hist_data, file); 2467 if (!hist_data) { 2468 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 2469 return ERR_PTR(-EINVAL); 2470 } 2471 2472 /* See if a synthetic field variable has already been created */ 2473 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2474 event_name, field_name); 2475 if (!IS_ERR_OR_NULL(event_var)) 2476 return event_var; 2477 2478 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 2479 if (!var_hist) 2480 return ERR_PTR(-ENOMEM); 2481 2482 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2483 if (!cmd) { 2484 kfree(var_hist); 2485 return ERR_PTR(-ENOMEM); 2486 } 2487 2488 /* Use the same keys as the compatible histogram */ 2489 strcat(cmd, "keys="); 2490 2491 for_each_hist_key_field(i, hist_data) { 2492 key_field = hist_data->fields[i]; 2493 if (!first) 2494 strcat(cmd, ","); 2495 strcat(cmd, key_field->field->name); 2496 first = false; 2497 } 2498 2499 /* Create the synthetic field variable specification */ 2500 strcat(cmd, ":synthetic_"); 2501 strcat(cmd, field_name); 2502 strcat(cmd, "="); 2503 strcat(cmd, field_name); 2504 2505 /* Use the same filter as the compatible histogram */ 2506 saved_filter = find_trigger_filter(hist_data, file); 2507 if (saved_filter) { 2508 strcat(cmd, " if "); 2509 strcat(cmd, saved_filter); 2510 } 2511 2512 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 2513 if (!var_hist->cmd) { 2514 kfree(cmd); 2515 kfree(var_hist); 2516 return ERR_PTR(-ENOMEM); 2517 } 2518 2519 /* Save the compatible histogram information */ 2520 var_hist->hist_data = hist_data; 2521 2522 /* Create the new histogram with our variable */ 2523 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 2524 "", "hist", cmd); 2525 if (ret) { 2526 kfree(cmd); 2527 kfree(var_hist->cmd); 2528 kfree(var_hist); 2529 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 2530 return ERR_PTR(ret); 2531 } 2532 2533 kfree(cmd); 2534 2535 /* If we can't find the variable, something went wrong */ 2536 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 2537 event_name, field_name); 2538 if (IS_ERR_OR_NULL(event_var)) { 2539 kfree(var_hist->cmd); 2540 kfree(var_hist); 2541 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 2542 return ERR_PTR(-EINVAL); 2543 } 2544 2545 n = target_hist_data->n_field_var_hists; 2546 target_hist_data->field_var_hists[n] = var_hist; 2547 target_hist_data->n_field_var_hists++; 2548 2549 return event_var; 2550 } 2551 2552 static struct hist_field * 2553 find_target_event_var(struct hist_trigger_data *hist_data, 2554 char *subsys_name, char *event_name, char *var_name) 2555 { 2556 struct trace_event_file *file = hist_data->event_file; 2557 struct hist_field *hist_field = NULL; 2558 2559 if (subsys_name) { 2560 struct trace_event_call *call; 2561 2562 if (!event_name) 2563 return NULL; 2564 2565 call = file->event_call; 2566 2567 if (strcmp(subsys_name, call->class->system) != 0) 2568 return NULL; 2569 2570 if (strcmp(event_name, trace_event_name(call)) != 0) 2571 return NULL; 2572 } 2573 2574 hist_field = find_var_field(hist_data, var_name); 2575 2576 return hist_field; 2577 } 2578 2579 static inline void __update_field_vars(struct tracing_map_elt *elt, 2580 struct trace_buffer *buffer, 2581 struct ring_buffer_event *rbe, 2582 void *rec, 2583 struct field_var **field_vars, 2584 unsigned int n_field_vars, 2585 unsigned int field_var_str_start) 2586 { 2587 struct hist_elt_data *elt_data = elt->private_data; 2588 unsigned int i, j, var_idx; 2589 u64 var_val; 2590 2591 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 2592 struct field_var *field_var = field_vars[i]; 2593 struct hist_field *var = field_var->var; 2594 struct hist_field *val = field_var->val; 2595 2596 var_val = val->fn(val, elt, buffer, rbe, rec); 2597 var_idx = var->var.idx; 2598 2599 if (val->flags & HIST_FIELD_FL_STRING) { 2600 char *str = elt_data->field_var_str[j++]; 2601 char *val_str = (char *)(uintptr_t)var_val; 2602 2603 strscpy(str, val_str, STR_VAR_LEN_MAX); 2604 var_val = (u64)(uintptr_t)str; 2605 } 2606 tracing_map_set_var(elt, var_idx, var_val); 2607 } 2608 } 2609 2610 static void update_field_vars(struct hist_trigger_data *hist_data, 2611 struct tracing_map_elt *elt, 2612 struct trace_buffer *buffer, 2613 struct ring_buffer_event *rbe, 2614 void *rec) 2615 { 2616 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars, 2617 hist_data->n_field_vars, 0); 2618 } 2619 2620 static void save_track_data_vars(struct hist_trigger_data *hist_data, 2621 struct tracing_map_elt *elt, 2622 struct trace_buffer *buffer, void *rec, 2623 struct ring_buffer_event *rbe, void *key, 2624 struct action_data *data, u64 *var_ref_vals) 2625 { 2626 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars, 2627 hist_data->n_save_vars, hist_data->n_field_var_str); 2628 } 2629 2630 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 2631 struct trace_event_file *file, 2632 char *name, int size, const char *type) 2633 { 2634 struct hist_field *var; 2635 int idx; 2636 2637 if (find_var(hist_data, file, name) && !hist_data->remove) { 2638 var = ERR_PTR(-EINVAL); 2639 goto out; 2640 } 2641 2642 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 2643 if (!var) { 2644 var = ERR_PTR(-ENOMEM); 2645 goto out; 2646 } 2647 2648 idx = tracing_map_add_var(hist_data->map); 2649 if (idx < 0) { 2650 kfree(var); 2651 var = ERR_PTR(-EINVAL); 2652 goto out; 2653 } 2654 2655 var->ref = 1; 2656 var->flags = HIST_FIELD_FL_VAR; 2657 var->var.idx = idx; 2658 var->var.hist_data = var->hist_data = hist_data; 2659 var->size = size; 2660 var->var.name = kstrdup(name, GFP_KERNEL); 2661 var->type = kstrdup(type, GFP_KERNEL); 2662 if (!var->var.name || !var->type) { 2663 kfree(var->var.name); 2664 kfree(var->type); 2665 kfree(var); 2666 var = ERR_PTR(-ENOMEM); 2667 } 2668 out: 2669 return var; 2670 } 2671 2672 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 2673 struct trace_event_file *file, 2674 char *field_name) 2675 { 2676 struct hist_field *val = NULL, *var = NULL; 2677 unsigned long flags = HIST_FIELD_FL_VAR; 2678 struct trace_array *tr = file->tr; 2679 struct field_var *field_var; 2680 int ret = 0; 2681 2682 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 2683 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 2684 ret = -EINVAL; 2685 goto err; 2686 } 2687 2688 val = parse_atom(hist_data, file, field_name, &flags, NULL); 2689 if (IS_ERR(val)) { 2690 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 2691 ret = PTR_ERR(val); 2692 goto err; 2693 } 2694 2695 var = create_var(hist_data, file, field_name, val->size, val->type); 2696 if (IS_ERR(var)) { 2697 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 2698 kfree(val); 2699 ret = PTR_ERR(var); 2700 goto err; 2701 } 2702 2703 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 2704 if (!field_var) { 2705 kfree(val); 2706 kfree(var); 2707 ret = -ENOMEM; 2708 goto err; 2709 } 2710 2711 field_var->var = var; 2712 field_var->val = val; 2713 out: 2714 return field_var; 2715 err: 2716 field_var = ERR_PTR(ret); 2717 goto out; 2718 } 2719 2720 /** 2721 * create_target_field_var - Automatically create a variable for a field 2722 * @target_hist_data: The target hist trigger 2723 * @subsys_name: Optional subsystem name 2724 * @event_name: Optional event name 2725 * @var_name: The name of the field (and the resulting variable) 2726 * 2727 * Hist trigger actions fetch data from variables, not directly from 2728 * events. However, for convenience, users are allowed to directly 2729 * specify an event field in an action, which will be automatically 2730 * converted into a variable on their behalf. 2731 2732 * This function creates a field variable with the name var_name on 2733 * the hist trigger currently being defined on the target event. If 2734 * subsys_name and event_name are specified, this function simply 2735 * verifies that they do in fact match the target event subsystem and 2736 * event name. 2737 * 2738 * Return: The variable created for the field. 2739 */ 2740 static struct field_var * 2741 create_target_field_var(struct hist_trigger_data *target_hist_data, 2742 char *subsys_name, char *event_name, char *var_name) 2743 { 2744 struct trace_event_file *file = target_hist_data->event_file; 2745 2746 if (subsys_name) { 2747 struct trace_event_call *call; 2748 2749 if (!event_name) 2750 return NULL; 2751 2752 call = file->event_call; 2753 2754 if (strcmp(subsys_name, call->class->system) != 0) 2755 return NULL; 2756 2757 if (strcmp(event_name, trace_event_name(call)) != 0) 2758 return NULL; 2759 } 2760 2761 return create_field_var(target_hist_data, file, var_name); 2762 } 2763 2764 static bool check_track_val_max(u64 track_val, u64 var_val) 2765 { 2766 if (var_val <= track_val) 2767 return false; 2768 2769 return true; 2770 } 2771 2772 static bool check_track_val_changed(u64 track_val, u64 var_val) 2773 { 2774 if (var_val == track_val) 2775 return false; 2776 2777 return true; 2778 } 2779 2780 static u64 get_track_val(struct hist_trigger_data *hist_data, 2781 struct tracing_map_elt *elt, 2782 struct action_data *data) 2783 { 2784 unsigned int track_var_idx = data->track_data.track_var->var.idx; 2785 u64 track_val; 2786 2787 track_val = tracing_map_read_var(elt, track_var_idx); 2788 2789 return track_val; 2790 } 2791 2792 static void save_track_val(struct hist_trigger_data *hist_data, 2793 struct tracing_map_elt *elt, 2794 struct action_data *data, u64 var_val) 2795 { 2796 unsigned int track_var_idx = data->track_data.track_var->var.idx; 2797 2798 tracing_map_set_var(elt, track_var_idx, var_val); 2799 } 2800 2801 static void save_track_data(struct hist_trigger_data *hist_data, 2802 struct tracing_map_elt *elt, 2803 struct trace_buffer *buffer, void *rec, 2804 struct ring_buffer_event *rbe, void *key, 2805 struct action_data *data, u64 *var_ref_vals) 2806 { 2807 if (data->track_data.save_data) 2808 data->track_data.save_data(hist_data, elt, buffer, rec, rbe, 2809 key, data, var_ref_vals); 2810 } 2811 2812 static bool check_track_val(struct tracing_map_elt *elt, 2813 struct action_data *data, 2814 u64 var_val) 2815 { 2816 struct hist_trigger_data *hist_data; 2817 u64 track_val; 2818 2819 hist_data = data->track_data.track_var->hist_data; 2820 track_val = get_track_val(hist_data, elt, data); 2821 2822 return data->track_data.check_val(track_val, var_val); 2823 } 2824 2825 #ifdef CONFIG_TRACER_SNAPSHOT 2826 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 2827 { 2828 /* called with tr->max_lock held */ 2829 struct track_data *track_data = tr->cond_snapshot->cond_data; 2830 struct hist_elt_data *elt_data, *track_elt_data; 2831 struct snapshot_context *context = cond_data; 2832 struct action_data *action; 2833 u64 track_val; 2834 2835 if (!track_data) 2836 return false; 2837 2838 action = track_data->action_data; 2839 2840 track_val = get_track_val(track_data->hist_data, context->elt, 2841 track_data->action_data); 2842 2843 if (!action->track_data.check_val(track_data->track_val, track_val)) 2844 return false; 2845 2846 track_data->track_val = track_val; 2847 memcpy(track_data->key, context->key, track_data->key_len); 2848 2849 elt_data = context->elt->private_data; 2850 track_elt_data = track_data->elt.private_data; 2851 if (elt_data->comm) 2852 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 2853 2854 track_data->updated = true; 2855 2856 return true; 2857 } 2858 2859 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 2860 struct tracing_map_elt *elt, 2861 struct trace_buffer *buffer, void *rec, 2862 struct ring_buffer_event *rbe, void *key, 2863 struct action_data *data, 2864 u64 *var_ref_vals) 2865 { 2866 struct trace_event_file *file = hist_data->event_file; 2867 struct snapshot_context context; 2868 2869 context.elt = elt; 2870 context.key = key; 2871 2872 tracing_snapshot_cond(file->tr, &context); 2873 } 2874 2875 static void hist_trigger_print_key(struct seq_file *m, 2876 struct hist_trigger_data *hist_data, 2877 void *key, 2878 struct tracing_map_elt *elt); 2879 2880 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 2881 { 2882 unsigned int i; 2883 2884 if (!hist_data->n_actions) 2885 return NULL; 2886 2887 for (i = 0; i < hist_data->n_actions; i++) { 2888 struct action_data *data = hist_data->actions[i]; 2889 2890 if (data->action == ACTION_SNAPSHOT) 2891 return data; 2892 } 2893 2894 return NULL; 2895 } 2896 2897 static void track_data_snapshot_print(struct seq_file *m, 2898 struct hist_trigger_data *hist_data) 2899 { 2900 struct trace_event_file *file = hist_data->event_file; 2901 struct track_data *track_data; 2902 struct action_data *action; 2903 2904 track_data = tracing_cond_snapshot_data(file->tr); 2905 if (!track_data) 2906 return; 2907 2908 if (!track_data->updated) 2909 return; 2910 2911 action = snapshot_action(hist_data); 2912 if (!action) 2913 return; 2914 2915 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 2916 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 2917 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 2918 action->track_data.var_str, track_data->track_val); 2919 2920 seq_puts(m, "\ttriggered by event with key: "); 2921 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 2922 seq_putc(m, '\n'); 2923 } 2924 #else 2925 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 2926 { 2927 return false; 2928 } 2929 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 2930 struct tracing_map_elt *elt, 2931 struct trace_buffer *buffer, void *rec, 2932 struct ring_buffer_event *rbe, void *key, 2933 struct action_data *data, 2934 u64 *var_ref_vals) {} 2935 static void track_data_snapshot_print(struct seq_file *m, 2936 struct hist_trigger_data *hist_data) {} 2937 #endif /* CONFIG_TRACER_SNAPSHOT */ 2938 2939 static void track_data_print(struct seq_file *m, 2940 struct hist_trigger_data *hist_data, 2941 struct tracing_map_elt *elt, 2942 struct action_data *data) 2943 { 2944 u64 track_val = get_track_val(hist_data, elt, data); 2945 unsigned int i, save_var_idx; 2946 2947 if (data->handler == HANDLER_ONMAX) 2948 seq_printf(m, "\n\tmax: %10llu", track_val); 2949 else if (data->handler == HANDLER_ONCHANGE) 2950 seq_printf(m, "\n\tchanged: %10llu", track_val); 2951 2952 if (data->action == ACTION_SNAPSHOT) 2953 return; 2954 2955 for (i = 0; i < hist_data->n_save_vars; i++) { 2956 struct hist_field *save_val = hist_data->save_vars[i]->val; 2957 struct hist_field *save_var = hist_data->save_vars[i]->var; 2958 u64 val; 2959 2960 save_var_idx = save_var->var.idx; 2961 2962 val = tracing_map_read_var(elt, save_var_idx); 2963 2964 if (save_val->flags & HIST_FIELD_FL_STRING) { 2965 seq_printf(m, " %s: %-32s", save_var->var.name, 2966 (char *)(uintptr_t)(val)); 2967 } else 2968 seq_printf(m, " %s: %10llu", save_var->var.name, val); 2969 } 2970 } 2971 2972 static void ontrack_action(struct hist_trigger_data *hist_data, 2973 struct tracing_map_elt *elt, 2974 struct trace_buffer *buffer, void *rec, 2975 struct ring_buffer_event *rbe, void *key, 2976 struct action_data *data, u64 *var_ref_vals) 2977 { 2978 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 2979 2980 if (check_track_val(elt, data, var_val)) { 2981 save_track_val(hist_data, elt, data, var_val); 2982 save_track_data(hist_data, elt, buffer, rec, rbe, 2983 key, data, var_ref_vals); 2984 } 2985 } 2986 2987 static void action_data_destroy(struct action_data *data) 2988 { 2989 unsigned int i; 2990 2991 lockdep_assert_held(&event_mutex); 2992 2993 kfree(data->action_name); 2994 2995 for (i = 0; i < data->n_params; i++) 2996 kfree(data->params[i]); 2997 2998 if (data->synth_event) 2999 data->synth_event->ref--; 3000 3001 kfree(data->synth_event_name); 3002 3003 kfree(data); 3004 } 3005 3006 static void track_data_destroy(struct hist_trigger_data *hist_data, 3007 struct action_data *data) 3008 { 3009 struct trace_event_file *file = hist_data->event_file; 3010 3011 destroy_hist_field(data->track_data.track_var, 0); 3012 3013 if (data->action == ACTION_SNAPSHOT) { 3014 struct track_data *track_data; 3015 3016 track_data = tracing_cond_snapshot_data(file->tr); 3017 if (track_data && track_data->hist_data == hist_data) { 3018 tracing_snapshot_cond_disable(file->tr); 3019 track_data_free(track_data); 3020 } 3021 } 3022 3023 kfree(data->track_data.var_str); 3024 3025 action_data_destroy(data); 3026 } 3027 3028 static int action_create(struct hist_trigger_data *hist_data, 3029 struct action_data *data); 3030 3031 static int track_data_create(struct hist_trigger_data *hist_data, 3032 struct action_data *data) 3033 { 3034 struct hist_field *var_field, *ref_field, *track_var = NULL; 3035 struct trace_event_file *file = hist_data->event_file; 3036 struct trace_array *tr = file->tr; 3037 char *track_data_var_str; 3038 int ret = 0; 3039 3040 track_data_var_str = data->track_data.var_str; 3041 if (track_data_var_str[0] != '$') { 3042 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3043 return -EINVAL; 3044 } 3045 track_data_var_str++; 3046 3047 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3048 if (!var_field) { 3049 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3050 return -EINVAL; 3051 } 3052 3053 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3054 if (!ref_field) 3055 return -ENOMEM; 3056 3057 data->track_data.var_ref = ref_field; 3058 3059 if (data->handler == HANDLER_ONMAX) 3060 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3061 if (IS_ERR(track_var)) { 3062 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3063 ret = PTR_ERR(track_var); 3064 goto out; 3065 } 3066 3067 if (data->handler == HANDLER_ONCHANGE) 3068 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3069 if (IS_ERR(track_var)) { 3070 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3071 ret = PTR_ERR(track_var); 3072 goto out; 3073 } 3074 data->track_data.track_var = track_var; 3075 3076 ret = action_create(hist_data, data); 3077 out: 3078 return ret; 3079 } 3080 3081 static int parse_action_params(struct trace_array *tr, char *params, 3082 struct action_data *data) 3083 { 3084 char *param, *saved_param; 3085 bool first_param = true; 3086 int ret = 0; 3087 3088 while (params) { 3089 if (data->n_params >= SYNTH_FIELDS_MAX) { 3090 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3091 goto out; 3092 } 3093 3094 param = strsep(¶ms, ","); 3095 if (!param) { 3096 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3097 ret = -EINVAL; 3098 goto out; 3099 } 3100 3101 param = strstrip(param); 3102 if (strlen(param) < 2) { 3103 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3104 ret = -EINVAL; 3105 goto out; 3106 } 3107 3108 saved_param = kstrdup(param, GFP_KERNEL); 3109 if (!saved_param) { 3110 ret = -ENOMEM; 3111 goto out; 3112 } 3113 3114 if (first_param && data->use_trace_keyword) { 3115 data->synth_event_name = saved_param; 3116 first_param = false; 3117 continue; 3118 } 3119 first_param = false; 3120 3121 data->params[data->n_params++] = saved_param; 3122 } 3123 out: 3124 return ret; 3125 } 3126 3127 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3128 enum handler_id handler) 3129 { 3130 char *action_name; 3131 int ret = 0; 3132 3133 strsep(&str, "."); 3134 if (!str) { 3135 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3136 ret = -EINVAL; 3137 goto out; 3138 } 3139 3140 action_name = strsep(&str, "("); 3141 if (!action_name || !str) { 3142 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3143 ret = -EINVAL; 3144 goto out; 3145 } 3146 3147 if (str_has_prefix(action_name, "save")) { 3148 char *params = strsep(&str, ")"); 3149 3150 if (!params) { 3151 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3152 ret = -EINVAL; 3153 goto out; 3154 } 3155 3156 ret = parse_action_params(tr, params, data); 3157 if (ret) 3158 goto out; 3159 3160 if (handler == HANDLER_ONMAX) 3161 data->track_data.check_val = check_track_val_max; 3162 else if (handler == HANDLER_ONCHANGE) 3163 data->track_data.check_val = check_track_val_changed; 3164 else { 3165 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3166 ret = -EINVAL; 3167 goto out; 3168 } 3169 3170 data->track_data.save_data = save_track_data_vars; 3171 data->fn = ontrack_action; 3172 data->action = ACTION_SAVE; 3173 } else if (str_has_prefix(action_name, "snapshot")) { 3174 char *params = strsep(&str, ")"); 3175 3176 if (!str) { 3177 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3178 ret = -EINVAL; 3179 goto out; 3180 } 3181 3182 if (handler == HANDLER_ONMAX) 3183 data->track_data.check_val = check_track_val_max; 3184 else if (handler == HANDLER_ONCHANGE) 3185 data->track_data.check_val = check_track_val_changed; 3186 else { 3187 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3188 ret = -EINVAL; 3189 goto out; 3190 } 3191 3192 data->track_data.save_data = save_track_data_snapshot; 3193 data->fn = ontrack_action; 3194 data->action = ACTION_SNAPSHOT; 3195 } else { 3196 char *params = strsep(&str, ")"); 3197 3198 if (str_has_prefix(action_name, "trace")) 3199 data->use_trace_keyword = true; 3200 3201 if (params) { 3202 ret = parse_action_params(tr, params, data); 3203 if (ret) 3204 goto out; 3205 } 3206 3207 if (handler == HANDLER_ONMAX) 3208 data->track_data.check_val = check_track_val_max; 3209 else if (handler == HANDLER_ONCHANGE) 3210 data->track_data.check_val = check_track_val_changed; 3211 3212 if (handler != HANDLER_ONMATCH) { 3213 data->track_data.save_data = action_trace; 3214 data->fn = ontrack_action; 3215 } else 3216 data->fn = action_trace; 3217 3218 data->action = ACTION_TRACE; 3219 } 3220 3221 data->action_name = kstrdup(action_name, GFP_KERNEL); 3222 if (!data->action_name) { 3223 ret = -ENOMEM; 3224 goto out; 3225 } 3226 3227 data->handler = handler; 3228 out: 3229 return ret; 3230 } 3231 3232 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3233 char *str, enum handler_id handler) 3234 { 3235 struct action_data *data; 3236 int ret = -EINVAL; 3237 char *var_str; 3238 3239 data = kzalloc(sizeof(*data), GFP_KERNEL); 3240 if (!data) 3241 return ERR_PTR(-ENOMEM); 3242 3243 var_str = strsep(&str, ")"); 3244 if (!var_str || !str) { 3245 ret = -EINVAL; 3246 goto free; 3247 } 3248 3249 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3250 if (!data->track_data.var_str) { 3251 ret = -ENOMEM; 3252 goto free; 3253 } 3254 3255 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3256 if (ret) 3257 goto free; 3258 out: 3259 return data; 3260 free: 3261 track_data_destroy(hist_data, data); 3262 data = ERR_PTR(ret); 3263 goto out; 3264 } 3265 3266 static void onmatch_destroy(struct action_data *data) 3267 { 3268 kfree(data->match_data.event); 3269 kfree(data->match_data.event_system); 3270 3271 action_data_destroy(data); 3272 } 3273 3274 static void destroy_field_var(struct field_var *field_var) 3275 { 3276 if (!field_var) 3277 return; 3278 3279 destroy_hist_field(field_var->var, 0); 3280 destroy_hist_field(field_var->val, 0); 3281 3282 kfree(field_var); 3283 } 3284 3285 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3286 { 3287 unsigned int i; 3288 3289 for (i = 0; i < hist_data->n_field_vars; i++) 3290 destroy_field_var(hist_data->field_vars[i]); 3291 3292 for (i = 0; i < hist_data->n_save_vars; i++) 3293 destroy_field_var(hist_data->save_vars[i]); 3294 } 3295 3296 static void save_field_var(struct hist_trigger_data *hist_data, 3297 struct field_var *field_var) 3298 { 3299 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3300 3301 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3302 hist_data->n_field_var_str++; 3303 } 3304 3305 3306 static int check_synth_field(struct synth_event *event, 3307 struct hist_field *hist_field, 3308 unsigned int field_pos) 3309 { 3310 struct synth_field *field; 3311 3312 if (field_pos >= event->n_fields) 3313 return -EINVAL; 3314 3315 field = event->fields[field_pos]; 3316 3317 /* 3318 * A dynamic string synth field can accept static or 3319 * dynamic. A static string synth field can only accept a 3320 * same-sized static string, which is checked for later. 3321 */ 3322 if (strstr(hist_field->type, "char[") && field->is_string 3323 && field->is_dynamic) 3324 return 0; 3325 3326 if (strcmp(field->type, hist_field->type) != 0) { 3327 if (field->size != hist_field->size || 3328 field->is_signed != hist_field->is_signed) 3329 return -EINVAL; 3330 } 3331 3332 return 0; 3333 } 3334 3335 static struct hist_field * 3336 trace_action_find_var(struct hist_trigger_data *hist_data, 3337 struct action_data *data, 3338 char *system, char *event, char *var) 3339 { 3340 struct trace_array *tr = hist_data->event_file->tr; 3341 struct hist_field *hist_field; 3342 3343 var++; /* skip '$' */ 3344 3345 hist_field = find_target_event_var(hist_data, system, event, var); 3346 if (!hist_field) { 3347 if (!system && data->handler == HANDLER_ONMATCH) { 3348 system = data->match_data.event_system; 3349 event = data->match_data.event; 3350 } 3351 3352 hist_field = find_event_var(hist_data, system, event, var); 3353 } 3354 3355 if (!hist_field) 3356 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3357 3358 return hist_field; 3359 } 3360 3361 static struct hist_field * 3362 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3363 struct action_data *data, char *system, 3364 char *event, char *var) 3365 { 3366 struct hist_field *hist_field = NULL; 3367 struct field_var *field_var; 3368 3369 /* 3370 * First try to create a field var on the target event (the 3371 * currently being defined). This will create a variable for 3372 * unqualified fields on the target event, or if qualified, 3373 * target fields that have qualified names matching the target. 3374 */ 3375 field_var = create_target_field_var(hist_data, system, event, var); 3376 3377 if (field_var && !IS_ERR(field_var)) { 3378 save_field_var(hist_data, field_var); 3379 hist_field = field_var->var; 3380 } else { 3381 field_var = NULL; 3382 /* 3383 * If no explicit system.event is specified, default to 3384 * looking for fields on the onmatch(system.event.xxx) 3385 * event. 3386 */ 3387 if (!system && data->handler == HANDLER_ONMATCH) { 3388 system = data->match_data.event_system; 3389 event = data->match_data.event; 3390 } 3391 3392 /* 3393 * At this point, we're looking at a field on another 3394 * event. Because we can't modify a hist trigger on 3395 * another event to add a variable for a field, we need 3396 * to create a new trigger on that event and create the 3397 * variable at the same time. 3398 */ 3399 hist_field = create_field_var_hist(hist_data, system, event, var); 3400 if (IS_ERR(hist_field)) 3401 goto free; 3402 } 3403 out: 3404 return hist_field; 3405 free: 3406 destroy_field_var(field_var); 3407 hist_field = NULL; 3408 goto out; 3409 } 3410 3411 static int trace_action_create(struct hist_trigger_data *hist_data, 3412 struct action_data *data) 3413 { 3414 struct trace_array *tr = hist_data->event_file->tr; 3415 char *event_name, *param, *system = NULL; 3416 struct hist_field *hist_field, *var_ref; 3417 unsigned int i; 3418 unsigned int field_pos = 0; 3419 struct synth_event *event; 3420 char *synth_event_name; 3421 int var_ref_idx, ret = 0; 3422 3423 lockdep_assert_held(&event_mutex); 3424 3425 if (data->use_trace_keyword) 3426 synth_event_name = data->synth_event_name; 3427 else 3428 synth_event_name = data->action_name; 3429 3430 event = find_synth_event(synth_event_name); 3431 if (!event) { 3432 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 3433 return -EINVAL; 3434 } 3435 3436 event->ref++; 3437 3438 for (i = 0; i < data->n_params; i++) { 3439 char *p; 3440 3441 p = param = kstrdup(data->params[i], GFP_KERNEL); 3442 if (!param) { 3443 ret = -ENOMEM; 3444 goto err; 3445 } 3446 3447 system = strsep(¶m, "."); 3448 if (!param) { 3449 param = (char *)system; 3450 system = event_name = NULL; 3451 } else { 3452 event_name = strsep(¶m, "."); 3453 if (!param) { 3454 kfree(p); 3455 ret = -EINVAL; 3456 goto err; 3457 } 3458 } 3459 3460 if (param[0] == '$') 3461 hist_field = trace_action_find_var(hist_data, data, 3462 system, event_name, 3463 param); 3464 else 3465 hist_field = trace_action_create_field_var(hist_data, 3466 data, 3467 system, 3468 event_name, 3469 param); 3470 3471 if (!hist_field) { 3472 kfree(p); 3473 ret = -EINVAL; 3474 goto err; 3475 } 3476 3477 if (check_synth_field(event, hist_field, field_pos) == 0) { 3478 var_ref = create_var_ref(hist_data, hist_field, 3479 system, event_name); 3480 if (!var_ref) { 3481 kfree(p); 3482 ret = -ENOMEM; 3483 goto err; 3484 } 3485 3486 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 3487 if (WARN_ON(var_ref_idx < 0)) { 3488 ret = var_ref_idx; 3489 goto err; 3490 } 3491 3492 data->var_ref_idx[i] = var_ref_idx; 3493 3494 field_pos++; 3495 kfree(p); 3496 continue; 3497 } 3498 3499 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 3500 kfree(p); 3501 ret = -EINVAL; 3502 goto err; 3503 } 3504 3505 if (field_pos != event->n_fields) { 3506 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 3507 ret = -EINVAL; 3508 goto err; 3509 } 3510 3511 data->synth_event = event; 3512 out: 3513 return ret; 3514 err: 3515 event->ref--; 3516 3517 goto out; 3518 } 3519 3520 static int action_create(struct hist_trigger_data *hist_data, 3521 struct action_data *data) 3522 { 3523 struct trace_event_file *file = hist_data->event_file; 3524 struct trace_array *tr = file->tr; 3525 struct track_data *track_data; 3526 struct field_var *field_var; 3527 unsigned int i; 3528 char *param; 3529 int ret = 0; 3530 3531 if (data->action == ACTION_TRACE) 3532 return trace_action_create(hist_data, data); 3533 3534 if (data->action == ACTION_SNAPSHOT) { 3535 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 3536 if (IS_ERR(track_data)) { 3537 ret = PTR_ERR(track_data); 3538 goto out; 3539 } 3540 3541 ret = tracing_snapshot_cond_enable(file->tr, track_data, 3542 cond_snapshot_update); 3543 if (ret) 3544 track_data_free(track_data); 3545 3546 goto out; 3547 } 3548 3549 if (data->action == ACTION_SAVE) { 3550 if (hist_data->n_save_vars) { 3551 ret = -EEXIST; 3552 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 3553 goto out; 3554 } 3555 3556 for (i = 0; i < data->n_params; i++) { 3557 param = kstrdup(data->params[i], GFP_KERNEL); 3558 if (!param) { 3559 ret = -ENOMEM; 3560 goto out; 3561 } 3562 3563 field_var = create_target_field_var(hist_data, NULL, NULL, param); 3564 if (IS_ERR(field_var)) { 3565 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 3566 errpos(param)); 3567 ret = PTR_ERR(field_var); 3568 kfree(param); 3569 goto out; 3570 } 3571 3572 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 3573 if (field_var->val->flags & HIST_FIELD_FL_STRING) 3574 hist_data->n_save_var_str++; 3575 kfree(param); 3576 } 3577 } 3578 out: 3579 return ret; 3580 } 3581 3582 static int onmatch_create(struct hist_trigger_data *hist_data, 3583 struct action_data *data) 3584 { 3585 return action_create(hist_data, data); 3586 } 3587 3588 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 3589 { 3590 char *match_event, *match_event_system; 3591 struct action_data *data; 3592 int ret = -EINVAL; 3593 3594 data = kzalloc(sizeof(*data), GFP_KERNEL); 3595 if (!data) 3596 return ERR_PTR(-ENOMEM); 3597 3598 match_event = strsep(&str, ")"); 3599 if (!match_event || !str) { 3600 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 3601 goto free; 3602 } 3603 3604 match_event_system = strsep(&match_event, "."); 3605 if (!match_event) { 3606 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 3607 goto free; 3608 } 3609 3610 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 3611 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 3612 goto free; 3613 } 3614 3615 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 3616 if (!data->match_data.event) { 3617 ret = -ENOMEM; 3618 goto free; 3619 } 3620 3621 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 3622 if (!data->match_data.event_system) { 3623 ret = -ENOMEM; 3624 goto free; 3625 } 3626 3627 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 3628 if (ret) 3629 goto free; 3630 out: 3631 return data; 3632 free: 3633 onmatch_destroy(data); 3634 data = ERR_PTR(ret); 3635 goto out; 3636 } 3637 3638 static int create_hitcount_val(struct hist_trigger_data *hist_data) 3639 { 3640 hist_data->fields[HITCOUNT_IDX] = 3641 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 3642 if (!hist_data->fields[HITCOUNT_IDX]) 3643 return -ENOMEM; 3644 3645 hist_data->n_vals++; 3646 hist_data->n_fields++; 3647 3648 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 3649 return -EINVAL; 3650 3651 return 0; 3652 } 3653 3654 static int __create_val_field(struct hist_trigger_data *hist_data, 3655 unsigned int val_idx, 3656 struct trace_event_file *file, 3657 char *var_name, char *field_str, 3658 unsigned long flags) 3659 { 3660 struct hist_field *hist_field; 3661 int ret = 0; 3662 3663 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0); 3664 if (IS_ERR(hist_field)) { 3665 ret = PTR_ERR(hist_field); 3666 goto out; 3667 } 3668 3669 hist_data->fields[val_idx] = hist_field; 3670 3671 ++hist_data->n_vals; 3672 ++hist_data->n_fields; 3673 3674 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3675 ret = -EINVAL; 3676 out: 3677 return ret; 3678 } 3679 3680 static int create_val_field(struct hist_trigger_data *hist_data, 3681 unsigned int val_idx, 3682 struct trace_event_file *file, 3683 char *field_str) 3684 { 3685 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 3686 return -EINVAL; 3687 3688 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 3689 } 3690 3691 static int create_var_field(struct hist_trigger_data *hist_data, 3692 unsigned int val_idx, 3693 struct trace_event_file *file, 3694 char *var_name, char *expr_str) 3695 { 3696 struct trace_array *tr = hist_data->event_file->tr; 3697 unsigned long flags = 0; 3698 int ret; 3699 3700 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 3701 return -EINVAL; 3702 3703 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 3704 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 3705 return -EINVAL; 3706 } 3707 3708 flags |= HIST_FIELD_FL_VAR; 3709 hist_data->n_vars++; 3710 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 3711 return -EINVAL; 3712 3713 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 3714 3715 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING) 3716 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; 3717 3718 return ret; 3719 } 3720 3721 static int create_val_fields(struct hist_trigger_data *hist_data, 3722 struct trace_event_file *file) 3723 { 3724 char *fields_str, *field_str; 3725 unsigned int i, j = 1; 3726 int ret; 3727 3728 ret = create_hitcount_val(hist_data); 3729 if (ret) 3730 goto out; 3731 3732 fields_str = hist_data->attrs->vals_str; 3733 if (!fields_str) 3734 goto out; 3735 3736 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 3737 j < TRACING_MAP_VALS_MAX; i++) { 3738 field_str = strsep(&fields_str, ","); 3739 if (!field_str) 3740 break; 3741 3742 if (strcmp(field_str, "hitcount") == 0) 3743 continue; 3744 3745 ret = create_val_field(hist_data, j++, file, field_str); 3746 if (ret) 3747 goto out; 3748 } 3749 3750 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 3751 ret = -EINVAL; 3752 out: 3753 return ret; 3754 } 3755 3756 static int create_key_field(struct hist_trigger_data *hist_data, 3757 unsigned int key_idx, 3758 unsigned int key_offset, 3759 struct trace_event_file *file, 3760 char *field_str) 3761 { 3762 struct trace_array *tr = hist_data->event_file->tr; 3763 struct hist_field *hist_field = NULL; 3764 unsigned long flags = 0; 3765 unsigned int key_size; 3766 int ret = 0; 3767 3768 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 3769 return -EINVAL; 3770 3771 flags |= HIST_FIELD_FL_KEY; 3772 3773 if (strcmp(field_str, "stacktrace") == 0) { 3774 flags |= HIST_FIELD_FL_STACKTRACE; 3775 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 3776 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 3777 } else { 3778 hist_field = parse_expr(hist_data, file, field_str, flags, 3779 NULL, 0); 3780 if (IS_ERR(hist_field)) { 3781 ret = PTR_ERR(hist_field); 3782 goto out; 3783 } 3784 3785 if (field_has_hist_vars(hist_field, 0)) { 3786 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 3787 destroy_hist_field(hist_field, 0); 3788 ret = -EINVAL; 3789 goto out; 3790 } 3791 3792 key_size = hist_field->size; 3793 } 3794 3795 hist_data->fields[key_idx] = hist_field; 3796 3797 key_size = ALIGN(key_size, sizeof(u64)); 3798 hist_data->fields[key_idx]->size = key_size; 3799 hist_data->fields[key_idx]->offset = key_offset; 3800 3801 hist_data->key_size += key_size; 3802 3803 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 3804 ret = -EINVAL; 3805 goto out; 3806 } 3807 3808 hist_data->n_keys++; 3809 hist_data->n_fields++; 3810 3811 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 3812 return -EINVAL; 3813 3814 ret = key_size; 3815 out: 3816 return ret; 3817 } 3818 3819 static int create_key_fields(struct hist_trigger_data *hist_data, 3820 struct trace_event_file *file) 3821 { 3822 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 3823 char *fields_str, *field_str; 3824 int ret = -EINVAL; 3825 3826 fields_str = hist_data->attrs->keys_str; 3827 if (!fields_str) 3828 goto out; 3829 3830 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 3831 field_str = strsep(&fields_str, ","); 3832 if (!field_str) 3833 break; 3834 ret = create_key_field(hist_data, i, key_offset, 3835 file, field_str); 3836 if (ret < 0) 3837 goto out; 3838 key_offset += ret; 3839 } 3840 if (fields_str) { 3841 ret = -EINVAL; 3842 goto out; 3843 } 3844 ret = 0; 3845 out: 3846 return ret; 3847 } 3848 3849 static int create_var_fields(struct hist_trigger_data *hist_data, 3850 struct trace_event_file *file) 3851 { 3852 unsigned int i, j = hist_data->n_vals; 3853 int ret = 0; 3854 3855 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 3856 3857 for (i = 0; i < n_vars; i++) { 3858 char *var_name = hist_data->attrs->var_defs.name[i]; 3859 char *expr = hist_data->attrs->var_defs.expr[i]; 3860 3861 ret = create_var_field(hist_data, j++, file, var_name, expr); 3862 if (ret) 3863 goto out; 3864 } 3865 out: 3866 return ret; 3867 } 3868 3869 static void free_var_defs(struct hist_trigger_data *hist_data) 3870 { 3871 unsigned int i; 3872 3873 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 3874 kfree(hist_data->attrs->var_defs.name[i]); 3875 kfree(hist_data->attrs->var_defs.expr[i]); 3876 } 3877 3878 hist_data->attrs->var_defs.n_vars = 0; 3879 } 3880 3881 static int parse_var_defs(struct hist_trigger_data *hist_data) 3882 { 3883 struct trace_array *tr = hist_data->event_file->tr; 3884 char *s, *str, *var_name, *field_str; 3885 unsigned int i, j, n_vars = 0; 3886 int ret = 0; 3887 3888 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 3889 str = hist_data->attrs->assignment_str[i]; 3890 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 3891 field_str = strsep(&str, ","); 3892 if (!field_str) 3893 break; 3894 3895 var_name = strsep(&field_str, "="); 3896 if (!var_name || !field_str) { 3897 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 3898 errpos(var_name)); 3899 ret = -EINVAL; 3900 goto free; 3901 } 3902 3903 if (n_vars == TRACING_MAP_VARS_MAX) { 3904 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 3905 ret = -EINVAL; 3906 goto free; 3907 } 3908 3909 s = kstrdup(var_name, GFP_KERNEL); 3910 if (!s) { 3911 ret = -ENOMEM; 3912 goto free; 3913 } 3914 hist_data->attrs->var_defs.name[n_vars] = s; 3915 3916 s = kstrdup(field_str, GFP_KERNEL); 3917 if (!s) { 3918 ret = -ENOMEM; 3919 goto free; 3920 } 3921 hist_data->attrs->var_defs.expr[n_vars++] = s; 3922 3923 hist_data->attrs->var_defs.n_vars = n_vars; 3924 } 3925 } 3926 3927 return ret; 3928 free: 3929 free_var_defs(hist_data); 3930 3931 return ret; 3932 } 3933 3934 static int create_hist_fields(struct hist_trigger_data *hist_data, 3935 struct trace_event_file *file) 3936 { 3937 int ret; 3938 3939 ret = parse_var_defs(hist_data); 3940 if (ret) 3941 goto out; 3942 3943 ret = create_val_fields(hist_data, file); 3944 if (ret) 3945 goto out; 3946 3947 ret = create_var_fields(hist_data, file); 3948 if (ret) 3949 goto out; 3950 3951 ret = create_key_fields(hist_data, file); 3952 if (ret) 3953 goto out; 3954 out: 3955 free_var_defs(hist_data); 3956 3957 return ret; 3958 } 3959 3960 static int is_descending(struct trace_array *tr, const char *str) 3961 { 3962 if (!str) 3963 return 0; 3964 3965 if (strcmp(str, "descending") == 0) 3966 return 1; 3967 3968 if (strcmp(str, "ascending") == 0) 3969 return 0; 3970 3971 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 3972 3973 return -EINVAL; 3974 } 3975 3976 static int create_sort_keys(struct hist_trigger_data *hist_data) 3977 { 3978 struct trace_array *tr = hist_data->event_file->tr; 3979 char *fields_str = hist_data->attrs->sort_key_str; 3980 struct tracing_map_sort_key *sort_key; 3981 int descending, ret = 0; 3982 unsigned int i, j, k; 3983 3984 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 3985 3986 if (!fields_str) 3987 goto out; 3988 3989 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 3990 struct hist_field *hist_field; 3991 char *field_str, *field_name; 3992 const char *test_name; 3993 3994 sort_key = &hist_data->sort_keys[i]; 3995 3996 field_str = strsep(&fields_str, ","); 3997 if (!field_str) 3998 break; 3999 4000 if (!*field_str) { 4001 ret = -EINVAL; 4002 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4003 break; 4004 } 4005 4006 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4007 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 4008 ret = -EINVAL; 4009 break; 4010 } 4011 4012 field_name = strsep(&field_str, "."); 4013 if (!field_name || !*field_name) { 4014 ret = -EINVAL; 4015 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4016 break; 4017 } 4018 4019 if (strcmp(field_name, "hitcount") == 0) { 4020 descending = is_descending(tr, field_str); 4021 if (descending < 0) { 4022 ret = descending; 4023 break; 4024 } 4025 sort_key->descending = descending; 4026 continue; 4027 } 4028 4029 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4030 unsigned int idx; 4031 4032 hist_field = hist_data->fields[j]; 4033 if (hist_field->flags & HIST_FIELD_FL_VAR) 4034 continue; 4035 4036 idx = k++; 4037 4038 test_name = hist_field_name(hist_field, 0); 4039 4040 if (strcmp(field_name, test_name) == 0) { 4041 sort_key->field_idx = idx; 4042 descending = is_descending(tr, field_str); 4043 if (descending < 0) { 4044 ret = descending; 4045 goto out; 4046 } 4047 sort_key->descending = descending; 4048 break; 4049 } 4050 } 4051 if (j == hist_data->n_fields) { 4052 ret = -EINVAL; 4053 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4054 break; 4055 } 4056 } 4057 4058 hist_data->n_sort_keys = i; 4059 out: 4060 return ret; 4061 } 4062 4063 static void destroy_actions(struct hist_trigger_data *hist_data) 4064 { 4065 unsigned int i; 4066 4067 for (i = 0; i < hist_data->n_actions; i++) { 4068 struct action_data *data = hist_data->actions[i]; 4069 4070 if (data->handler == HANDLER_ONMATCH) 4071 onmatch_destroy(data); 4072 else if (data->handler == HANDLER_ONMAX || 4073 data->handler == HANDLER_ONCHANGE) 4074 track_data_destroy(hist_data, data); 4075 else 4076 kfree(data); 4077 } 4078 } 4079 4080 static int parse_actions(struct hist_trigger_data *hist_data) 4081 { 4082 struct trace_array *tr = hist_data->event_file->tr; 4083 struct action_data *data; 4084 unsigned int i; 4085 int ret = 0; 4086 char *str; 4087 int len; 4088 4089 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4090 str = hist_data->attrs->action_str[i]; 4091 4092 if ((len = str_has_prefix(str, "onmatch("))) { 4093 char *action_str = str + len; 4094 4095 data = onmatch_parse(tr, action_str); 4096 if (IS_ERR(data)) { 4097 ret = PTR_ERR(data); 4098 break; 4099 } 4100 } else if ((len = str_has_prefix(str, "onmax("))) { 4101 char *action_str = str + len; 4102 4103 data = track_data_parse(hist_data, action_str, 4104 HANDLER_ONMAX); 4105 if (IS_ERR(data)) { 4106 ret = PTR_ERR(data); 4107 break; 4108 } 4109 } else if ((len = str_has_prefix(str, "onchange("))) { 4110 char *action_str = str + len; 4111 4112 data = track_data_parse(hist_data, action_str, 4113 HANDLER_ONCHANGE); 4114 if (IS_ERR(data)) { 4115 ret = PTR_ERR(data); 4116 break; 4117 } 4118 } else { 4119 ret = -EINVAL; 4120 break; 4121 } 4122 4123 hist_data->actions[hist_data->n_actions++] = data; 4124 } 4125 4126 return ret; 4127 } 4128 4129 static int create_actions(struct hist_trigger_data *hist_data) 4130 { 4131 struct action_data *data; 4132 unsigned int i; 4133 int ret = 0; 4134 4135 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4136 data = hist_data->actions[i]; 4137 4138 if (data->handler == HANDLER_ONMATCH) { 4139 ret = onmatch_create(hist_data, data); 4140 if (ret) 4141 break; 4142 } else if (data->handler == HANDLER_ONMAX || 4143 data->handler == HANDLER_ONCHANGE) { 4144 ret = track_data_create(hist_data, data); 4145 if (ret) 4146 break; 4147 } else { 4148 ret = -EINVAL; 4149 break; 4150 } 4151 } 4152 4153 return ret; 4154 } 4155 4156 static void print_actions(struct seq_file *m, 4157 struct hist_trigger_data *hist_data, 4158 struct tracing_map_elt *elt) 4159 { 4160 unsigned int i; 4161 4162 for (i = 0; i < hist_data->n_actions; i++) { 4163 struct action_data *data = hist_data->actions[i]; 4164 4165 if (data->action == ACTION_SNAPSHOT) 4166 continue; 4167 4168 if (data->handler == HANDLER_ONMAX || 4169 data->handler == HANDLER_ONCHANGE) 4170 track_data_print(m, hist_data, elt, data); 4171 } 4172 } 4173 4174 static void print_action_spec(struct seq_file *m, 4175 struct hist_trigger_data *hist_data, 4176 struct action_data *data) 4177 { 4178 unsigned int i; 4179 4180 if (data->action == ACTION_SAVE) { 4181 for (i = 0; i < hist_data->n_save_vars; i++) { 4182 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4183 if (i < hist_data->n_save_vars - 1) 4184 seq_puts(m, ","); 4185 } 4186 } else if (data->action == ACTION_TRACE) { 4187 if (data->use_trace_keyword) 4188 seq_printf(m, "%s", data->synth_event_name); 4189 for (i = 0; i < data->n_params; i++) { 4190 if (i || data->use_trace_keyword) 4191 seq_puts(m, ","); 4192 seq_printf(m, "%s", data->params[i]); 4193 } 4194 } 4195 } 4196 4197 static void print_track_data_spec(struct seq_file *m, 4198 struct hist_trigger_data *hist_data, 4199 struct action_data *data) 4200 { 4201 if (data->handler == HANDLER_ONMAX) 4202 seq_puts(m, ":onmax("); 4203 else if (data->handler == HANDLER_ONCHANGE) 4204 seq_puts(m, ":onchange("); 4205 seq_printf(m, "%s", data->track_data.var_str); 4206 seq_printf(m, ").%s(", data->action_name); 4207 4208 print_action_spec(m, hist_data, data); 4209 4210 seq_puts(m, ")"); 4211 } 4212 4213 static void print_onmatch_spec(struct seq_file *m, 4214 struct hist_trigger_data *hist_data, 4215 struct action_data *data) 4216 { 4217 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4218 data->match_data.event); 4219 4220 seq_printf(m, "%s(", data->action_name); 4221 4222 print_action_spec(m, hist_data, data); 4223 4224 seq_puts(m, ")"); 4225 } 4226 4227 static bool actions_match(struct hist_trigger_data *hist_data, 4228 struct hist_trigger_data *hist_data_test) 4229 { 4230 unsigned int i, j; 4231 4232 if (hist_data->n_actions != hist_data_test->n_actions) 4233 return false; 4234 4235 for (i = 0; i < hist_data->n_actions; i++) { 4236 struct action_data *data = hist_data->actions[i]; 4237 struct action_data *data_test = hist_data_test->actions[i]; 4238 char *action_name, *action_name_test; 4239 4240 if (data->handler != data_test->handler) 4241 return false; 4242 if (data->action != data_test->action) 4243 return false; 4244 4245 if (data->n_params != data_test->n_params) 4246 return false; 4247 4248 for (j = 0; j < data->n_params; j++) { 4249 if (strcmp(data->params[j], data_test->params[j]) != 0) 4250 return false; 4251 } 4252 4253 if (data->use_trace_keyword) 4254 action_name = data->synth_event_name; 4255 else 4256 action_name = data->action_name; 4257 4258 if (data_test->use_trace_keyword) 4259 action_name_test = data_test->synth_event_name; 4260 else 4261 action_name_test = data_test->action_name; 4262 4263 if (strcmp(action_name, action_name_test) != 0) 4264 return false; 4265 4266 if (data->handler == HANDLER_ONMATCH) { 4267 if (strcmp(data->match_data.event_system, 4268 data_test->match_data.event_system) != 0) 4269 return false; 4270 if (strcmp(data->match_data.event, 4271 data_test->match_data.event) != 0) 4272 return false; 4273 } else if (data->handler == HANDLER_ONMAX || 4274 data->handler == HANDLER_ONCHANGE) { 4275 if (strcmp(data->track_data.var_str, 4276 data_test->track_data.var_str) != 0) 4277 return false; 4278 } 4279 } 4280 4281 return true; 4282 } 4283 4284 4285 static void print_actions_spec(struct seq_file *m, 4286 struct hist_trigger_data *hist_data) 4287 { 4288 unsigned int i; 4289 4290 for (i = 0; i < hist_data->n_actions; i++) { 4291 struct action_data *data = hist_data->actions[i]; 4292 4293 if (data->handler == HANDLER_ONMATCH) 4294 print_onmatch_spec(m, hist_data, data); 4295 else if (data->handler == HANDLER_ONMAX || 4296 data->handler == HANDLER_ONCHANGE) 4297 print_track_data_spec(m, hist_data, data); 4298 } 4299 } 4300 4301 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 4302 { 4303 unsigned int i; 4304 4305 for (i = 0; i < hist_data->n_field_var_hists; i++) { 4306 kfree(hist_data->field_var_hists[i]->cmd); 4307 kfree(hist_data->field_var_hists[i]); 4308 } 4309 } 4310 4311 static void destroy_hist_data(struct hist_trigger_data *hist_data) 4312 { 4313 if (!hist_data) 4314 return; 4315 4316 destroy_hist_trigger_attrs(hist_data->attrs); 4317 destroy_hist_fields(hist_data); 4318 tracing_map_destroy(hist_data->map); 4319 4320 destroy_actions(hist_data); 4321 destroy_field_vars(hist_data); 4322 destroy_field_var_hists(hist_data); 4323 4324 kfree(hist_data); 4325 } 4326 4327 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 4328 { 4329 struct tracing_map *map = hist_data->map; 4330 struct ftrace_event_field *field; 4331 struct hist_field *hist_field; 4332 int i, idx = 0; 4333 4334 for_each_hist_field(i, hist_data) { 4335 hist_field = hist_data->fields[i]; 4336 if (hist_field->flags & HIST_FIELD_FL_KEY) { 4337 tracing_map_cmp_fn_t cmp_fn; 4338 4339 field = hist_field->field; 4340 4341 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 4342 cmp_fn = tracing_map_cmp_none; 4343 else if (!field) 4344 cmp_fn = tracing_map_cmp_num(hist_field->size, 4345 hist_field->is_signed); 4346 else if (is_string_field(field)) 4347 cmp_fn = tracing_map_cmp_string; 4348 else 4349 cmp_fn = tracing_map_cmp_num(field->size, 4350 field->is_signed); 4351 idx = tracing_map_add_key_field(map, 4352 hist_field->offset, 4353 cmp_fn); 4354 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 4355 idx = tracing_map_add_sum_field(map); 4356 4357 if (idx < 0) 4358 return idx; 4359 4360 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4361 idx = tracing_map_add_var(map); 4362 if (idx < 0) 4363 return idx; 4364 hist_field->var.idx = idx; 4365 hist_field->var.hist_data = hist_data; 4366 } 4367 } 4368 4369 return 0; 4370 } 4371 4372 static struct hist_trigger_data * 4373 create_hist_data(unsigned int map_bits, 4374 struct hist_trigger_attrs *attrs, 4375 struct trace_event_file *file, 4376 bool remove) 4377 { 4378 const struct tracing_map_ops *map_ops = NULL; 4379 struct hist_trigger_data *hist_data; 4380 int ret = 0; 4381 4382 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 4383 if (!hist_data) 4384 return ERR_PTR(-ENOMEM); 4385 4386 hist_data->attrs = attrs; 4387 hist_data->remove = remove; 4388 hist_data->event_file = file; 4389 4390 ret = parse_actions(hist_data); 4391 if (ret) 4392 goto free; 4393 4394 ret = create_hist_fields(hist_data, file); 4395 if (ret) 4396 goto free; 4397 4398 ret = create_sort_keys(hist_data); 4399 if (ret) 4400 goto free; 4401 4402 map_ops = &hist_trigger_elt_data_ops; 4403 4404 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 4405 map_ops, hist_data); 4406 if (IS_ERR(hist_data->map)) { 4407 ret = PTR_ERR(hist_data->map); 4408 hist_data->map = NULL; 4409 goto free; 4410 } 4411 4412 ret = create_tracing_map_fields(hist_data); 4413 if (ret) 4414 goto free; 4415 out: 4416 return hist_data; 4417 free: 4418 hist_data->attrs = NULL; 4419 4420 destroy_hist_data(hist_data); 4421 4422 hist_data = ERR_PTR(ret); 4423 4424 goto out; 4425 } 4426 4427 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 4428 struct tracing_map_elt *elt, 4429 struct trace_buffer *buffer, void *rec, 4430 struct ring_buffer_event *rbe, 4431 u64 *var_ref_vals) 4432 { 4433 struct hist_elt_data *elt_data; 4434 struct hist_field *hist_field; 4435 unsigned int i, var_idx; 4436 u64 hist_val; 4437 4438 elt_data = elt->private_data; 4439 elt_data->var_ref_vals = var_ref_vals; 4440 4441 for_each_hist_val_field(i, hist_data) { 4442 hist_field = hist_data->fields[i]; 4443 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec); 4444 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4445 var_idx = hist_field->var.idx; 4446 4447 if (hist_field->flags & HIST_FIELD_FL_STRING) { 4448 unsigned int str_start, var_str_idx, idx; 4449 char *str, *val_str; 4450 4451 str_start = hist_data->n_field_var_str + 4452 hist_data->n_save_var_str; 4453 var_str_idx = hist_field->var_str_idx; 4454 idx = str_start + var_str_idx; 4455 4456 str = elt_data->field_var_str[idx]; 4457 val_str = (char *)(uintptr_t)hist_val; 4458 strscpy(str, val_str, STR_VAR_LEN_MAX); 4459 4460 hist_val = (u64)(uintptr_t)str; 4461 } 4462 tracing_map_set_var(elt, var_idx, hist_val); 4463 continue; 4464 } 4465 tracing_map_update_sum(elt, i, hist_val); 4466 } 4467 4468 for_each_hist_key_field(i, hist_data) { 4469 hist_field = hist_data->fields[i]; 4470 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4471 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec); 4472 var_idx = hist_field->var.idx; 4473 tracing_map_set_var(elt, var_idx, hist_val); 4474 } 4475 } 4476 4477 update_field_vars(hist_data, elt, buffer, rbe, rec); 4478 } 4479 4480 static inline void add_to_key(char *compound_key, void *key, 4481 struct hist_field *key_field, void *rec) 4482 { 4483 size_t size = key_field->size; 4484 4485 if (key_field->flags & HIST_FIELD_FL_STRING) { 4486 struct ftrace_event_field *field; 4487 4488 field = key_field->field; 4489 if (field->filter_type == FILTER_DYN_STRING) 4490 size = *(u32 *)(rec + field->offset) >> 16; 4491 else if (field->filter_type == FILTER_PTR_STRING) 4492 size = strlen(key); 4493 else if (field->filter_type == FILTER_STATIC_STRING) 4494 size = field->size; 4495 4496 /* ensure NULL-termination */ 4497 if (size > key_field->size - 1) 4498 size = key_field->size - 1; 4499 4500 strncpy(compound_key + key_field->offset, (char *)key, size); 4501 } else 4502 memcpy(compound_key + key_field->offset, key, size); 4503 } 4504 4505 static void 4506 hist_trigger_actions(struct hist_trigger_data *hist_data, 4507 struct tracing_map_elt *elt, 4508 struct trace_buffer *buffer, void *rec, 4509 struct ring_buffer_event *rbe, void *key, 4510 u64 *var_ref_vals) 4511 { 4512 struct action_data *data; 4513 unsigned int i; 4514 4515 for (i = 0; i < hist_data->n_actions; i++) { 4516 data = hist_data->actions[i]; 4517 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals); 4518 } 4519 } 4520 4521 static void event_hist_trigger(struct event_trigger_data *data, 4522 struct trace_buffer *buffer, void *rec, 4523 struct ring_buffer_event *rbe) 4524 { 4525 struct hist_trigger_data *hist_data = data->private_data; 4526 bool use_compound_key = (hist_data->n_keys > 1); 4527 unsigned long entries[HIST_STACKTRACE_DEPTH]; 4528 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 4529 char compound_key[HIST_KEY_SIZE_MAX]; 4530 struct tracing_map_elt *elt = NULL; 4531 struct hist_field *key_field; 4532 u64 field_contents; 4533 void *key = NULL; 4534 unsigned int i; 4535 4536 memset(compound_key, 0, hist_data->key_size); 4537 4538 for_each_hist_key_field(i, hist_data) { 4539 key_field = hist_data->fields[i]; 4540 4541 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4542 memset(entries, 0, HIST_STACKTRACE_SIZE); 4543 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 4544 HIST_STACKTRACE_SKIP); 4545 key = entries; 4546 } else { 4547 field_contents = key_field->fn(key_field, elt, buffer, rbe, rec); 4548 if (key_field->flags & HIST_FIELD_FL_STRING) { 4549 key = (void *)(unsigned long)field_contents; 4550 use_compound_key = true; 4551 } else 4552 key = (void *)&field_contents; 4553 } 4554 4555 if (use_compound_key) 4556 add_to_key(compound_key, key, key_field, rec); 4557 } 4558 4559 if (use_compound_key) 4560 key = compound_key; 4561 4562 if (hist_data->n_var_refs && 4563 !resolve_var_refs(hist_data, key, var_ref_vals, false)) 4564 return; 4565 4566 elt = tracing_map_insert(hist_data->map, key); 4567 if (!elt) 4568 return; 4569 4570 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals); 4571 4572 if (resolve_var_refs(hist_data, key, var_ref_vals, true)) 4573 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals); 4574 } 4575 4576 static void hist_trigger_stacktrace_print(struct seq_file *m, 4577 unsigned long *stacktrace_entries, 4578 unsigned int max_entries) 4579 { 4580 char str[KSYM_SYMBOL_LEN]; 4581 unsigned int spaces = 8; 4582 unsigned int i; 4583 4584 for (i = 0; i < max_entries; i++) { 4585 if (!stacktrace_entries[i]) 4586 return; 4587 4588 seq_printf(m, "%*c", 1 + spaces, ' '); 4589 sprint_symbol(str, stacktrace_entries[i]); 4590 seq_printf(m, "%s\n", str); 4591 } 4592 } 4593 4594 static void hist_trigger_print_key(struct seq_file *m, 4595 struct hist_trigger_data *hist_data, 4596 void *key, 4597 struct tracing_map_elt *elt) 4598 { 4599 struct hist_field *key_field; 4600 char str[KSYM_SYMBOL_LEN]; 4601 bool multiline = false; 4602 const char *field_name; 4603 unsigned int i; 4604 u64 uval; 4605 4606 seq_puts(m, "{ "); 4607 4608 for_each_hist_key_field(i, hist_data) { 4609 key_field = hist_data->fields[i]; 4610 4611 if (i > hist_data->n_vals) 4612 seq_puts(m, ", "); 4613 4614 field_name = hist_field_name(key_field, 0); 4615 4616 if (key_field->flags & HIST_FIELD_FL_HEX) { 4617 uval = *(u64 *)(key + key_field->offset); 4618 seq_printf(m, "%s: %llx", field_name, uval); 4619 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 4620 uval = *(u64 *)(key + key_field->offset); 4621 sprint_symbol_no_offset(str, uval); 4622 seq_printf(m, "%s: [%llx] %-45s", field_name, 4623 uval, str); 4624 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 4625 uval = *(u64 *)(key + key_field->offset); 4626 sprint_symbol(str, uval); 4627 seq_printf(m, "%s: [%llx] %-55s", field_name, 4628 uval, str); 4629 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 4630 struct hist_elt_data *elt_data = elt->private_data; 4631 char *comm; 4632 4633 if (WARN_ON_ONCE(!elt_data)) 4634 return; 4635 4636 comm = elt_data->comm; 4637 4638 uval = *(u64 *)(key + key_field->offset); 4639 seq_printf(m, "%s: %-16s[%10llu]", field_name, 4640 comm, uval); 4641 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 4642 const char *syscall_name; 4643 4644 uval = *(u64 *)(key + key_field->offset); 4645 syscall_name = get_syscall_name(uval); 4646 if (!syscall_name) 4647 syscall_name = "unknown_syscall"; 4648 4649 seq_printf(m, "%s: %-30s[%3llu]", field_name, 4650 syscall_name, uval); 4651 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 4652 seq_puts(m, "stacktrace:\n"); 4653 hist_trigger_stacktrace_print(m, 4654 key + key_field->offset, 4655 HIST_STACKTRACE_DEPTH); 4656 multiline = true; 4657 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 4658 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 4659 *(u64 *)(key + key_field->offset)); 4660 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 4661 seq_printf(m, "%s: %-50s", field_name, 4662 (char *)(key + key_field->offset)); 4663 } else { 4664 uval = *(u64 *)(key + key_field->offset); 4665 seq_printf(m, "%s: %10llu", field_name, uval); 4666 } 4667 } 4668 4669 if (!multiline) 4670 seq_puts(m, " "); 4671 4672 seq_puts(m, "}"); 4673 } 4674 4675 static void hist_trigger_entry_print(struct seq_file *m, 4676 struct hist_trigger_data *hist_data, 4677 void *key, 4678 struct tracing_map_elt *elt) 4679 { 4680 const char *field_name; 4681 unsigned int i; 4682 4683 hist_trigger_print_key(m, hist_data, key, elt); 4684 4685 seq_printf(m, " hitcount: %10llu", 4686 tracing_map_read_sum(elt, HITCOUNT_IDX)); 4687 4688 for (i = 1; i < hist_data->n_vals; i++) { 4689 field_name = hist_field_name(hist_data->fields[i], 0); 4690 4691 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR || 4692 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR) 4693 continue; 4694 4695 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) { 4696 seq_printf(m, " %s: %10llx", field_name, 4697 tracing_map_read_sum(elt, i)); 4698 } else { 4699 seq_printf(m, " %s: %10llu", field_name, 4700 tracing_map_read_sum(elt, i)); 4701 } 4702 } 4703 4704 print_actions(m, hist_data, elt); 4705 4706 seq_puts(m, "\n"); 4707 } 4708 4709 static int print_entries(struct seq_file *m, 4710 struct hist_trigger_data *hist_data) 4711 { 4712 struct tracing_map_sort_entry **sort_entries = NULL; 4713 struct tracing_map *map = hist_data->map; 4714 int i, n_entries; 4715 4716 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 4717 hist_data->n_sort_keys, 4718 &sort_entries); 4719 if (n_entries < 0) 4720 return n_entries; 4721 4722 for (i = 0; i < n_entries; i++) 4723 hist_trigger_entry_print(m, hist_data, 4724 sort_entries[i]->key, 4725 sort_entries[i]->elt); 4726 4727 tracing_map_destroy_sort_entries(sort_entries, n_entries); 4728 4729 return n_entries; 4730 } 4731 4732 static void hist_trigger_show(struct seq_file *m, 4733 struct event_trigger_data *data, int n) 4734 { 4735 struct hist_trigger_data *hist_data; 4736 int n_entries; 4737 4738 if (n > 0) 4739 seq_puts(m, "\n\n"); 4740 4741 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4742 data->ops->print(m, data->ops, data); 4743 seq_puts(m, "#\n\n"); 4744 4745 hist_data = data->private_data; 4746 n_entries = print_entries(m, hist_data); 4747 if (n_entries < 0) 4748 n_entries = 0; 4749 4750 track_data_snapshot_print(m, hist_data); 4751 4752 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 4753 (u64)atomic64_read(&hist_data->map->hits), 4754 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 4755 } 4756 4757 static int hist_show(struct seq_file *m, void *v) 4758 { 4759 struct event_trigger_data *data; 4760 struct trace_event_file *event_file; 4761 int n = 0, ret = 0; 4762 4763 mutex_lock(&event_mutex); 4764 4765 event_file = event_file_data(m->private); 4766 if (unlikely(!event_file)) { 4767 ret = -ENODEV; 4768 goto out_unlock; 4769 } 4770 4771 list_for_each_entry(data, &event_file->triggers, list) { 4772 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 4773 hist_trigger_show(m, data, n++); 4774 } 4775 4776 out_unlock: 4777 mutex_unlock(&event_mutex); 4778 4779 return ret; 4780 } 4781 4782 static int event_hist_open(struct inode *inode, struct file *file) 4783 { 4784 int ret; 4785 4786 ret = security_locked_down(LOCKDOWN_TRACEFS); 4787 if (ret) 4788 return ret; 4789 4790 return single_open(file, hist_show, file); 4791 } 4792 4793 const struct file_operations event_hist_fops = { 4794 .open = event_hist_open, 4795 .read = seq_read, 4796 .llseek = seq_lseek, 4797 .release = single_release, 4798 }; 4799 4800 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 4801 static void hist_field_debug_show_flags(struct seq_file *m, 4802 unsigned long flags) 4803 { 4804 seq_puts(m, " flags:\n"); 4805 4806 if (flags & HIST_FIELD_FL_KEY) 4807 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 4808 else if (flags & HIST_FIELD_FL_HITCOUNT) 4809 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 4810 else if (flags & HIST_FIELD_FL_VAR) 4811 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 4812 else if (flags & HIST_FIELD_FL_VAR_REF) 4813 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 4814 else 4815 seq_puts(m, " VAL: normal u64 value\n"); 4816 4817 if (flags & HIST_FIELD_FL_ALIAS) 4818 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 4819 } 4820 4821 static int hist_field_debug_show(struct seq_file *m, 4822 struct hist_field *field, unsigned long flags) 4823 { 4824 if ((field->flags & flags) != flags) { 4825 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 4826 return -EINVAL; 4827 } 4828 4829 hist_field_debug_show_flags(m, field->flags); 4830 if (field->field) 4831 seq_printf(m, " ftrace_event_field name: %s\n", 4832 field->field->name); 4833 4834 if (field->flags & HIST_FIELD_FL_VAR) { 4835 seq_printf(m, " var.name: %s\n", field->var.name); 4836 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4837 field->var.idx); 4838 } 4839 4840 if (field->flags & HIST_FIELD_FL_ALIAS) 4841 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 4842 field->var_ref_idx); 4843 4844 if (field->flags & HIST_FIELD_FL_VAR_REF) { 4845 seq_printf(m, " name: %s\n", field->name); 4846 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4847 field->var.idx); 4848 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 4849 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 4850 field->var_ref_idx); 4851 if (field->system) 4852 seq_printf(m, " system: %s\n", field->system); 4853 if (field->event_name) 4854 seq_printf(m, " event_name: %s\n", field->event_name); 4855 } 4856 4857 seq_printf(m, " type: %s\n", field->type); 4858 seq_printf(m, " size: %u\n", field->size); 4859 seq_printf(m, " is_signed: %u\n", field->is_signed); 4860 4861 return 0; 4862 } 4863 4864 static int field_var_debug_show(struct seq_file *m, 4865 struct field_var *field_var, unsigned int i, 4866 bool save_vars) 4867 { 4868 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 4869 struct hist_field *field; 4870 int ret = 0; 4871 4872 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 4873 4874 field = field_var->var; 4875 4876 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 4877 4878 hist_field_debug_show_flags(m, field->flags); 4879 seq_printf(m, " var.name: %s\n", field->var.name); 4880 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 4881 field->var.idx); 4882 4883 field = field_var->val; 4884 4885 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 4886 if (field->field) 4887 seq_printf(m, " ftrace_event_field name: %s\n", 4888 field->field->name); 4889 else { 4890 ret = -EINVAL; 4891 goto out; 4892 } 4893 4894 seq_printf(m, " type: %s\n", field->type); 4895 seq_printf(m, " size: %u\n", field->size); 4896 seq_printf(m, " is_signed: %u\n", field->is_signed); 4897 out: 4898 return ret; 4899 } 4900 4901 static int hist_action_debug_show(struct seq_file *m, 4902 struct action_data *data, int i) 4903 { 4904 int ret = 0; 4905 4906 if (data->handler == HANDLER_ONMAX || 4907 data->handler == HANDLER_ONCHANGE) { 4908 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 4909 ret = hist_field_debug_show(m, data->track_data.var_ref, 4910 HIST_FIELD_FL_VAR_REF); 4911 if (ret) 4912 goto out; 4913 4914 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 4915 ret = hist_field_debug_show(m, data->track_data.track_var, 4916 HIST_FIELD_FL_VAR); 4917 if (ret) 4918 goto out; 4919 } 4920 4921 if (data->handler == HANDLER_ONMATCH) { 4922 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 4923 i, data->match_data.event_system); 4924 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 4925 i, data->match_data.event); 4926 } 4927 out: 4928 return ret; 4929 } 4930 4931 static int hist_actions_debug_show(struct seq_file *m, 4932 struct hist_trigger_data *hist_data) 4933 { 4934 int i, ret = 0; 4935 4936 if (hist_data->n_actions) 4937 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 4938 4939 for (i = 0; i < hist_data->n_actions; i++) { 4940 struct action_data *action = hist_data->actions[i]; 4941 4942 ret = hist_action_debug_show(m, action, i); 4943 if (ret) 4944 goto out; 4945 } 4946 4947 if (hist_data->n_save_vars) 4948 seq_puts(m, "\n save action variables (save() params):\n"); 4949 4950 for (i = 0; i < hist_data->n_save_vars; i++) { 4951 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 4952 if (ret) 4953 goto out; 4954 } 4955 out: 4956 return ret; 4957 } 4958 4959 static void hist_trigger_debug_show(struct seq_file *m, 4960 struct event_trigger_data *data, int n) 4961 { 4962 struct hist_trigger_data *hist_data; 4963 int i, ret; 4964 4965 if (n > 0) 4966 seq_puts(m, "\n\n"); 4967 4968 seq_puts(m, "# event histogram\n#\n# trigger info: "); 4969 data->ops->print(m, data->ops, data); 4970 seq_puts(m, "#\n\n"); 4971 4972 hist_data = data->private_data; 4973 4974 seq_printf(m, "hist_data: %p\n\n", hist_data); 4975 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 4976 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 4977 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 4978 4979 seq_puts(m, "\n val fields:\n\n"); 4980 4981 seq_puts(m, " hist_data->fields[0]:\n"); 4982 ret = hist_field_debug_show(m, hist_data->fields[0], 4983 HIST_FIELD_FL_HITCOUNT); 4984 if (ret) 4985 return; 4986 4987 for (i = 1; i < hist_data->n_vals; i++) { 4988 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 4989 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 4990 if (ret) 4991 return; 4992 } 4993 4994 seq_puts(m, "\n key fields:\n"); 4995 4996 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 4997 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 4998 ret = hist_field_debug_show(m, hist_data->fields[i], 4999 HIST_FIELD_FL_KEY); 5000 if (ret) 5001 return; 5002 } 5003 5004 if (hist_data->n_var_refs) 5005 seq_puts(m, "\n variable reference fields:\n"); 5006 5007 for (i = 0; i < hist_data->n_var_refs; i++) { 5008 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 5009 ret = hist_field_debug_show(m, hist_data->var_refs[i], 5010 HIST_FIELD_FL_VAR_REF); 5011 if (ret) 5012 return; 5013 } 5014 5015 if (hist_data->n_field_vars) 5016 seq_puts(m, "\n field variables:\n"); 5017 5018 for (i = 0; i < hist_data->n_field_vars; i++) { 5019 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 5020 if (ret) 5021 return; 5022 } 5023 5024 ret = hist_actions_debug_show(m, hist_data); 5025 if (ret) 5026 return; 5027 } 5028 5029 static int hist_debug_show(struct seq_file *m, void *v) 5030 { 5031 struct event_trigger_data *data; 5032 struct trace_event_file *event_file; 5033 int n = 0, ret = 0; 5034 5035 mutex_lock(&event_mutex); 5036 5037 event_file = event_file_data(m->private); 5038 if (unlikely(!event_file)) { 5039 ret = -ENODEV; 5040 goto out_unlock; 5041 } 5042 5043 list_for_each_entry(data, &event_file->triggers, list) { 5044 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5045 hist_trigger_debug_show(m, data, n++); 5046 } 5047 5048 out_unlock: 5049 mutex_unlock(&event_mutex); 5050 5051 return ret; 5052 } 5053 5054 static int event_hist_debug_open(struct inode *inode, struct file *file) 5055 { 5056 int ret; 5057 5058 ret = security_locked_down(LOCKDOWN_TRACEFS); 5059 if (ret) 5060 return ret; 5061 5062 return single_open(file, hist_debug_show, file); 5063 } 5064 5065 const struct file_operations event_hist_debug_fops = { 5066 .open = event_hist_debug_open, 5067 .read = seq_read, 5068 .llseek = seq_lseek, 5069 .release = single_release, 5070 }; 5071 #endif 5072 5073 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 5074 { 5075 const char *field_name = hist_field_name(hist_field, 0); 5076 5077 if (hist_field->var.name) 5078 seq_printf(m, "%s=", hist_field->var.name); 5079 5080 if (hist_field->flags & HIST_FIELD_FL_CPU) 5081 seq_puts(m, "cpu"); 5082 else if (field_name) { 5083 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 5084 hist_field->flags & HIST_FIELD_FL_ALIAS) 5085 seq_putc(m, '$'); 5086 seq_printf(m, "%s", field_name); 5087 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 5088 seq_puts(m, "common_timestamp"); 5089 5090 if (hist_field->flags) { 5091 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 5092 !(hist_field->flags & HIST_FIELD_FL_EXPR)) { 5093 const char *flags = get_hist_field_flags(hist_field); 5094 5095 if (flags) 5096 seq_printf(m, ".%s", flags); 5097 } 5098 } 5099 } 5100 5101 static int event_hist_trigger_print(struct seq_file *m, 5102 struct event_trigger_ops *ops, 5103 struct event_trigger_data *data) 5104 { 5105 struct hist_trigger_data *hist_data = data->private_data; 5106 struct hist_field *field; 5107 bool have_var = false; 5108 unsigned int i; 5109 5110 seq_puts(m, "hist:"); 5111 5112 if (data->name) 5113 seq_printf(m, "%s:", data->name); 5114 5115 seq_puts(m, "keys="); 5116 5117 for_each_hist_key_field(i, hist_data) { 5118 field = hist_data->fields[i]; 5119 5120 if (i > hist_data->n_vals) 5121 seq_puts(m, ","); 5122 5123 if (field->flags & HIST_FIELD_FL_STACKTRACE) 5124 seq_puts(m, "stacktrace"); 5125 else 5126 hist_field_print(m, field); 5127 } 5128 5129 seq_puts(m, ":vals="); 5130 5131 for_each_hist_val_field(i, hist_data) { 5132 field = hist_data->fields[i]; 5133 if (field->flags & HIST_FIELD_FL_VAR) { 5134 have_var = true; 5135 continue; 5136 } 5137 5138 if (i == HITCOUNT_IDX) 5139 seq_puts(m, "hitcount"); 5140 else { 5141 seq_puts(m, ","); 5142 hist_field_print(m, field); 5143 } 5144 } 5145 5146 if (have_var) { 5147 unsigned int n = 0; 5148 5149 seq_puts(m, ":"); 5150 5151 for_each_hist_val_field(i, hist_data) { 5152 field = hist_data->fields[i]; 5153 5154 if (field->flags & HIST_FIELD_FL_VAR) { 5155 if (n++) 5156 seq_puts(m, ","); 5157 hist_field_print(m, field); 5158 } 5159 } 5160 } 5161 5162 seq_puts(m, ":sort="); 5163 5164 for (i = 0; i < hist_data->n_sort_keys; i++) { 5165 struct tracing_map_sort_key *sort_key; 5166 unsigned int idx, first_key_idx; 5167 5168 /* skip VAR vals */ 5169 first_key_idx = hist_data->n_vals - hist_data->n_vars; 5170 5171 sort_key = &hist_data->sort_keys[i]; 5172 idx = sort_key->field_idx; 5173 5174 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 5175 return -EINVAL; 5176 5177 if (i > 0) 5178 seq_puts(m, ","); 5179 5180 if (idx == HITCOUNT_IDX) 5181 seq_puts(m, "hitcount"); 5182 else { 5183 if (idx >= first_key_idx) 5184 idx += hist_data->n_vars; 5185 hist_field_print(m, hist_data->fields[idx]); 5186 } 5187 5188 if (sort_key->descending) 5189 seq_puts(m, ".descending"); 5190 } 5191 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 5192 if (hist_data->enable_timestamps) 5193 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 5194 5195 print_actions_spec(m, hist_data); 5196 5197 if (data->filter_str) 5198 seq_printf(m, " if %s", data->filter_str); 5199 5200 if (data->paused) 5201 seq_puts(m, " [paused]"); 5202 else 5203 seq_puts(m, " [active]"); 5204 5205 seq_putc(m, '\n'); 5206 5207 return 0; 5208 } 5209 5210 static int event_hist_trigger_init(struct event_trigger_ops *ops, 5211 struct event_trigger_data *data) 5212 { 5213 struct hist_trigger_data *hist_data = data->private_data; 5214 5215 if (!data->ref && hist_data->attrs->name) 5216 save_named_trigger(hist_data->attrs->name, data); 5217 5218 data->ref++; 5219 5220 return 0; 5221 } 5222 5223 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 5224 { 5225 struct trace_event_file *file; 5226 unsigned int i; 5227 char *cmd; 5228 int ret; 5229 5230 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5231 file = hist_data->field_var_hists[i]->hist_data->event_file; 5232 cmd = hist_data->field_var_hists[i]->cmd; 5233 ret = event_hist_trigger_func(&trigger_hist_cmd, file, 5234 "!hist", "hist", cmd); 5235 WARN_ON_ONCE(ret < 0); 5236 } 5237 } 5238 5239 static void event_hist_trigger_free(struct event_trigger_ops *ops, 5240 struct event_trigger_data *data) 5241 { 5242 struct hist_trigger_data *hist_data = data->private_data; 5243 5244 if (WARN_ON_ONCE(data->ref <= 0)) 5245 return; 5246 5247 data->ref--; 5248 if (!data->ref) { 5249 if (data->name) 5250 del_named_trigger(data); 5251 5252 trigger_data_free(data); 5253 5254 remove_hist_vars(hist_data); 5255 5256 unregister_field_var_hists(hist_data); 5257 5258 destroy_hist_data(hist_data); 5259 } 5260 } 5261 5262 static struct event_trigger_ops event_hist_trigger_ops = { 5263 .func = event_hist_trigger, 5264 .print = event_hist_trigger_print, 5265 .init = event_hist_trigger_init, 5266 .free = event_hist_trigger_free, 5267 }; 5268 5269 static int event_hist_trigger_named_init(struct event_trigger_ops *ops, 5270 struct event_trigger_data *data) 5271 { 5272 data->ref++; 5273 5274 save_named_trigger(data->named_data->name, data); 5275 5276 event_hist_trigger_init(ops, data->named_data); 5277 5278 return 0; 5279 } 5280 5281 static void event_hist_trigger_named_free(struct event_trigger_ops *ops, 5282 struct event_trigger_data *data) 5283 { 5284 if (WARN_ON_ONCE(data->ref <= 0)) 5285 return; 5286 5287 event_hist_trigger_free(ops, data->named_data); 5288 5289 data->ref--; 5290 if (!data->ref) { 5291 del_named_trigger(data); 5292 trigger_data_free(data); 5293 } 5294 } 5295 5296 static struct event_trigger_ops event_hist_trigger_named_ops = { 5297 .func = event_hist_trigger, 5298 .print = event_hist_trigger_print, 5299 .init = event_hist_trigger_named_init, 5300 .free = event_hist_trigger_named_free, 5301 }; 5302 5303 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, 5304 char *param) 5305 { 5306 return &event_hist_trigger_ops; 5307 } 5308 5309 static void hist_clear(struct event_trigger_data *data) 5310 { 5311 struct hist_trigger_data *hist_data = data->private_data; 5312 5313 if (data->name) 5314 pause_named_trigger(data); 5315 5316 tracepoint_synchronize_unregister(); 5317 5318 tracing_map_clear(hist_data->map); 5319 5320 if (data->name) 5321 unpause_named_trigger(data); 5322 } 5323 5324 static bool compatible_field(struct ftrace_event_field *field, 5325 struct ftrace_event_field *test_field) 5326 { 5327 if (field == test_field) 5328 return true; 5329 if (field == NULL || test_field == NULL) 5330 return false; 5331 if (strcmp(field->name, test_field->name) != 0) 5332 return false; 5333 if (strcmp(field->type, test_field->type) != 0) 5334 return false; 5335 if (field->size != test_field->size) 5336 return false; 5337 if (field->is_signed != test_field->is_signed) 5338 return false; 5339 5340 return true; 5341 } 5342 5343 static bool hist_trigger_match(struct event_trigger_data *data, 5344 struct event_trigger_data *data_test, 5345 struct event_trigger_data *named_data, 5346 bool ignore_filter) 5347 { 5348 struct tracing_map_sort_key *sort_key, *sort_key_test; 5349 struct hist_trigger_data *hist_data, *hist_data_test; 5350 struct hist_field *key_field, *key_field_test; 5351 unsigned int i; 5352 5353 if (named_data && (named_data != data_test) && 5354 (named_data != data_test->named_data)) 5355 return false; 5356 5357 if (!named_data && is_named_trigger(data_test)) 5358 return false; 5359 5360 hist_data = data->private_data; 5361 hist_data_test = data_test->private_data; 5362 5363 if (hist_data->n_vals != hist_data_test->n_vals || 5364 hist_data->n_fields != hist_data_test->n_fields || 5365 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 5366 return false; 5367 5368 if (!ignore_filter) { 5369 if ((data->filter_str && !data_test->filter_str) || 5370 (!data->filter_str && data_test->filter_str)) 5371 return false; 5372 } 5373 5374 for_each_hist_field(i, hist_data) { 5375 key_field = hist_data->fields[i]; 5376 key_field_test = hist_data_test->fields[i]; 5377 5378 if (key_field->flags != key_field_test->flags) 5379 return false; 5380 if (!compatible_field(key_field->field, key_field_test->field)) 5381 return false; 5382 if (key_field->offset != key_field_test->offset) 5383 return false; 5384 if (key_field->size != key_field_test->size) 5385 return false; 5386 if (key_field->is_signed != key_field_test->is_signed) 5387 return false; 5388 if (!!key_field->var.name != !!key_field_test->var.name) 5389 return false; 5390 if (key_field->var.name && 5391 strcmp(key_field->var.name, key_field_test->var.name) != 0) 5392 return false; 5393 } 5394 5395 for (i = 0; i < hist_data->n_sort_keys; i++) { 5396 sort_key = &hist_data->sort_keys[i]; 5397 sort_key_test = &hist_data_test->sort_keys[i]; 5398 5399 if (sort_key->field_idx != sort_key_test->field_idx || 5400 sort_key->descending != sort_key_test->descending) 5401 return false; 5402 } 5403 5404 if (!ignore_filter && data->filter_str && 5405 (strcmp(data->filter_str, data_test->filter_str) != 0)) 5406 return false; 5407 5408 if (!actions_match(hist_data, hist_data_test)) 5409 return false; 5410 5411 return true; 5412 } 5413 5414 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops, 5415 struct event_trigger_data *data, 5416 struct trace_event_file *file) 5417 { 5418 struct hist_trigger_data *hist_data = data->private_data; 5419 struct event_trigger_data *test, *named_data = NULL; 5420 struct trace_array *tr = file->tr; 5421 int ret = 0; 5422 5423 if (hist_data->attrs->name) { 5424 named_data = find_named_trigger(hist_data->attrs->name); 5425 if (named_data) { 5426 if (!hist_trigger_match(data, named_data, named_data, 5427 true)) { 5428 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 5429 ret = -EINVAL; 5430 goto out; 5431 } 5432 } 5433 } 5434 5435 if (hist_data->attrs->name && !named_data) 5436 goto new; 5437 5438 lockdep_assert_held(&event_mutex); 5439 5440 list_for_each_entry(test, &file->triggers, list) { 5441 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5442 if (!hist_trigger_match(data, test, named_data, false)) 5443 continue; 5444 if (hist_data->attrs->pause) 5445 test->paused = true; 5446 else if (hist_data->attrs->cont) 5447 test->paused = false; 5448 else if (hist_data->attrs->clear) 5449 hist_clear(test); 5450 else { 5451 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 5452 ret = -EEXIST; 5453 } 5454 goto out; 5455 } 5456 } 5457 new: 5458 if (hist_data->attrs->cont || hist_data->attrs->clear) { 5459 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 5460 ret = -ENOENT; 5461 goto out; 5462 } 5463 5464 if (hist_data->attrs->pause) 5465 data->paused = true; 5466 5467 if (named_data) { 5468 data->private_data = named_data->private_data; 5469 set_named_trigger_data(data, named_data); 5470 data->ops = &event_hist_trigger_named_ops; 5471 } 5472 5473 if (data->ops->init) { 5474 ret = data->ops->init(data->ops, data); 5475 if (ret < 0) 5476 goto out; 5477 } 5478 5479 if (hist_data->enable_timestamps) { 5480 char *clock = hist_data->attrs->clock; 5481 5482 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 5483 if (ret) { 5484 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 5485 goto out; 5486 } 5487 5488 tracing_set_filter_buffering(file->tr, true); 5489 } 5490 5491 if (named_data) 5492 destroy_hist_data(hist_data); 5493 5494 ret++; 5495 out: 5496 return ret; 5497 } 5498 5499 static int hist_trigger_enable(struct event_trigger_data *data, 5500 struct trace_event_file *file) 5501 { 5502 int ret = 0; 5503 5504 list_add_tail_rcu(&data->list, &file->triggers); 5505 5506 update_cond_flag(file); 5507 5508 if (trace_event_trigger_enable_disable(file, 1) < 0) { 5509 list_del_rcu(&data->list); 5510 update_cond_flag(file); 5511 ret--; 5512 } 5513 5514 return ret; 5515 } 5516 5517 static bool have_hist_trigger_match(struct event_trigger_data *data, 5518 struct trace_event_file *file) 5519 { 5520 struct hist_trigger_data *hist_data = data->private_data; 5521 struct event_trigger_data *test, *named_data = NULL; 5522 bool match = false; 5523 5524 lockdep_assert_held(&event_mutex); 5525 5526 if (hist_data->attrs->name) 5527 named_data = find_named_trigger(hist_data->attrs->name); 5528 5529 list_for_each_entry(test, &file->triggers, list) { 5530 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5531 if (hist_trigger_match(data, test, named_data, false)) { 5532 match = true; 5533 break; 5534 } 5535 } 5536 } 5537 5538 return match; 5539 } 5540 5541 static bool hist_trigger_check_refs(struct event_trigger_data *data, 5542 struct trace_event_file *file) 5543 { 5544 struct hist_trigger_data *hist_data = data->private_data; 5545 struct event_trigger_data *test, *named_data = NULL; 5546 5547 lockdep_assert_held(&event_mutex); 5548 5549 if (hist_data->attrs->name) 5550 named_data = find_named_trigger(hist_data->attrs->name); 5551 5552 list_for_each_entry(test, &file->triggers, list) { 5553 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5554 if (!hist_trigger_match(data, test, named_data, false)) 5555 continue; 5556 hist_data = test->private_data; 5557 if (check_var_refs(hist_data)) 5558 return true; 5559 break; 5560 } 5561 } 5562 5563 return false; 5564 } 5565 5566 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops, 5567 struct event_trigger_data *data, 5568 struct trace_event_file *file) 5569 { 5570 struct hist_trigger_data *hist_data = data->private_data; 5571 struct event_trigger_data *test, *named_data = NULL; 5572 bool unregistered = false; 5573 5574 lockdep_assert_held(&event_mutex); 5575 5576 if (hist_data->attrs->name) 5577 named_data = find_named_trigger(hist_data->attrs->name); 5578 5579 list_for_each_entry(test, &file->triggers, list) { 5580 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5581 if (!hist_trigger_match(data, test, named_data, false)) 5582 continue; 5583 unregistered = true; 5584 list_del_rcu(&test->list); 5585 trace_event_trigger_enable_disable(file, 0); 5586 update_cond_flag(file); 5587 break; 5588 } 5589 } 5590 5591 if (unregistered && test->ops->free) 5592 test->ops->free(test->ops, test); 5593 5594 if (hist_data->enable_timestamps) { 5595 if (!hist_data->remove || unregistered) 5596 tracing_set_filter_buffering(file->tr, false); 5597 } 5598 } 5599 5600 static bool hist_file_check_refs(struct trace_event_file *file) 5601 { 5602 struct hist_trigger_data *hist_data; 5603 struct event_trigger_data *test; 5604 5605 lockdep_assert_held(&event_mutex); 5606 5607 list_for_each_entry(test, &file->triggers, list) { 5608 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5609 hist_data = test->private_data; 5610 if (check_var_refs(hist_data)) 5611 return true; 5612 } 5613 } 5614 5615 return false; 5616 } 5617 5618 static void hist_unreg_all(struct trace_event_file *file) 5619 { 5620 struct event_trigger_data *test, *n; 5621 struct hist_trigger_data *hist_data; 5622 struct synth_event *se; 5623 const char *se_name; 5624 5625 lockdep_assert_held(&event_mutex); 5626 5627 if (hist_file_check_refs(file)) 5628 return; 5629 5630 list_for_each_entry_safe(test, n, &file->triggers, list) { 5631 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5632 hist_data = test->private_data; 5633 list_del_rcu(&test->list); 5634 trace_event_trigger_enable_disable(file, 0); 5635 5636 se_name = trace_event_name(file->event_call); 5637 se = find_synth_event(se_name); 5638 if (se) 5639 se->ref--; 5640 5641 update_cond_flag(file); 5642 if (hist_data->enable_timestamps) 5643 tracing_set_filter_buffering(file->tr, false); 5644 if (test->ops->free) 5645 test->ops->free(test->ops, test); 5646 } 5647 } 5648 } 5649 5650 static int event_hist_trigger_func(struct event_command *cmd_ops, 5651 struct trace_event_file *file, 5652 char *glob, char *cmd, char *param) 5653 { 5654 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 5655 struct event_trigger_data *trigger_data; 5656 struct hist_trigger_attrs *attrs; 5657 struct event_trigger_ops *trigger_ops; 5658 struct hist_trigger_data *hist_data; 5659 struct synth_event *se; 5660 const char *se_name; 5661 bool remove = false; 5662 char *trigger, *p; 5663 int ret = 0; 5664 5665 lockdep_assert_held(&event_mutex); 5666 5667 if (glob && strlen(glob)) { 5668 hist_err_clear(); 5669 last_cmd_set(file, param); 5670 } 5671 5672 if (!param) 5673 return -EINVAL; 5674 5675 if (glob[0] == '!') 5676 remove = true; 5677 5678 /* 5679 * separate the trigger from the filter (k:v [if filter]) 5680 * allowing for whitespace in the trigger 5681 */ 5682 p = trigger = param; 5683 do { 5684 p = strstr(p, "if"); 5685 if (!p) 5686 break; 5687 if (p == param) 5688 return -EINVAL; 5689 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 5690 p++; 5691 continue; 5692 } 5693 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1) 5694 return -EINVAL; 5695 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 5696 p++; 5697 continue; 5698 } 5699 break; 5700 } while (p); 5701 5702 if (!p) 5703 param = NULL; 5704 else { 5705 *(p - 1) = '\0'; 5706 param = strstrip(p); 5707 trigger = strstrip(trigger); 5708 } 5709 5710 attrs = parse_hist_trigger_attrs(file->tr, trigger); 5711 if (IS_ERR(attrs)) 5712 return PTR_ERR(attrs); 5713 5714 if (attrs->map_bits) 5715 hist_trigger_bits = attrs->map_bits; 5716 5717 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 5718 if (IS_ERR(hist_data)) { 5719 destroy_hist_trigger_attrs(attrs); 5720 return PTR_ERR(hist_data); 5721 } 5722 5723 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger); 5724 5725 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); 5726 if (!trigger_data) { 5727 ret = -ENOMEM; 5728 goto out_free; 5729 } 5730 5731 trigger_data->count = -1; 5732 trigger_data->ops = trigger_ops; 5733 trigger_data->cmd_ops = cmd_ops; 5734 5735 INIT_LIST_HEAD(&trigger_data->list); 5736 RCU_INIT_POINTER(trigger_data->filter, NULL); 5737 5738 trigger_data->private_data = hist_data; 5739 5740 /* if param is non-empty, it's supposed to be a filter */ 5741 if (param && cmd_ops->set_filter) { 5742 ret = cmd_ops->set_filter(param, trigger_data, file); 5743 if (ret < 0) 5744 goto out_free; 5745 } 5746 5747 if (remove) { 5748 if (!have_hist_trigger_match(trigger_data, file)) 5749 goto out_free; 5750 5751 if (hist_trigger_check_refs(trigger_data, file)) { 5752 ret = -EBUSY; 5753 goto out_free; 5754 } 5755 5756 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5757 se_name = trace_event_name(file->event_call); 5758 se = find_synth_event(se_name); 5759 if (se) 5760 se->ref--; 5761 ret = 0; 5762 goto out_free; 5763 } 5764 5765 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file); 5766 /* 5767 * The above returns on success the # of triggers registered, 5768 * but if it didn't register any it returns zero. Consider no 5769 * triggers registered a failure too. 5770 */ 5771 if (!ret) { 5772 if (!(attrs->pause || attrs->cont || attrs->clear)) 5773 ret = -ENOENT; 5774 goto out_free; 5775 } else if (ret < 0) 5776 goto out_free; 5777 5778 if (get_named_trigger_data(trigger_data)) 5779 goto enable; 5780 5781 if (has_hist_vars(hist_data)) 5782 save_hist_vars(hist_data); 5783 5784 ret = create_actions(hist_data); 5785 if (ret) 5786 goto out_unreg; 5787 5788 ret = tracing_map_init(hist_data->map); 5789 if (ret) 5790 goto out_unreg; 5791 enable: 5792 ret = hist_trigger_enable(trigger_data, file); 5793 if (ret) 5794 goto out_unreg; 5795 5796 se_name = trace_event_name(file->event_call); 5797 se = find_synth_event(se_name); 5798 if (se) 5799 se->ref++; 5800 /* Just return zero, not the number of registered triggers */ 5801 ret = 0; 5802 out: 5803 if (ret == 0) 5804 hist_err_clear(); 5805 5806 return ret; 5807 out_unreg: 5808 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file); 5809 out_free: 5810 if (cmd_ops->set_filter) 5811 cmd_ops->set_filter(NULL, trigger_data, NULL); 5812 5813 remove_hist_vars(hist_data); 5814 5815 kfree(trigger_data); 5816 5817 destroy_hist_data(hist_data); 5818 goto out; 5819 } 5820 5821 static struct event_command trigger_hist_cmd = { 5822 .name = "hist", 5823 .trigger_type = ETT_EVENT_HIST, 5824 .flags = EVENT_CMD_FL_NEEDS_REC, 5825 .func = event_hist_trigger_func, 5826 .reg = hist_register_trigger, 5827 .unreg = hist_unregister_trigger, 5828 .unreg_all = hist_unreg_all, 5829 .get_trigger_ops = event_hist_get_trigger_ops, 5830 .set_filter = set_trigger_filter, 5831 }; 5832 5833 __init int register_trigger_hist_cmd(void) 5834 { 5835 int ret; 5836 5837 ret = register_event_command(&trigger_hist_cmd); 5838 WARN_ON(ret < 0); 5839 5840 return ret; 5841 } 5842 5843 static void 5844 hist_enable_trigger(struct event_trigger_data *data, 5845 struct trace_buffer *buffer, void *rec, 5846 struct ring_buffer_event *event) 5847 { 5848 struct enable_trigger_data *enable_data = data->private_data; 5849 struct event_trigger_data *test; 5850 5851 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 5852 lockdep_is_held(&event_mutex)) { 5853 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5854 if (enable_data->enable) 5855 test->paused = false; 5856 else 5857 test->paused = true; 5858 } 5859 } 5860 } 5861 5862 static void 5863 hist_enable_count_trigger(struct event_trigger_data *data, 5864 struct trace_buffer *buffer, void *rec, 5865 struct ring_buffer_event *event) 5866 { 5867 if (!data->count) 5868 return; 5869 5870 if (data->count != -1) 5871 (data->count)--; 5872 5873 hist_enable_trigger(data, buffer, rec, event); 5874 } 5875 5876 static struct event_trigger_ops hist_enable_trigger_ops = { 5877 .func = hist_enable_trigger, 5878 .print = event_enable_trigger_print, 5879 .init = event_trigger_init, 5880 .free = event_enable_trigger_free, 5881 }; 5882 5883 static struct event_trigger_ops hist_enable_count_trigger_ops = { 5884 .func = hist_enable_count_trigger, 5885 .print = event_enable_trigger_print, 5886 .init = event_trigger_init, 5887 .free = event_enable_trigger_free, 5888 }; 5889 5890 static struct event_trigger_ops hist_disable_trigger_ops = { 5891 .func = hist_enable_trigger, 5892 .print = event_enable_trigger_print, 5893 .init = event_trigger_init, 5894 .free = event_enable_trigger_free, 5895 }; 5896 5897 static struct event_trigger_ops hist_disable_count_trigger_ops = { 5898 .func = hist_enable_count_trigger, 5899 .print = event_enable_trigger_print, 5900 .init = event_trigger_init, 5901 .free = event_enable_trigger_free, 5902 }; 5903 5904 static struct event_trigger_ops * 5905 hist_enable_get_trigger_ops(char *cmd, char *param) 5906 { 5907 struct event_trigger_ops *ops; 5908 bool enable; 5909 5910 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); 5911 5912 if (enable) 5913 ops = param ? &hist_enable_count_trigger_ops : 5914 &hist_enable_trigger_ops; 5915 else 5916 ops = param ? &hist_disable_count_trigger_ops : 5917 &hist_disable_trigger_ops; 5918 5919 return ops; 5920 } 5921 5922 static void hist_enable_unreg_all(struct trace_event_file *file) 5923 { 5924 struct event_trigger_data *test, *n; 5925 5926 list_for_each_entry_safe(test, n, &file->triggers, list) { 5927 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 5928 list_del_rcu(&test->list); 5929 update_cond_flag(file); 5930 trace_event_trigger_enable_disable(file, 0); 5931 if (test->ops->free) 5932 test->ops->free(test->ops, test); 5933 } 5934 } 5935 } 5936 5937 static struct event_command trigger_hist_enable_cmd = { 5938 .name = ENABLE_HIST_STR, 5939 .trigger_type = ETT_HIST_ENABLE, 5940 .func = event_enable_trigger_func, 5941 .reg = event_enable_register_trigger, 5942 .unreg = event_enable_unregister_trigger, 5943 .unreg_all = hist_enable_unreg_all, 5944 .get_trigger_ops = hist_enable_get_trigger_ops, 5945 .set_filter = set_trigger_filter, 5946 }; 5947 5948 static struct event_command trigger_hist_disable_cmd = { 5949 .name = DISABLE_HIST_STR, 5950 .trigger_type = ETT_HIST_ENABLE, 5951 .func = event_enable_trigger_func, 5952 .reg = event_enable_register_trigger, 5953 .unreg = event_enable_unregister_trigger, 5954 .unreg_all = hist_enable_unreg_all, 5955 .get_trigger_ops = hist_enable_get_trigger_ops, 5956 .set_filter = set_trigger_filter, 5957 }; 5958 5959 static __init void unregister_trigger_hist_enable_disable_cmds(void) 5960 { 5961 unregister_event_command(&trigger_hist_enable_cmd); 5962 unregister_event_command(&trigger_hist_disable_cmd); 5963 } 5964 5965 __init int register_trigger_hist_enable_disable_cmds(void) 5966 { 5967 int ret; 5968 5969 ret = register_event_command(&trigger_hist_enable_cmd); 5970 if (WARN_ON(ret < 0)) 5971 return ret; 5972 ret = register_event_command(&trigger_hist_disable_cmd); 5973 if (WARN_ON(ret < 0)) 5974 unregister_trigger_hist_enable_disable_cmds(); 5975 5976 return ret; 5977 } 5978