1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/seq_buf.h> 12 #include <linux/mutex.h> 13 #include <linux/slab.h> 14 #include <linux/stacktrace.h> 15 #include <linux/rculist.h> 16 #include <linux/tracefs.h> 17 18 /* for gfp flag names */ 19 #include <linux/trace_events.h> 20 #include <trace/events/mmflags.h> 21 22 #include "tracing_map.h" 23 #include "trace_synth.h" 24 25 #define ERRORS \ 26 C(NONE, "No error"), \ 27 C(DUPLICATE_VAR, "Variable already defined"), \ 28 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 29 C(TOO_MANY_VARS, "Too many variables defined"), \ 30 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 31 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 32 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 33 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 34 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 35 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 36 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 37 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 38 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 39 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 40 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 41 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 42 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 43 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 44 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 45 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 46 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 47 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 48 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 49 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 50 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 51 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 52 C(TOO_MANY_PARAMS, "Too many action params"), \ 53 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 54 C(INVALID_PARAM, "Invalid action param"), \ 55 C(ACTION_NOT_FOUND, "No action found"), \ 56 C(NO_SAVE_PARAMS, "No params found for save()"), \ 57 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 58 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 59 C(NO_CLOSING_PAREN, "No closing paren found"), \ 60 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 61 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 62 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 63 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 64 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 65 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 66 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 67 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 68 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 69 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \ 70 C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \ 71 C(EXPECT_NUMBER, "Expecting numeric literal"), \ 72 C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \ 73 C(DIVISION_BY_ZERO, "Division by zero"), \ 74 C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"), 75 76 #undef C 77 #define C(a, b) HIST_ERR_##a 78 79 enum { ERRORS }; 80 81 #undef C 82 #define C(a, b) b 83 84 static const char *err_text[] = { ERRORS }; 85 86 struct hist_field; 87 88 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 89 struct tracing_map_elt *elt, 90 struct trace_buffer *buffer, 91 struct ring_buffer_event *rbe, 92 void *event); 93 94 #define HIST_FIELD_OPERANDS_MAX 2 95 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 96 #define HIST_ACTIONS_MAX 8 97 #define HIST_CONST_DIGITS_MAX 21 98 #define HIST_DIV_SHIFT 20 /* For optimizing division by constants */ 99 100 enum field_op_id { 101 FIELD_OP_NONE, 102 FIELD_OP_PLUS, 103 FIELD_OP_MINUS, 104 FIELD_OP_UNARY_MINUS, 105 FIELD_OP_DIV, 106 FIELD_OP_MULT, 107 }; 108 109 #define FIELD_FUNCS \ 110 C(NOP, "nop"), \ 111 C(VAR_REF, "var_ref"), \ 112 C(COUNTER, "counter"), \ 113 C(CONST, "const"), \ 114 C(LOG2, "log2"), \ 115 C(BUCKET, "bucket"), \ 116 C(TIMESTAMP, "timestamp"), \ 117 C(CPU, "cpu"), \ 118 C(COMM, "comm"), \ 119 C(STRING, "string"), \ 120 C(DYNSTRING, "dynstring"), \ 121 C(RELDYNSTRING, "reldynstring"), \ 122 C(PSTRING, "pstring"), \ 123 C(S64, "s64"), \ 124 C(U64, "u64"), \ 125 C(S32, "s32"), \ 126 C(U32, "u32"), \ 127 C(S16, "s16"), \ 128 C(U16, "u16"), \ 129 C(S8, "s8"), \ 130 C(U8, "u8"), \ 131 C(UMINUS, "uminus"), \ 132 C(MINUS, "minus"), \ 133 C(PLUS, "plus"), \ 134 C(DIV, "div"), \ 135 C(MULT, "mult"), \ 136 C(DIV_POWER2, "div_power2"), \ 137 C(DIV_NOT_POWER2, "div_not_power2"), \ 138 C(DIV_MULT_SHIFT, "div_mult_shift"), \ 139 C(EXECNAME, "execname"), \ 140 C(STACK, "stack"), 141 142 #undef C 143 #define C(a, b) HIST_FIELD_FN_##a 144 145 enum hist_field_fn { 146 FIELD_FUNCS 147 }; 148 149 /* 150 * A hist_var (histogram variable) contains variable information for 151 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 152 * flag set. A hist_var has a variable name e.g. ts0, and is 153 * associated with a given histogram trigger, as specified by 154 * hist_data. The hist_var idx is the unique index assigned to the 155 * variable by the hist trigger's tracing_map. The idx is what is 156 * used to set a variable's value and, by a variable reference, to 157 * retrieve it. 158 */ 159 struct hist_var { 160 char *name; 161 struct hist_trigger_data *hist_data; 162 unsigned int idx; 163 }; 164 165 struct hist_field { 166 struct ftrace_event_field *field; 167 unsigned long flags; 168 unsigned long buckets; 169 const char *type; 170 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 171 struct hist_trigger_data *hist_data; 172 enum hist_field_fn fn_num; 173 unsigned int ref; 174 unsigned int size; 175 unsigned int offset; 176 unsigned int is_signed; 177 178 /* 179 * Variable fields contain variable-specific info in var. 180 */ 181 struct hist_var var; 182 enum field_op_id operator; 183 char *system; 184 char *event_name; 185 186 /* 187 * The name field is used for EXPR and VAR_REF fields. VAR 188 * fields contain the variable name in var.name. 189 */ 190 char *name; 191 192 /* 193 * When a histogram trigger is hit, if it has any references 194 * to variables, the values of those variables are collected 195 * into a var_ref_vals array by resolve_var_refs(). The 196 * current value of each variable is read from the tracing_map 197 * using the hist field's hist_var.idx and entered into the 198 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 199 */ 200 unsigned int var_ref_idx; 201 bool read_once; 202 203 unsigned int var_str_idx; 204 205 /* Numeric literals are represented as u64 */ 206 u64 constant; 207 /* Used to optimize division by constants */ 208 u64 div_multiplier; 209 }; 210 211 static u64 hist_fn_call(struct hist_field *hist_field, 212 struct tracing_map_elt *elt, 213 struct trace_buffer *buffer, 214 struct ring_buffer_event *rbe, 215 void *event); 216 217 static u64 hist_field_const(struct hist_field *field, 218 struct tracing_map_elt *elt, 219 struct trace_buffer *buffer, 220 struct ring_buffer_event *rbe, 221 void *event) 222 { 223 return field->constant; 224 } 225 226 static u64 hist_field_counter(struct hist_field *field, 227 struct tracing_map_elt *elt, 228 struct trace_buffer *buffer, 229 struct ring_buffer_event *rbe, 230 void *event) 231 { 232 return 1; 233 } 234 235 static u64 hist_field_string(struct hist_field *hist_field, 236 struct tracing_map_elt *elt, 237 struct trace_buffer *buffer, 238 struct ring_buffer_event *rbe, 239 void *event) 240 { 241 char *addr = (char *)(event + hist_field->field->offset); 242 243 return (u64)(unsigned long)addr; 244 } 245 246 static u64 hist_field_dynstring(struct hist_field *hist_field, 247 struct tracing_map_elt *elt, 248 struct trace_buffer *buffer, 249 struct ring_buffer_event *rbe, 250 void *event) 251 { 252 u32 str_item = *(u32 *)(event + hist_field->field->offset); 253 int str_loc = str_item & 0xffff; 254 char *addr = (char *)(event + str_loc); 255 256 return (u64)(unsigned long)addr; 257 } 258 259 static u64 hist_field_reldynstring(struct hist_field *hist_field, 260 struct tracing_map_elt *elt, 261 struct trace_buffer *buffer, 262 struct ring_buffer_event *rbe, 263 void *event) 264 { 265 u32 *item = event + hist_field->field->offset; 266 u32 str_item = *item; 267 int str_loc = str_item & 0xffff; 268 char *addr = (char *)&item[1] + str_loc; 269 270 return (u64)(unsigned long)addr; 271 } 272 273 static u64 hist_field_pstring(struct hist_field *hist_field, 274 struct tracing_map_elt *elt, 275 struct trace_buffer *buffer, 276 struct ring_buffer_event *rbe, 277 void *event) 278 { 279 char **addr = (char **)(event + hist_field->field->offset); 280 281 return (u64)(unsigned long)*addr; 282 } 283 284 static u64 hist_field_log2(struct hist_field *hist_field, 285 struct tracing_map_elt *elt, 286 struct trace_buffer *buffer, 287 struct ring_buffer_event *rbe, 288 void *event) 289 { 290 struct hist_field *operand = hist_field->operands[0]; 291 292 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 293 294 return (u64) ilog2(roundup_pow_of_two(val)); 295 } 296 297 static u64 hist_field_bucket(struct hist_field *hist_field, 298 struct tracing_map_elt *elt, 299 struct trace_buffer *buffer, 300 struct ring_buffer_event *rbe, 301 void *event) 302 { 303 struct hist_field *operand = hist_field->operands[0]; 304 unsigned long buckets = hist_field->buckets; 305 306 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 307 308 if (WARN_ON_ONCE(!buckets)) 309 return val; 310 311 if (val >= LONG_MAX) 312 val = div64_ul(val, buckets); 313 else 314 val = (u64)((unsigned long)val / buckets); 315 return val * buckets; 316 } 317 318 static u64 hist_field_plus(struct hist_field *hist_field, 319 struct tracing_map_elt *elt, 320 struct trace_buffer *buffer, 321 struct ring_buffer_event *rbe, 322 void *event) 323 { 324 struct hist_field *operand1 = hist_field->operands[0]; 325 struct hist_field *operand2 = hist_field->operands[1]; 326 327 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 328 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 329 330 return val1 + val2; 331 } 332 333 static u64 hist_field_minus(struct hist_field *hist_field, 334 struct tracing_map_elt *elt, 335 struct trace_buffer *buffer, 336 struct ring_buffer_event *rbe, 337 void *event) 338 { 339 struct hist_field *operand1 = hist_field->operands[0]; 340 struct hist_field *operand2 = hist_field->operands[1]; 341 342 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 343 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 344 345 return val1 - val2; 346 } 347 348 static u64 hist_field_div(struct hist_field *hist_field, 349 struct tracing_map_elt *elt, 350 struct trace_buffer *buffer, 351 struct ring_buffer_event *rbe, 352 void *event) 353 { 354 struct hist_field *operand1 = hist_field->operands[0]; 355 struct hist_field *operand2 = hist_field->operands[1]; 356 357 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 358 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 359 360 /* Return -1 for the undefined case */ 361 if (!val2) 362 return -1; 363 364 /* Use shift if the divisor is a power of 2 */ 365 if (!(val2 & (val2 - 1))) 366 return val1 >> __ffs64(val2); 367 368 return div64_u64(val1, val2); 369 } 370 371 static u64 div_by_power_of_two(struct hist_field *hist_field, 372 struct tracing_map_elt *elt, 373 struct trace_buffer *buffer, 374 struct ring_buffer_event *rbe, 375 void *event) 376 { 377 struct hist_field *operand1 = hist_field->operands[0]; 378 struct hist_field *operand2 = hist_field->operands[1]; 379 380 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 381 382 return val1 >> __ffs64(operand2->constant); 383 } 384 385 static u64 div_by_not_power_of_two(struct hist_field *hist_field, 386 struct tracing_map_elt *elt, 387 struct trace_buffer *buffer, 388 struct ring_buffer_event *rbe, 389 void *event) 390 { 391 struct hist_field *operand1 = hist_field->operands[0]; 392 struct hist_field *operand2 = hist_field->operands[1]; 393 394 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 395 396 return div64_u64(val1, operand2->constant); 397 } 398 399 static u64 div_by_mult_and_shift(struct hist_field *hist_field, 400 struct tracing_map_elt *elt, 401 struct trace_buffer *buffer, 402 struct ring_buffer_event *rbe, 403 void *event) 404 { 405 struct hist_field *operand1 = hist_field->operands[0]; 406 struct hist_field *operand2 = hist_field->operands[1]; 407 408 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 409 410 /* 411 * If the divisor is a constant, do a multiplication and shift instead. 412 * 413 * Choose Z = some power of 2. If Y <= Z, then: 414 * X / Y = (X * (Z / Y)) / Z 415 * 416 * (Z / Y) is a constant (mult) which is calculated at parse time, so: 417 * X / Y = (X * mult) / Z 418 * 419 * The division by Z can be replaced by a shift since Z is a power of 2: 420 * X / Y = (X * mult) >> HIST_DIV_SHIFT 421 * 422 * As long, as X < Z the results will not be off by more than 1. 423 */ 424 if (val1 < (1 << HIST_DIV_SHIFT)) { 425 u64 mult = operand2->div_multiplier; 426 427 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT; 428 } 429 430 return div64_u64(val1, operand2->constant); 431 } 432 433 static u64 hist_field_mult(struct hist_field *hist_field, 434 struct tracing_map_elt *elt, 435 struct trace_buffer *buffer, 436 struct ring_buffer_event *rbe, 437 void *event) 438 { 439 struct hist_field *operand1 = hist_field->operands[0]; 440 struct hist_field *operand2 = hist_field->operands[1]; 441 442 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 443 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 444 445 return val1 * val2; 446 } 447 448 static u64 hist_field_unary_minus(struct hist_field *hist_field, 449 struct tracing_map_elt *elt, 450 struct trace_buffer *buffer, 451 struct ring_buffer_event *rbe, 452 void *event) 453 { 454 struct hist_field *operand = hist_field->operands[0]; 455 456 s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event); 457 u64 val = (u64)-sval; 458 459 return val; 460 } 461 462 #define DEFINE_HIST_FIELD_FN(type) \ 463 static u64 hist_field_##type(struct hist_field *hist_field, \ 464 struct tracing_map_elt *elt, \ 465 struct trace_buffer *buffer, \ 466 struct ring_buffer_event *rbe, \ 467 void *event) \ 468 { \ 469 type *addr = (type *)(event + hist_field->field->offset); \ 470 \ 471 return (u64)(unsigned long)*addr; \ 472 } 473 474 DEFINE_HIST_FIELD_FN(s64); 475 DEFINE_HIST_FIELD_FN(u64); 476 DEFINE_HIST_FIELD_FN(s32); 477 DEFINE_HIST_FIELD_FN(u32); 478 DEFINE_HIST_FIELD_FN(s16); 479 DEFINE_HIST_FIELD_FN(u16); 480 DEFINE_HIST_FIELD_FN(s8); 481 DEFINE_HIST_FIELD_FN(u8); 482 483 #define for_each_hist_field(i, hist_data) \ 484 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 485 486 #define for_each_hist_val_field(i, hist_data) \ 487 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 488 489 #define for_each_hist_key_field(i, hist_data) \ 490 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 491 492 #define HITCOUNT_IDX 0 493 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 494 495 enum hist_field_flags { 496 HIST_FIELD_FL_HITCOUNT = 1 << 0, 497 HIST_FIELD_FL_KEY = 1 << 1, 498 HIST_FIELD_FL_STRING = 1 << 2, 499 HIST_FIELD_FL_HEX = 1 << 3, 500 HIST_FIELD_FL_SYM = 1 << 4, 501 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 502 HIST_FIELD_FL_EXECNAME = 1 << 6, 503 HIST_FIELD_FL_SYSCALL = 1 << 7, 504 HIST_FIELD_FL_STACKTRACE = 1 << 8, 505 HIST_FIELD_FL_LOG2 = 1 << 9, 506 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 507 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 508 HIST_FIELD_FL_VAR = 1 << 12, 509 HIST_FIELD_FL_EXPR = 1 << 13, 510 HIST_FIELD_FL_VAR_REF = 1 << 14, 511 HIST_FIELD_FL_CPU = 1 << 15, 512 HIST_FIELD_FL_ALIAS = 1 << 16, 513 HIST_FIELD_FL_BUCKET = 1 << 17, 514 HIST_FIELD_FL_CONST = 1 << 18, 515 HIST_FIELD_FL_PERCENT = 1 << 19, 516 HIST_FIELD_FL_GRAPH = 1 << 20, 517 HIST_FIELD_FL_COMM = 1 << 21, 518 }; 519 520 struct var_defs { 521 unsigned int n_vars; 522 char *name[TRACING_MAP_VARS_MAX]; 523 char *expr[TRACING_MAP_VARS_MAX]; 524 }; 525 526 struct hist_trigger_attrs { 527 char *keys_str; 528 char *vals_str; 529 char *sort_key_str; 530 char *name; 531 char *clock; 532 bool pause; 533 bool cont; 534 bool clear; 535 bool ts_in_usecs; 536 bool no_hitcount; 537 unsigned int map_bits; 538 539 char *assignment_str[TRACING_MAP_VARS_MAX]; 540 unsigned int n_assignments; 541 542 char *action_str[HIST_ACTIONS_MAX]; 543 unsigned int n_actions; 544 545 struct var_defs var_defs; 546 }; 547 548 struct field_var { 549 struct hist_field *var; 550 struct hist_field *val; 551 }; 552 553 struct field_var_hist { 554 struct hist_trigger_data *hist_data; 555 char *cmd; 556 }; 557 558 struct hist_trigger_data { 559 struct hist_field *fields[HIST_FIELDS_MAX]; 560 unsigned int n_vals; 561 unsigned int n_keys; 562 unsigned int n_fields; 563 unsigned int n_vars; 564 unsigned int n_var_str; 565 unsigned int key_size; 566 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 567 unsigned int n_sort_keys; 568 struct trace_event_file *event_file; 569 struct hist_trigger_attrs *attrs; 570 struct tracing_map *map; 571 bool enable_timestamps; 572 bool remove; 573 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 574 unsigned int n_var_refs; 575 576 struct action_data *actions[HIST_ACTIONS_MAX]; 577 unsigned int n_actions; 578 579 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 580 unsigned int n_field_vars; 581 unsigned int n_field_var_str; 582 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 583 unsigned int n_field_var_hists; 584 585 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 586 unsigned int n_save_vars; 587 unsigned int n_save_var_str; 588 }; 589 590 struct action_data; 591 592 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 593 struct tracing_map_elt *elt, 594 struct trace_buffer *buffer, void *rec, 595 struct ring_buffer_event *rbe, void *key, 596 struct action_data *data, u64 *var_ref_vals); 597 598 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 599 600 enum handler_id { 601 HANDLER_ONMATCH = 1, 602 HANDLER_ONMAX, 603 HANDLER_ONCHANGE, 604 }; 605 606 enum action_id { 607 ACTION_SAVE = 1, 608 ACTION_TRACE, 609 ACTION_SNAPSHOT, 610 }; 611 612 struct action_data { 613 enum handler_id handler; 614 enum action_id action; 615 char *action_name; 616 action_fn_t fn; 617 618 unsigned int n_params; 619 char *params[SYNTH_FIELDS_MAX]; 620 621 /* 622 * When a histogram trigger is hit, the values of any 623 * references to variables, including variables being passed 624 * as parameters to synthetic events, are collected into a 625 * var_ref_vals array. This var_ref_idx array is an array of 626 * indices into the var_ref_vals array, one for each synthetic 627 * event param, and is passed to the synthetic event 628 * invocation. 629 */ 630 unsigned int var_ref_idx[SYNTH_FIELDS_MAX]; 631 struct synth_event *synth_event; 632 bool use_trace_keyword; 633 char *synth_event_name; 634 635 union { 636 struct { 637 char *event; 638 char *event_system; 639 } match_data; 640 641 struct { 642 /* 643 * var_str contains the $-unstripped variable 644 * name referenced by var_ref, and used when 645 * printing the action. Because var_ref 646 * creation is deferred to create_actions(), 647 * we need a per-action way to save it until 648 * then, thus var_str. 649 */ 650 char *var_str; 651 652 /* 653 * var_ref refers to the variable being 654 * tracked e.g onmax($var). 655 */ 656 struct hist_field *var_ref; 657 658 /* 659 * track_var contains the 'invisible' tracking 660 * variable created to keep the current 661 * e.g. max value. 662 */ 663 struct hist_field *track_var; 664 665 check_track_val_fn_t check_val; 666 action_fn_t save_data; 667 } track_data; 668 }; 669 }; 670 671 struct track_data { 672 u64 track_val; 673 bool updated; 674 675 unsigned int key_len; 676 void *key; 677 struct tracing_map_elt elt; 678 679 struct action_data *action_data; 680 struct hist_trigger_data *hist_data; 681 }; 682 683 struct hist_elt_data { 684 char *comm; 685 u64 *var_ref_vals; 686 int n_field_var_str; 687 char *field_var_str[] __counted_by(n_field_var_str); 688 }; 689 690 struct snapshot_context { 691 struct tracing_map_elt *elt; 692 void *key; 693 }; 694 695 /* 696 * Returns the specific division function to use if the divisor 697 * is constant. This avoids extra branches when the trigger is hit. 698 */ 699 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor) 700 { 701 u64 div = divisor->constant; 702 703 if (!(div & (div - 1))) 704 return HIST_FIELD_FN_DIV_POWER2; 705 706 /* If the divisor is too large, do a regular division */ 707 if (div > (1 << HIST_DIV_SHIFT)) 708 return HIST_FIELD_FN_DIV_NOT_POWER2; 709 710 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div); 711 return HIST_FIELD_FN_DIV_MULT_SHIFT; 712 } 713 714 static void track_data_free(struct track_data *track_data) 715 { 716 struct hist_elt_data *elt_data; 717 718 if (!track_data) 719 return; 720 721 kfree(track_data->key); 722 723 elt_data = track_data->elt.private_data; 724 if (elt_data) { 725 kfree(elt_data->comm); 726 kfree(elt_data); 727 } 728 729 kfree(track_data); 730 } 731 732 static struct track_data *track_data_alloc(unsigned int key_len, 733 struct action_data *action_data, 734 struct hist_trigger_data *hist_data) 735 { 736 struct track_data *data = kzalloc_obj(*data); 737 struct hist_elt_data *elt_data; 738 739 if (!data) 740 return ERR_PTR(-ENOMEM); 741 742 data->key = kzalloc(key_len, GFP_KERNEL); 743 if (!data->key) { 744 track_data_free(data); 745 return ERR_PTR(-ENOMEM); 746 } 747 748 data->key_len = key_len; 749 data->action_data = action_data; 750 data->hist_data = hist_data; 751 752 elt_data = kzalloc_obj(*elt_data); 753 if (!elt_data) { 754 track_data_free(data); 755 return ERR_PTR(-ENOMEM); 756 } 757 758 data->elt.private_data = elt_data; 759 760 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 761 if (!elt_data->comm) { 762 track_data_free(data); 763 return ERR_PTR(-ENOMEM); 764 } 765 766 return data; 767 } 768 769 #define HIST_PREFIX "hist:" 770 771 static char *last_cmd; 772 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 773 774 static int errpos(char *str) 775 { 776 if (!str || !last_cmd) 777 return 0; 778 779 return err_pos(last_cmd, str); 780 } 781 782 static void last_cmd_set(struct trace_event_file *file, char *str) 783 { 784 const char *system = NULL, *name = NULL; 785 struct trace_event_call *call; 786 787 if (!str) 788 return; 789 790 kfree(last_cmd); 791 792 last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str); 793 if (!last_cmd) 794 return; 795 796 if (file) { 797 call = file->event_call; 798 system = call->class->system; 799 if (system) { 800 name = trace_event_name(call); 801 if (!name) 802 system = NULL; 803 } 804 } 805 806 if (system) 807 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name); 808 } 809 810 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos) 811 { 812 if (!last_cmd) 813 return; 814 815 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 816 err_type, err_pos); 817 } 818 819 static void hist_err_clear(void) 820 { 821 if (last_cmd) 822 last_cmd[0] = '\0'; 823 last_cmd_loc[0] = '\0'; 824 } 825 826 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 827 unsigned int *var_ref_idx); 828 829 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 830 unsigned int *var_ref_idx) 831 { 832 struct tracepoint *tp = event->tp; 833 834 if (unlikely(static_key_enabled(&tp->key))) { 835 struct tracepoint_func *probe_func_ptr; 836 synth_probe_func_t probe_func; 837 void *__data; 838 839 if (!(cpu_online(raw_smp_processor_id()))) 840 return; 841 842 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 843 if (probe_func_ptr) { 844 do { 845 probe_func = probe_func_ptr->func; 846 __data = probe_func_ptr->data; 847 probe_func(__data, var_ref_vals, var_ref_idx); 848 } while ((++probe_func_ptr)->func); 849 } 850 } 851 } 852 853 static void action_trace(struct hist_trigger_data *hist_data, 854 struct tracing_map_elt *elt, 855 struct trace_buffer *buffer, void *rec, 856 struct ring_buffer_event *rbe, void *key, 857 struct action_data *data, u64 *var_ref_vals) 858 { 859 struct synth_event *event = data->synth_event; 860 861 trace_synth(event, var_ref_vals, data->var_ref_idx); 862 } 863 864 struct hist_var_data { 865 struct list_head list; 866 struct hist_trigger_data *hist_data; 867 }; 868 869 static u64 hist_field_timestamp(struct hist_field *hist_field, 870 struct tracing_map_elt *elt, 871 struct trace_buffer *buffer, 872 struct ring_buffer_event *rbe, 873 void *event) 874 { 875 struct hist_trigger_data *hist_data = hist_field->hist_data; 876 struct trace_array *tr = hist_data->event_file->tr; 877 878 u64 ts = ring_buffer_event_time_stamp(buffer, rbe); 879 880 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 881 ts = ns2usecs(ts); 882 883 return ts; 884 } 885 886 static u64 hist_field_cpu(struct hist_field *hist_field, 887 struct tracing_map_elt *elt, 888 struct trace_buffer *buffer, 889 struct ring_buffer_event *rbe, 890 void *event) 891 { 892 int cpu = smp_processor_id(); 893 894 return cpu; 895 } 896 897 static u64 hist_field_comm(struct hist_field *hist_field, 898 struct tracing_map_elt *elt, 899 struct trace_buffer *buffer, 900 struct ring_buffer_event *rbe, 901 void *event) 902 { 903 return (u64)(unsigned long)current->comm; 904 } 905 906 /** 907 * check_field_for_var_ref - Check if a VAR_REF field references a variable 908 * @hist_field: The VAR_REF field to check 909 * @var_data: The hist trigger that owns the variable 910 * @var_idx: The trigger variable identifier 911 * 912 * Check the given VAR_REF field to see whether or not it references 913 * the given variable associated with the given trigger. 914 * 915 * Return: The VAR_REF field if it does reference the variable, NULL if not 916 */ 917 static struct hist_field * 918 check_field_for_var_ref(struct hist_field *hist_field, 919 struct hist_trigger_data *var_data, 920 unsigned int var_idx) 921 { 922 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 923 924 if (hist_field && hist_field->var.idx == var_idx && 925 hist_field->var.hist_data == var_data) 926 return hist_field; 927 928 return NULL; 929 } 930 931 /** 932 * find_var_ref - Check if a trigger has a reference to a trigger variable 933 * @hist_data: The hist trigger that might have a reference to the variable 934 * @var_data: The hist trigger that owns the variable 935 * @var_idx: The trigger variable identifier 936 * 937 * Check the list of var_refs[] on the first hist trigger to see 938 * whether any of them are references to the variable on the second 939 * trigger. 940 * 941 * Return: The VAR_REF field referencing the variable if so, NULL if not 942 */ 943 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 944 struct hist_trigger_data *var_data, 945 unsigned int var_idx) 946 { 947 struct hist_field *hist_field; 948 unsigned int i; 949 950 for (i = 0; i < hist_data->n_var_refs; i++) { 951 hist_field = hist_data->var_refs[i]; 952 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 953 return hist_field; 954 } 955 956 return NULL; 957 } 958 959 /** 960 * find_any_var_ref - Check if there is a reference to a given trigger variable 961 * @hist_data: The hist trigger 962 * @var_idx: The trigger variable identifier 963 * 964 * Check to see whether the given variable is currently referenced by 965 * any other trigger. 966 * 967 * The trigger the variable is defined on is explicitly excluded - the 968 * assumption being that a self-reference doesn't prevent a trigger 969 * from being removed. 970 * 971 * Return: The VAR_REF field referencing the variable if so, NULL if not 972 */ 973 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 974 unsigned int var_idx) 975 { 976 struct trace_array *tr = hist_data->event_file->tr; 977 struct hist_field *found = NULL; 978 struct hist_var_data *var_data; 979 980 list_for_each_entry(var_data, &tr->hist_vars, list) { 981 if (var_data->hist_data == hist_data) 982 continue; 983 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 984 if (found) 985 break; 986 } 987 988 return found; 989 } 990 991 /** 992 * check_var_refs - Check if there is a reference to any of trigger's variables 993 * @hist_data: The hist trigger 994 * 995 * A trigger can define one or more variables. If any one of them is 996 * currently referenced by any other trigger, this function will 997 * determine that. 998 * 999 * Typically used to determine whether or not a trigger can be removed 1000 * - if there are any references to a trigger's variables, it cannot. 1001 * 1002 * Return: True if there is a reference to any of trigger's variables 1003 */ 1004 static bool check_var_refs(struct hist_trigger_data *hist_data) 1005 { 1006 struct hist_field *field; 1007 bool found = false; 1008 int i; 1009 1010 for_each_hist_field(i, hist_data) { 1011 field = hist_data->fields[i]; 1012 if (field && field->flags & HIST_FIELD_FL_VAR) { 1013 if (find_any_var_ref(hist_data, field->var.idx)) { 1014 found = true; 1015 break; 1016 } 1017 } 1018 } 1019 1020 return found; 1021 } 1022 1023 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1024 { 1025 struct trace_array *tr = hist_data->event_file->tr; 1026 struct hist_var_data *var_data, *found = NULL; 1027 1028 list_for_each_entry(var_data, &tr->hist_vars, list) { 1029 if (var_data->hist_data == hist_data) { 1030 found = var_data; 1031 break; 1032 } 1033 } 1034 1035 return found; 1036 } 1037 1038 static bool field_has_hist_vars(struct hist_field *hist_field, 1039 unsigned int level) 1040 { 1041 int i; 1042 1043 if (level > 3) 1044 return false; 1045 1046 if (!hist_field) 1047 return false; 1048 1049 if (hist_field->flags & HIST_FIELD_FL_VAR || 1050 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1051 return true; 1052 1053 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1054 struct hist_field *operand; 1055 1056 operand = hist_field->operands[i]; 1057 if (field_has_hist_vars(operand, level + 1)) 1058 return true; 1059 } 1060 1061 return false; 1062 } 1063 1064 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1065 { 1066 struct hist_field *hist_field; 1067 int i; 1068 1069 for_each_hist_field(i, hist_data) { 1070 hist_field = hist_data->fields[i]; 1071 if (field_has_hist_vars(hist_field, 0)) 1072 return true; 1073 } 1074 1075 return false; 1076 } 1077 1078 static int save_hist_vars(struct hist_trigger_data *hist_data) 1079 { 1080 struct trace_array *tr = hist_data->event_file->tr; 1081 struct hist_var_data *var_data; 1082 1083 var_data = find_hist_vars(hist_data); 1084 if (var_data) 1085 return 0; 1086 1087 if (tracing_check_open_get_tr(tr)) 1088 return -ENODEV; 1089 1090 var_data = kzalloc_obj(*var_data); 1091 if (!var_data) { 1092 trace_array_put(tr); 1093 return -ENOMEM; 1094 } 1095 1096 var_data->hist_data = hist_data; 1097 list_add(&var_data->list, &tr->hist_vars); 1098 1099 return 0; 1100 } 1101 1102 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1103 { 1104 struct trace_array *tr = hist_data->event_file->tr; 1105 struct hist_var_data *var_data; 1106 1107 var_data = find_hist_vars(hist_data); 1108 if (!var_data) 1109 return; 1110 1111 if (WARN_ON(check_var_refs(hist_data))) 1112 return; 1113 1114 list_del(&var_data->list); 1115 1116 kfree(var_data); 1117 1118 trace_array_put(tr); 1119 } 1120 1121 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1122 const char *var_name) 1123 { 1124 struct hist_field *hist_field, *found = NULL; 1125 int i; 1126 1127 for_each_hist_field(i, hist_data) { 1128 hist_field = hist_data->fields[i]; 1129 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1130 strcmp(hist_field->var.name, var_name) == 0) { 1131 found = hist_field; 1132 break; 1133 } 1134 } 1135 1136 return found; 1137 } 1138 1139 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1140 struct trace_event_file *file, 1141 const char *var_name) 1142 { 1143 struct hist_trigger_data *test_data; 1144 struct event_trigger_data *test; 1145 struct hist_field *hist_field; 1146 1147 lockdep_assert_held(&event_mutex); 1148 1149 hist_field = find_var_field(hist_data, var_name); 1150 if (hist_field) 1151 return hist_field; 1152 1153 list_for_each_entry(test, &file->triggers, list) { 1154 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1155 test_data = test->private_data; 1156 hist_field = find_var_field(test_data, var_name); 1157 if (hist_field) 1158 return hist_field; 1159 } 1160 } 1161 1162 return NULL; 1163 } 1164 1165 static struct trace_event_file *find_var_file(struct trace_array *tr, 1166 char *system, 1167 char *event_name, 1168 char *var_name) 1169 { 1170 struct hist_trigger_data *var_hist_data; 1171 struct hist_var_data *var_data; 1172 struct trace_event_file *file, *found = NULL; 1173 1174 if (system) 1175 return find_event_file(tr, system, event_name); 1176 1177 list_for_each_entry(var_data, &tr->hist_vars, list) { 1178 var_hist_data = var_data->hist_data; 1179 file = var_hist_data->event_file; 1180 if (file == found) 1181 continue; 1182 1183 if (find_var_field(var_hist_data, var_name)) { 1184 if (found) { 1185 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1186 return NULL; 1187 } 1188 1189 found = file; 1190 } 1191 } 1192 1193 return found; 1194 } 1195 1196 static struct hist_field *find_file_var(struct trace_event_file *file, 1197 const char *var_name) 1198 { 1199 struct hist_trigger_data *test_data; 1200 struct event_trigger_data *test; 1201 struct hist_field *hist_field; 1202 1203 lockdep_assert_held(&event_mutex); 1204 1205 list_for_each_entry(test, &file->triggers, list) { 1206 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1207 test_data = test->private_data; 1208 hist_field = find_var_field(test_data, var_name); 1209 if (hist_field) 1210 return hist_field; 1211 } 1212 } 1213 1214 return NULL; 1215 } 1216 1217 static struct hist_field * 1218 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1219 { 1220 struct trace_array *tr = hist_data->event_file->tr; 1221 struct hist_field *hist_field, *found = NULL; 1222 struct trace_event_file *file; 1223 unsigned int i; 1224 1225 for (i = 0; i < hist_data->n_actions; i++) { 1226 struct action_data *data = hist_data->actions[i]; 1227 1228 if (data->handler == HANDLER_ONMATCH) { 1229 char *system = data->match_data.event_system; 1230 char *event_name = data->match_data.event; 1231 1232 file = find_var_file(tr, system, event_name, var_name); 1233 if (!file) 1234 continue; 1235 hist_field = find_file_var(file, var_name); 1236 if (hist_field) { 1237 if (found) { 1238 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1239 errpos(var_name)); 1240 return ERR_PTR(-EINVAL); 1241 } 1242 1243 found = hist_field; 1244 } 1245 } 1246 } 1247 return found; 1248 } 1249 1250 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1251 char *system, 1252 char *event_name, 1253 char *var_name) 1254 { 1255 struct trace_array *tr = hist_data->event_file->tr; 1256 struct hist_field *hist_field = NULL; 1257 struct trace_event_file *file; 1258 1259 if (!system || !event_name) { 1260 hist_field = find_match_var(hist_data, var_name); 1261 if (IS_ERR(hist_field)) 1262 return NULL; 1263 if (hist_field) 1264 return hist_field; 1265 } 1266 1267 file = find_var_file(tr, system, event_name, var_name); 1268 if (!file) 1269 return NULL; 1270 1271 hist_field = find_file_var(file, var_name); 1272 1273 return hist_field; 1274 } 1275 1276 static u64 hist_field_var_ref(struct hist_field *hist_field, 1277 struct tracing_map_elt *elt, 1278 struct trace_buffer *buffer, 1279 struct ring_buffer_event *rbe, 1280 void *event) 1281 { 1282 struct hist_elt_data *elt_data; 1283 u64 var_val = 0; 1284 1285 if (WARN_ON_ONCE(!elt)) 1286 return var_val; 1287 1288 elt_data = elt->private_data; 1289 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1290 1291 return var_val; 1292 } 1293 1294 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1295 u64 *var_ref_vals, bool self) 1296 { 1297 struct hist_trigger_data *var_data; 1298 struct tracing_map_elt *var_elt; 1299 struct hist_field *hist_field; 1300 unsigned int i, var_idx; 1301 bool resolved = true; 1302 u64 var_val = 0; 1303 1304 for (i = 0; i < hist_data->n_var_refs; i++) { 1305 hist_field = hist_data->var_refs[i]; 1306 var_idx = hist_field->var.idx; 1307 var_data = hist_field->var.hist_data; 1308 1309 if (var_data == NULL) { 1310 resolved = false; 1311 break; 1312 } 1313 1314 if ((self && var_data != hist_data) || 1315 (!self && var_data == hist_data)) 1316 continue; 1317 1318 var_elt = tracing_map_lookup(var_data->map, key); 1319 if (!var_elt) { 1320 resolved = false; 1321 break; 1322 } 1323 1324 if (!tracing_map_var_set(var_elt, var_idx)) { 1325 resolved = false; 1326 break; 1327 } 1328 1329 if (self || !hist_field->read_once) 1330 var_val = tracing_map_read_var(var_elt, var_idx); 1331 else 1332 var_val = tracing_map_read_var_once(var_elt, var_idx); 1333 1334 var_ref_vals[i] = var_val; 1335 } 1336 1337 return resolved; 1338 } 1339 1340 static const char *hist_field_name(struct hist_field *field, 1341 unsigned int level) 1342 { 1343 const char *field_name = ""; 1344 1345 if (WARN_ON_ONCE(!field)) 1346 return field_name; 1347 1348 if (level > 1) 1349 return field_name; 1350 1351 if (field->field) 1352 field_name = field->field->name; 1353 else if (field->flags & HIST_FIELD_FL_LOG2 || 1354 field->flags & HIST_FIELD_FL_ALIAS || 1355 field->flags & HIST_FIELD_FL_BUCKET) 1356 field_name = hist_field_name(field->operands[0], ++level); 1357 else if (field->flags & HIST_FIELD_FL_CPU) 1358 field_name = "common_cpu"; 1359 else if (field->flags & HIST_FIELD_FL_COMM) 1360 field_name = "common_comm"; 1361 else if (field->flags & HIST_FIELD_FL_EXPR || 1362 field->flags & HIST_FIELD_FL_VAR_REF) { 1363 if (field->system) { 1364 static char full_name[MAX_FILTER_STR_VAL]; 1365 static char *fmt; 1366 int len; 1367 1368 fmt = field->flags & HIST_FIELD_FL_VAR_REF ? "%s.%s.$%s" : "%s.%s.%s"; 1369 1370 len = snprintf(full_name, sizeof(full_name), fmt, 1371 field->system, field->event_name, 1372 field->name); 1373 if (len < sizeof(full_name)) 1374 field_name = full_name; 1375 } else 1376 field_name = field->name; 1377 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1378 field_name = "common_timestamp"; 1379 else if (field->flags & HIST_FIELD_FL_STACKTRACE) { 1380 field_name = "common_stacktrace"; 1381 } else if (field->flags & HIST_FIELD_FL_HITCOUNT) 1382 field_name = "hitcount"; 1383 1384 if (field_name == NULL) 1385 field_name = ""; 1386 1387 return field_name; 1388 } 1389 1390 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed) 1391 { 1392 switch (field_size) { 1393 case 8: 1394 if (field_is_signed) 1395 return HIST_FIELD_FN_S64; 1396 else 1397 return HIST_FIELD_FN_U64; 1398 case 4: 1399 if (field_is_signed) 1400 return HIST_FIELD_FN_S32; 1401 else 1402 return HIST_FIELD_FN_U32; 1403 case 2: 1404 if (field_is_signed) 1405 return HIST_FIELD_FN_S16; 1406 else 1407 return HIST_FIELD_FN_U16; 1408 case 1: 1409 if (field_is_signed) 1410 return HIST_FIELD_FN_S8; 1411 else 1412 return HIST_FIELD_FN_U8; 1413 } 1414 1415 return HIST_FIELD_FN_NOP; 1416 } 1417 1418 static int parse_map_size(char *str) 1419 { 1420 unsigned long size, map_bits; 1421 int ret; 1422 1423 ret = kstrtoul(str, 0, &size); 1424 if (ret) 1425 goto out; 1426 1427 map_bits = ilog2(roundup_pow_of_two(size)); 1428 if (map_bits < TRACING_MAP_BITS_MIN || 1429 map_bits > TRACING_MAP_BITS_MAX) 1430 ret = -EINVAL; 1431 else 1432 ret = map_bits; 1433 out: 1434 return ret; 1435 } 1436 1437 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1438 { 1439 unsigned int i; 1440 1441 if (!attrs) 1442 return; 1443 1444 for (i = 0; i < attrs->n_assignments; i++) 1445 kfree(attrs->assignment_str[i]); 1446 1447 for (i = 0; i < attrs->n_actions; i++) 1448 kfree(attrs->action_str[i]); 1449 1450 kfree(attrs->name); 1451 kfree(attrs->sort_key_str); 1452 kfree(attrs->keys_str); 1453 kfree(attrs->vals_str); 1454 kfree(attrs->clock); 1455 kfree(attrs); 1456 } 1457 1458 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1459 { 1460 int ret = -EINVAL; 1461 1462 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1463 return ret; 1464 1465 if ((str_has_prefix(str, "onmatch(")) || 1466 (str_has_prefix(str, "onmax(")) || 1467 (str_has_prefix(str, "onchange("))) { 1468 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1469 if (!attrs->action_str[attrs->n_actions]) { 1470 ret = -ENOMEM; 1471 return ret; 1472 } 1473 attrs->n_actions++; 1474 ret = 0; 1475 } 1476 return ret; 1477 } 1478 1479 static int parse_assignment(struct trace_array *tr, 1480 char *str, struct hist_trigger_attrs *attrs) 1481 { 1482 int len, ret = 0; 1483 1484 if ((len = str_has_prefix(str, "key=")) || 1485 (len = str_has_prefix(str, "keys="))) { 1486 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1487 if (!attrs->keys_str) { 1488 ret = -ENOMEM; 1489 goto out; 1490 } 1491 } else if ((len = str_has_prefix(str, "val=")) || 1492 (len = str_has_prefix(str, "vals=")) || 1493 (len = str_has_prefix(str, "values="))) { 1494 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1495 if (!attrs->vals_str) { 1496 ret = -ENOMEM; 1497 goto out; 1498 } 1499 } else if ((len = str_has_prefix(str, "sort="))) { 1500 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1501 if (!attrs->sort_key_str) { 1502 ret = -ENOMEM; 1503 goto out; 1504 } 1505 } else if (str_has_prefix(str, "name=")) { 1506 attrs->name = kstrdup(str, GFP_KERNEL); 1507 if (!attrs->name) { 1508 ret = -ENOMEM; 1509 goto out; 1510 } 1511 } else if ((len = str_has_prefix(str, "clock="))) { 1512 str += len; 1513 1514 str = strstrip(str); 1515 attrs->clock = kstrdup(str, GFP_KERNEL); 1516 if (!attrs->clock) { 1517 ret = -ENOMEM; 1518 goto out; 1519 } 1520 } else if ((len = str_has_prefix(str, "size="))) { 1521 int map_bits = parse_map_size(str + len); 1522 1523 if (map_bits < 0) { 1524 ret = map_bits; 1525 goto out; 1526 } 1527 attrs->map_bits = map_bits; 1528 } else { 1529 char *assignment; 1530 1531 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1532 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1533 ret = -EINVAL; 1534 goto out; 1535 } 1536 1537 assignment = kstrdup(str, GFP_KERNEL); 1538 if (!assignment) { 1539 ret = -ENOMEM; 1540 goto out; 1541 } 1542 1543 attrs->assignment_str[attrs->n_assignments++] = assignment; 1544 } 1545 out: 1546 return ret; 1547 } 1548 1549 static struct hist_trigger_attrs * 1550 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1551 { 1552 struct hist_trigger_attrs *attrs; 1553 int ret = 0; 1554 1555 attrs = kzalloc_obj(*attrs); 1556 if (!attrs) 1557 return ERR_PTR(-ENOMEM); 1558 1559 while (trigger_str) { 1560 char *str = strsep(&trigger_str, ":"); 1561 char *rhs; 1562 1563 rhs = strchr(str, '='); 1564 if (rhs) { 1565 if (!strlen(++rhs)) { 1566 ret = -EINVAL; 1567 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1568 goto free; 1569 } 1570 ret = parse_assignment(tr, str, attrs); 1571 if (ret) 1572 goto free; 1573 } else if (strcmp(str, "nohitcount") == 0 || 1574 strcmp(str, "NOHC") == 0) 1575 attrs->no_hitcount = true; 1576 else if (strcmp(str, "pause") == 0) 1577 attrs->pause = true; 1578 else if ((strcmp(str, "cont") == 0) || 1579 (strcmp(str, "continue") == 0)) 1580 attrs->cont = true; 1581 else if (strcmp(str, "clear") == 0) 1582 attrs->clear = true; 1583 else { 1584 ret = parse_action(str, attrs); 1585 if (ret) 1586 goto free; 1587 } 1588 } 1589 1590 if (!attrs->keys_str) { 1591 ret = -EINVAL; 1592 goto free; 1593 } 1594 1595 if (!attrs->clock) { 1596 attrs->clock = kstrdup("global", GFP_KERNEL); 1597 if (!attrs->clock) { 1598 ret = -ENOMEM; 1599 goto free; 1600 } 1601 } 1602 1603 return attrs; 1604 free: 1605 destroy_hist_trigger_attrs(attrs); 1606 1607 return ERR_PTR(ret); 1608 } 1609 1610 static inline void save_comm(char *comm, struct task_struct *task) 1611 { 1612 if (!task->pid) { 1613 strcpy(comm, "<idle>"); 1614 return; 1615 } 1616 1617 if (WARN_ON_ONCE(task->pid < 0)) { 1618 strcpy(comm, "<XXX>"); 1619 return; 1620 } 1621 1622 strscpy(comm, task->comm, TASK_COMM_LEN); 1623 } 1624 1625 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1626 { 1627 unsigned int i; 1628 1629 for (i = 0; i < elt_data->n_field_var_str; i++) 1630 kfree(elt_data->field_var_str[i]); 1631 1632 kfree(elt_data->comm); 1633 kfree(elt_data); 1634 } 1635 1636 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1637 { 1638 struct hist_elt_data *elt_data = elt->private_data; 1639 1640 hist_elt_data_free(elt_data); 1641 } 1642 1643 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1644 { 1645 struct hist_trigger_data *hist_data = elt->map->private_data; 1646 unsigned int size = TASK_COMM_LEN; 1647 struct hist_elt_data *elt_data; 1648 struct hist_field *hist_field; 1649 unsigned int i, n_str; 1650 1651 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); 1652 1653 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + 1654 hist_data->n_var_str; 1655 if (n_str > SYNTH_FIELDS_MAX) 1656 return -EINVAL; 1657 1658 elt_data = kzalloc_flex(*elt_data, field_var_str, n_str); 1659 if (!elt_data) 1660 return -ENOMEM; 1661 1662 elt_data->n_field_var_str = n_str; 1663 1664 for_each_hist_field(i, hist_data) { 1665 hist_field = hist_data->fields[i]; 1666 1667 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) { 1668 elt_data->comm = kzalloc(size, GFP_KERNEL); 1669 if (!elt_data->comm) { 1670 kfree(elt_data); 1671 return -ENOMEM; 1672 } 1673 break; 1674 } 1675 } 1676 1677 size = STR_VAR_LEN_MAX; 1678 1679 for (i = 0; i < n_str; i++) { 1680 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1681 if (!elt_data->field_var_str[i]) { 1682 hist_elt_data_free(elt_data); 1683 return -ENOMEM; 1684 } 1685 } 1686 1687 elt->private_data = elt_data; 1688 1689 return 0; 1690 } 1691 1692 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1693 { 1694 struct hist_elt_data *elt_data = elt->private_data; 1695 1696 if (elt_data->comm) 1697 save_comm(elt_data->comm, current); 1698 } 1699 1700 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1701 .elt_alloc = hist_trigger_elt_data_alloc, 1702 .elt_free = hist_trigger_elt_data_free, 1703 .elt_init = hist_trigger_elt_data_init, 1704 }; 1705 1706 static const char *get_hist_field_flags(struct hist_field *hist_field) 1707 { 1708 const char *flags_str = NULL; 1709 1710 if (hist_field->flags & HIST_FIELD_FL_HEX) 1711 flags_str = "hex"; 1712 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1713 flags_str = "sym"; 1714 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1715 flags_str = "sym-offset"; 1716 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1717 flags_str = "execname"; 1718 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1719 flags_str = "syscall"; 1720 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1721 flags_str = "log2"; 1722 else if (hist_field->flags & HIST_FIELD_FL_BUCKET) 1723 flags_str = "buckets"; 1724 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1725 flags_str = "usecs"; 1726 else if (hist_field->flags & HIST_FIELD_FL_PERCENT) 1727 flags_str = "percent"; 1728 else if (hist_field->flags & HIST_FIELD_FL_GRAPH) 1729 flags_str = "graph"; 1730 else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 1731 flags_str = "stacktrace"; 1732 1733 return flags_str; 1734 } 1735 1736 static void expr_field_str(struct hist_field *field, char *expr) 1737 { 1738 if (field->flags & HIST_FIELD_FL_VAR_REF) { 1739 if (!field->system) 1740 strcat(expr, "$"); 1741 } else if (field->flags & HIST_FIELD_FL_CONST) { 1742 char str[HIST_CONST_DIGITS_MAX]; 1743 1744 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant); 1745 strcat(expr, str); 1746 } 1747 1748 strcat(expr, hist_field_name(field, 0)); 1749 1750 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1751 const char *flags_str = get_hist_field_flags(field); 1752 1753 if (flags_str) { 1754 strcat(expr, "."); 1755 strcat(expr, flags_str); 1756 } 1757 } 1758 } 1759 1760 static char *expr_str(struct hist_field *field, unsigned int level) 1761 { 1762 char *expr; 1763 1764 if (level > 1) 1765 return NULL; 1766 1767 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1768 if (!expr) 1769 return NULL; 1770 1771 if (!field->operands[0]) { 1772 expr_field_str(field, expr); 1773 return expr; 1774 } 1775 1776 if (field->operator == FIELD_OP_UNARY_MINUS) { 1777 char *subexpr; 1778 1779 strcat(expr, "-("); 1780 subexpr = expr_str(field->operands[0], ++level); 1781 if (!subexpr) { 1782 kfree(expr); 1783 return NULL; 1784 } 1785 strcat(expr, subexpr); 1786 strcat(expr, ")"); 1787 1788 kfree(subexpr); 1789 1790 return expr; 1791 } 1792 1793 expr_field_str(field->operands[0], expr); 1794 1795 switch (field->operator) { 1796 case FIELD_OP_MINUS: 1797 strcat(expr, "-"); 1798 break; 1799 case FIELD_OP_PLUS: 1800 strcat(expr, "+"); 1801 break; 1802 case FIELD_OP_DIV: 1803 strcat(expr, "/"); 1804 break; 1805 case FIELD_OP_MULT: 1806 strcat(expr, "*"); 1807 break; 1808 default: 1809 kfree(expr); 1810 return NULL; 1811 } 1812 1813 expr_field_str(field->operands[1], expr); 1814 1815 return expr; 1816 } 1817 1818 /* 1819 * If field_op != FIELD_OP_NONE, *sep points to the root operator 1820 * of the expression tree to be evaluated. 1821 */ 1822 static int contains_operator(char *str, char **sep) 1823 { 1824 enum field_op_id field_op = FIELD_OP_NONE; 1825 char *minus_op, *plus_op, *div_op, *mult_op; 1826 1827 1828 /* 1829 * Report the last occurrence of the operators first, so that the 1830 * expression is evaluated left to right. This is important since 1831 * subtraction and division are not associative. 1832 * 1833 * e.g 1834 * 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2 1835 * 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2 1836 */ 1837 1838 /* 1839 * First, find lower precedence addition and subtraction 1840 * since the expression will be evaluated recursively. 1841 */ 1842 minus_op = strrchr(str, '-'); 1843 if (minus_op) { 1844 /* 1845 * Unary minus is not supported in sub-expressions. If 1846 * present, it is always the next root operator. 1847 */ 1848 if (minus_op == str) { 1849 field_op = FIELD_OP_UNARY_MINUS; 1850 goto out; 1851 } 1852 1853 field_op = FIELD_OP_MINUS; 1854 } 1855 1856 plus_op = strrchr(str, '+'); 1857 if (plus_op || minus_op) { 1858 /* 1859 * For operators of the same precedence use to rightmost as the 1860 * root, so that the expression is evaluated left to right. 1861 */ 1862 if (plus_op > minus_op) 1863 field_op = FIELD_OP_PLUS; 1864 goto out; 1865 } 1866 1867 /* 1868 * Multiplication and division have higher precedence than addition and 1869 * subtraction. 1870 */ 1871 div_op = strrchr(str, '/'); 1872 if (div_op) 1873 field_op = FIELD_OP_DIV; 1874 1875 mult_op = strrchr(str, '*'); 1876 /* 1877 * For operators of the same precedence use to rightmost as the 1878 * root, so that the expression is evaluated left to right. 1879 */ 1880 if (mult_op > div_op) 1881 field_op = FIELD_OP_MULT; 1882 1883 out: 1884 if (sep) { 1885 switch (field_op) { 1886 case FIELD_OP_UNARY_MINUS: 1887 case FIELD_OP_MINUS: 1888 *sep = minus_op; 1889 break; 1890 case FIELD_OP_PLUS: 1891 *sep = plus_op; 1892 break; 1893 case FIELD_OP_DIV: 1894 *sep = div_op; 1895 break; 1896 case FIELD_OP_MULT: 1897 *sep = mult_op; 1898 break; 1899 case FIELD_OP_NONE: 1900 default: 1901 *sep = NULL; 1902 break; 1903 } 1904 } 1905 1906 return field_op; 1907 } 1908 1909 static void get_hist_field(struct hist_field *hist_field) 1910 { 1911 hist_field->ref++; 1912 } 1913 1914 static void __destroy_hist_field(struct hist_field *hist_field) 1915 { 1916 if (--hist_field->ref > 1) 1917 return; 1918 1919 kfree(hist_field->var.name); 1920 kfree(hist_field->name); 1921 1922 /* Can likely be a const */ 1923 kfree_const(hist_field->type); 1924 1925 kfree(hist_field->system); 1926 kfree(hist_field->event_name); 1927 1928 kfree(hist_field); 1929 } 1930 1931 static void destroy_hist_field(struct hist_field *hist_field, 1932 unsigned int level) 1933 { 1934 unsigned int i; 1935 1936 if (level > 3) 1937 return; 1938 1939 if (!hist_field) 1940 return; 1941 1942 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1943 return; /* var refs will be destroyed separately */ 1944 1945 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1946 destroy_hist_field(hist_field->operands[i], level + 1); 1947 1948 __destroy_hist_field(hist_field); 1949 } 1950 1951 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1952 struct ftrace_event_field *field, 1953 unsigned long flags, 1954 char *var_name) 1955 { 1956 struct hist_field *hist_field; 1957 1958 if (field && is_function_field(field)) 1959 return NULL; 1960 1961 hist_field = kzalloc_obj(struct hist_field); 1962 if (!hist_field) 1963 return NULL; 1964 1965 hist_field->ref = 1; 1966 1967 hist_field->hist_data = hist_data; 1968 1969 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1970 goto out; /* caller will populate */ 1971 1972 if (flags & HIST_FIELD_FL_VAR_REF) { 1973 hist_field->fn_num = HIST_FIELD_FN_VAR_REF; 1974 goto out; 1975 } 1976 1977 if (flags & HIST_FIELD_FL_HITCOUNT) { 1978 hist_field->fn_num = HIST_FIELD_FN_COUNTER; 1979 hist_field->size = sizeof(u64); 1980 hist_field->type = "u64"; 1981 goto out; 1982 } 1983 1984 if (flags & HIST_FIELD_FL_CONST) { 1985 hist_field->fn_num = HIST_FIELD_FN_CONST; 1986 hist_field->size = sizeof(u64); 1987 hist_field->type = kstrdup("u64", GFP_KERNEL); 1988 if (!hist_field->type) 1989 goto free; 1990 goto out; 1991 } 1992 1993 if (flags & HIST_FIELD_FL_STACKTRACE) { 1994 if (field) 1995 hist_field->fn_num = HIST_FIELD_FN_STACK; 1996 else 1997 hist_field->fn_num = HIST_FIELD_FN_NOP; 1998 hist_field->size = HIST_STACKTRACE_SIZE; 1999 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL); 2000 if (!hist_field->type) 2001 goto free; 2002 goto out; 2003 } 2004 2005 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) { 2006 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET); 2007 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 : 2008 HIST_FIELD_FN_BUCKET; 2009 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2010 if (!hist_field->operands[0]) 2011 goto free; 2012 hist_field->size = hist_field->operands[0]->size; 2013 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL); 2014 if (!hist_field->type) 2015 goto free; 2016 goto out; 2017 } 2018 2019 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2020 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP; 2021 hist_field->size = sizeof(u64); 2022 hist_field->type = "u64"; 2023 goto out; 2024 } 2025 2026 if (flags & HIST_FIELD_FL_CPU) { 2027 hist_field->fn_num = HIST_FIELD_FN_CPU; 2028 hist_field->size = sizeof(int); 2029 hist_field->type = "unsigned int"; 2030 goto out; 2031 } 2032 2033 if (flags & HIST_FIELD_FL_COMM) { 2034 hist_field->fn_num = HIST_FIELD_FN_COMM; 2035 hist_field->size = MAX_FILTER_STR_VAL; 2036 hist_field->type = "char[]"; 2037 goto out; 2038 } 2039 2040 if (WARN_ON_ONCE(!field)) 2041 goto out; 2042 2043 /* Pointers to strings are just pointers and dangerous to dereference */ 2044 if (is_string_field(field) && 2045 (field->filter_type != FILTER_PTR_STRING)) { 2046 flags |= HIST_FIELD_FL_STRING; 2047 2048 hist_field->size = MAX_FILTER_STR_VAL; 2049 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2050 if (!hist_field->type) 2051 goto free; 2052 2053 if (field->filter_type == FILTER_STATIC_STRING) { 2054 hist_field->fn_num = HIST_FIELD_FN_STRING; 2055 hist_field->size = field->size; 2056 } else if (field->filter_type == FILTER_DYN_STRING) { 2057 hist_field->fn_num = HIST_FIELD_FN_DYNSTRING; 2058 } else if (field->filter_type == FILTER_RDYN_STRING) 2059 hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; 2060 else 2061 hist_field->fn_num = HIST_FIELD_FN_PSTRING; 2062 } else if (field->filter_type == FILTER_STACKTRACE) { 2063 flags |= HIST_FIELD_FL_STACKTRACE; 2064 2065 hist_field->size = MAX_FILTER_STR_VAL; 2066 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2067 if (!hist_field->type) 2068 goto free; 2069 2070 hist_field->fn_num = HIST_FIELD_FN_STACK; 2071 } else { 2072 hist_field->size = field->size; 2073 hist_field->is_signed = field->is_signed; 2074 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2075 if (!hist_field->type) 2076 goto free; 2077 2078 hist_field->fn_num = select_value_fn(field->size, 2079 field->is_signed); 2080 if (hist_field->fn_num == HIST_FIELD_FN_NOP) { 2081 destroy_hist_field(hist_field, 0); 2082 return NULL; 2083 } 2084 } 2085 out: 2086 hist_field->field = field; 2087 hist_field->flags = flags; 2088 2089 if (var_name) { 2090 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2091 if (!hist_field->var.name) 2092 goto free; 2093 } 2094 2095 return hist_field; 2096 free: 2097 destroy_hist_field(hist_field, 0); 2098 return NULL; 2099 } 2100 2101 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2102 { 2103 unsigned int i; 2104 2105 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2106 if (hist_data->fields[i]) { 2107 destroy_hist_field(hist_data->fields[i], 0); 2108 hist_data->fields[i] = NULL; 2109 } 2110 } 2111 2112 for (i = 0; i < hist_data->n_var_refs; i++) { 2113 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2114 __destroy_hist_field(hist_data->var_refs[i]); 2115 hist_data->var_refs[i] = NULL; 2116 } 2117 } 2118 2119 static int init_var_ref(struct hist_field *ref_field, 2120 struct hist_field *var_field, 2121 char *system, char *event_name) 2122 { 2123 int err = 0; 2124 2125 ref_field->var.idx = var_field->var.idx; 2126 ref_field->var.hist_data = var_field->hist_data; 2127 ref_field->size = var_field->size; 2128 ref_field->is_signed = var_field->is_signed; 2129 ref_field->flags |= var_field->flags & 2130 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2131 2132 if (system) { 2133 ref_field->system = kstrdup(system, GFP_KERNEL); 2134 if (!ref_field->system) 2135 return -ENOMEM; 2136 } 2137 2138 if (event_name) { 2139 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2140 if (!ref_field->event_name) { 2141 err = -ENOMEM; 2142 goto free; 2143 } 2144 } 2145 2146 if (var_field->var.name) { 2147 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2148 if (!ref_field->name) { 2149 err = -ENOMEM; 2150 goto free; 2151 } 2152 } else if (var_field->name) { 2153 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2154 if (!ref_field->name) { 2155 err = -ENOMEM; 2156 goto free; 2157 } 2158 } 2159 2160 ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL); 2161 if (!ref_field->type) { 2162 err = -ENOMEM; 2163 goto free; 2164 } 2165 out: 2166 return err; 2167 free: 2168 kfree(ref_field->system); 2169 ref_field->system = NULL; 2170 kfree(ref_field->event_name); 2171 ref_field->event_name = NULL; 2172 kfree(ref_field->name); 2173 ref_field->name = NULL; 2174 2175 goto out; 2176 } 2177 2178 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 2179 struct hist_field *var_field) 2180 { 2181 struct hist_field *ref_field; 2182 int i; 2183 2184 for (i = 0; i < hist_data->n_var_refs; i++) { 2185 ref_field = hist_data->var_refs[i]; 2186 if (ref_field->var.idx == var_field->var.idx && 2187 ref_field->var.hist_data == var_field->hist_data) 2188 return i; 2189 } 2190 2191 return -ENOENT; 2192 } 2193 2194 /** 2195 * create_var_ref - Create a variable reference and attach it to trigger 2196 * @hist_data: The trigger that will be referencing the variable 2197 * @var_field: The VAR field to create a reference to 2198 * @system: The optional system string 2199 * @event_name: The optional event_name string 2200 * 2201 * Given a variable hist_field, create a VAR_REF hist_field that 2202 * represents a reference to it. 2203 * 2204 * This function also adds the reference to the trigger that 2205 * now references the variable. 2206 * 2207 * Return: The VAR_REF field if successful, NULL if not 2208 */ 2209 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2210 struct hist_field *var_field, 2211 char *system, char *event_name) 2212 { 2213 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2214 struct hist_field *ref_field; 2215 int i; 2216 2217 /* Check if the variable already exists */ 2218 for (i = 0; i < hist_data->n_var_refs; i++) { 2219 ref_field = hist_data->var_refs[i]; 2220 if (ref_field->var.idx == var_field->var.idx && 2221 ref_field->var.hist_data == var_field->hist_data) { 2222 get_hist_field(ref_field); 2223 return ref_field; 2224 } 2225 } 2226 /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ 2227 if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) 2228 return NULL; 2229 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2230 if (ref_field) { 2231 if (init_var_ref(ref_field, var_field, system, event_name)) { 2232 destroy_hist_field(ref_field, 0); 2233 return NULL; 2234 } 2235 2236 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2237 ref_field->var_ref_idx = hist_data->n_var_refs++; 2238 } 2239 2240 return ref_field; 2241 } 2242 2243 static bool is_var_ref(char *var_name) 2244 { 2245 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2246 return false; 2247 2248 return true; 2249 } 2250 2251 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2252 char *var_name) 2253 { 2254 char *name, *field; 2255 unsigned int i; 2256 2257 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2258 name = hist_data->attrs->var_defs.name[i]; 2259 2260 if (strcmp(var_name, name) == 0) { 2261 field = hist_data->attrs->var_defs.expr[i]; 2262 if (contains_operator(field, NULL) || is_var_ref(field)) 2263 continue; 2264 return field; 2265 } 2266 } 2267 2268 return NULL; 2269 } 2270 2271 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2272 char *system, char *event_name, 2273 char *var_name) 2274 { 2275 struct trace_event_call *call; 2276 2277 if (system && event_name) { 2278 call = hist_data->event_file->event_call; 2279 2280 if (strcmp(system, call->class->system) != 0) 2281 return NULL; 2282 2283 if (strcmp(event_name, trace_event_name(call)) != 0) 2284 return NULL; 2285 } 2286 2287 if (!!system != !!event_name) 2288 return NULL; 2289 2290 if (!is_var_ref(var_name)) 2291 return NULL; 2292 2293 var_name++; 2294 2295 return field_name_from_var(hist_data, var_name); 2296 } 2297 2298 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2299 char *system, char *event_name, 2300 char *var_name) 2301 { 2302 struct hist_field *var_field = NULL, *ref_field = NULL; 2303 struct trace_array *tr = hist_data->event_file->tr; 2304 2305 if (!is_var_ref(var_name)) 2306 return NULL; 2307 2308 var_name++; 2309 2310 var_field = find_event_var(hist_data, system, event_name, var_name); 2311 if (var_field) 2312 ref_field = create_var_ref(hist_data, var_field, 2313 system, event_name); 2314 2315 if (!ref_field) 2316 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2317 2318 return ref_field; 2319 } 2320 2321 static struct ftrace_event_field * 2322 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2323 char *field_str, unsigned long *flags, unsigned long *buckets) 2324 { 2325 struct ftrace_event_field *field = NULL; 2326 char *field_name, *modifier, *str; 2327 struct trace_array *tr = file->tr; 2328 2329 modifier = str = kstrdup(field_str, GFP_KERNEL); 2330 if (!modifier) 2331 return ERR_PTR(-ENOMEM); 2332 2333 field_name = strsep(&modifier, "."); 2334 if (modifier) { 2335 if (strcmp(modifier, "hex") == 0) 2336 *flags |= HIST_FIELD_FL_HEX; 2337 else if (strcmp(modifier, "sym") == 0) 2338 *flags |= HIST_FIELD_FL_SYM; 2339 /* 2340 * 'sym-offset' occurrences in the trigger string are modified 2341 * to 'symXoffset' to simplify arithmetic expression parsing. 2342 */ 2343 else if (strcmp(modifier, "symXoffset") == 0) 2344 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2345 else if ((strcmp(modifier, "execname") == 0) && 2346 (strcmp(field_name, "common_pid") == 0)) 2347 *flags |= HIST_FIELD_FL_EXECNAME; 2348 else if (strcmp(modifier, "syscall") == 0) 2349 *flags |= HIST_FIELD_FL_SYSCALL; 2350 else if (strcmp(modifier, "stacktrace") == 0) 2351 *flags |= HIST_FIELD_FL_STACKTRACE; 2352 else if (strcmp(modifier, "log2") == 0) 2353 *flags |= HIST_FIELD_FL_LOG2; 2354 else if (strcmp(modifier, "usecs") == 0) 2355 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2356 else if (strncmp(modifier, "bucket", 6) == 0) { 2357 int ret; 2358 2359 modifier += 6; 2360 2361 if (*modifier == 's') 2362 modifier++; 2363 if (*modifier != '=') 2364 goto error; 2365 modifier++; 2366 ret = kstrtoul(modifier, 0, buckets); 2367 if (ret || !(*buckets)) 2368 goto error; 2369 *flags |= HIST_FIELD_FL_BUCKET; 2370 } else if (strncmp(modifier, "percent", 7) == 0) { 2371 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2372 goto error; 2373 *flags |= HIST_FIELD_FL_PERCENT; 2374 } else if (strncmp(modifier, "graph", 5) == 0) { 2375 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2376 goto error; 2377 *flags |= HIST_FIELD_FL_GRAPH; 2378 } else { 2379 error: 2380 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2381 field = ERR_PTR(-EINVAL); 2382 goto out; 2383 } 2384 } 2385 2386 if (strcmp(field_name, "common_timestamp") == 0) { 2387 *flags |= HIST_FIELD_FL_TIMESTAMP; 2388 hist_data->enable_timestamps = true; 2389 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2390 hist_data->attrs->ts_in_usecs = true; 2391 } else if (strcmp(field_name, "common_stacktrace") == 0) { 2392 *flags |= HIST_FIELD_FL_STACKTRACE; 2393 } else if (strcmp(field_name, "common_cpu") == 0) { 2394 *flags |= HIST_FIELD_FL_CPU; 2395 } else if (strcmp(field_name, "common_comm") == 0) { 2396 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2397 } else if (strcmp(field_name, "hitcount") == 0) 2398 *flags |= HIST_FIELD_FL_HITCOUNT; 2399 else { 2400 field = trace_find_event_field(file->event_call, field_name); 2401 if (!field || !field->size) { 2402 /* 2403 * For backward compatibility, if field_name 2404 * was "cpu" or "stacktrace", then we treat this 2405 * the same as common_cpu and common_stacktrace 2406 * respectively. This also works for "CPU", and 2407 * "STACKTRACE". 2408 */ 2409 if (field && field->filter_type == FILTER_CPU) { 2410 *flags |= HIST_FIELD_FL_CPU; 2411 } else if (field && field->filter_type == FILTER_STACKTRACE) { 2412 *flags |= HIST_FIELD_FL_STACKTRACE; 2413 } else if (field && field->filter_type == FILTER_COMM) { 2414 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2415 } else { 2416 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, 2417 errpos(field_name)); 2418 field = ERR_PTR(-EINVAL); 2419 goto out; 2420 } 2421 } 2422 } 2423 out: 2424 kfree(str); 2425 2426 return field; 2427 } 2428 2429 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2430 struct hist_field *var_ref, 2431 char *var_name) 2432 { 2433 struct hist_field *alias = NULL; 2434 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2435 2436 alias = create_hist_field(hist_data, NULL, flags, var_name); 2437 if (!alias) 2438 return NULL; 2439 2440 alias->fn_num = var_ref->fn_num; 2441 alias->operands[0] = var_ref; 2442 2443 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2444 destroy_hist_field(alias, 0); 2445 return NULL; 2446 } 2447 2448 alias->var_ref_idx = var_ref->var_ref_idx; 2449 2450 return alias; 2451 } 2452 2453 static struct hist_field *parse_const(struct hist_trigger_data *hist_data, 2454 char *str, char *var_name, 2455 unsigned long *flags) 2456 { 2457 struct trace_array *tr = hist_data->event_file->tr; 2458 struct hist_field *field = NULL; 2459 u64 constant; 2460 2461 if (kstrtoull(str, 0, &constant)) { 2462 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str)); 2463 return NULL; 2464 } 2465 2466 *flags |= HIST_FIELD_FL_CONST; 2467 field = create_hist_field(hist_data, NULL, *flags, var_name); 2468 if (!field) 2469 return NULL; 2470 2471 field->constant = constant; 2472 2473 return field; 2474 } 2475 2476 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2477 struct trace_event_file *file, char *str, 2478 unsigned long *flags, char *var_name) 2479 { 2480 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2481 struct ftrace_event_field *field = NULL; 2482 struct hist_field *hist_field = NULL; 2483 unsigned long buckets = 0; 2484 int ret = 0; 2485 2486 if (isdigit(str[0])) { 2487 hist_field = parse_const(hist_data, str, var_name, flags); 2488 if (!hist_field) { 2489 ret = -EINVAL; 2490 goto out; 2491 } 2492 return hist_field; 2493 } 2494 2495 s = strchr(str, '.'); 2496 if (s) { 2497 s = strchr(++s, '.'); 2498 if (s) { 2499 ref_system = strsep(&str, "."); 2500 if (!str) { 2501 ret = -EINVAL; 2502 goto out; 2503 } 2504 ref_event = strsep(&str, "."); 2505 if (!str) { 2506 ret = -EINVAL; 2507 goto out; 2508 } 2509 ref_var = str; 2510 } 2511 } 2512 2513 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2514 if (!s) { 2515 hist_field = parse_var_ref(hist_data, ref_system, 2516 ref_event, ref_var); 2517 if (hist_field) { 2518 if (var_name) { 2519 hist_field = create_alias(hist_data, hist_field, var_name); 2520 if (!hist_field) { 2521 ret = -ENOMEM; 2522 goto out; 2523 } 2524 } 2525 return hist_field; 2526 } 2527 } else 2528 str = s; 2529 2530 field = parse_field(hist_data, file, str, flags, &buckets); 2531 if (IS_ERR(field)) { 2532 ret = PTR_ERR(field); 2533 goto out; 2534 } 2535 2536 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2537 if (!hist_field) { 2538 ret = -ENOMEM; 2539 goto out; 2540 } 2541 hist_field->buckets = buckets; 2542 2543 return hist_field; 2544 out: 2545 return ERR_PTR(ret); 2546 } 2547 2548 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2549 struct trace_event_file *file, 2550 char *str, unsigned long flags, 2551 char *var_name, unsigned int *n_subexprs); 2552 2553 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2554 struct trace_event_file *file, 2555 char *str, unsigned long flags, 2556 char *var_name, unsigned int *n_subexprs) 2557 { 2558 struct hist_field *operand1, *expr = NULL; 2559 unsigned long operand_flags; 2560 int ret = 0; 2561 char *s; 2562 2563 /* Unary minus operator, increment n_subexprs */ 2564 ++*n_subexprs; 2565 2566 /* we support only -(xxx) i.e. explicit parens required */ 2567 2568 if (*n_subexprs > 3) { 2569 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2570 ret = -EINVAL; 2571 goto free; 2572 } 2573 2574 str++; /* skip leading '-' */ 2575 2576 s = strchr(str, '('); 2577 if (s) 2578 str++; 2579 else { 2580 ret = -EINVAL; 2581 goto free; 2582 } 2583 2584 s = strrchr(str, ')'); 2585 if (s) { 2586 /* unary minus not supported in sub-expressions */ 2587 if (*(s+1) != '\0') { 2588 hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR, 2589 errpos(str)); 2590 ret = -EINVAL; 2591 goto free; 2592 } 2593 *s = '\0'; 2594 } 2595 else { 2596 ret = -EINVAL; /* no closing ')' */ 2597 goto free; 2598 } 2599 2600 flags |= HIST_FIELD_FL_EXPR; 2601 expr = create_hist_field(hist_data, NULL, flags, var_name); 2602 if (!expr) { 2603 ret = -ENOMEM; 2604 goto free; 2605 } 2606 2607 operand_flags = 0; 2608 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2609 if (IS_ERR(operand1)) { 2610 ret = PTR_ERR(operand1); 2611 goto free; 2612 } 2613 if (operand1->flags & HIST_FIELD_FL_STRING) { 2614 /* String type can not be the operand of unary operator. */ 2615 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2616 destroy_hist_field(operand1, 0); 2617 ret = -EINVAL; 2618 goto free; 2619 } 2620 2621 expr->flags |= operand1->flags & 2622 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2623 expr->fn_num = HIST_FIELD_FN_UMINUS; 2624 expr->operands[0] = operand1; 2625 expr->size = operand1->size; 2626 expr->is_signed = operand1->is_signed; 2627 expr->operator = FIELD_OP_UNARY_MINUS; 2628 expr->name = expr_str(expr, 0); 2629 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2630 if (!expr->type) { 2631 ret = -ENOMEM; 2632 goto free; 2633 } 2634 2635 return expr; 2636 free: 2637 destroy_hist_field(expr, 0); 2638 return ERR_PTR(ret); 2639 } 2640 2641 /* 2642 * If the operands are var refs, return pointers the 2643 * variable(s) referenced in var1 and var2, else NULL. 2644 */ 2645 static int check_expr_operands(struct trace_array *tr, 2646 struct hist_field *operand1, 2647 struct hist_field *operand2, 2648 struct hist_field **var1, 2649 struct hist_field **var2) 2650 { 2651 unsigned long operand1_flags = operand1->flags; 2652 unsigned long operand2_flags = operand2->flags; 2653 2654 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2655 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2656 struct hist_field *var; 2657 2658 var = find_var_field(operand1->var.hist_data, operand1->name); 2659 if (!var) 2660 return -EINVAL; 2661 operand1_flags = var->flags; 2662 *var1 = var; 2663 } 2664 2665 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2666 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2667 struct hist_field *var; 2668 2669 var = find_var_field(operand2->var.hist_data, operand2->name); 2670 if (!var) 2671 return -EINVAL; 2672 operand2_flags = var->flags; 2673 *var2 = var; 2674 } 2675 2676 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2677 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2678 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2679 return -EINVAL; 2680 } 2681 2682 return 0; 2683 } 2684 2685 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2686 struct trace_event_file *file, 2687 char *str, unsigned long flags, 2688 char *var_name, unsigned int *n_subexprs) 2689 { 2690 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2691 struct hist_field *var1 = NULL, *var2 = NULL; 2692 unsigned long operand_flags, operand2_flags; 2693 int field_op, ret = -EINVAL; 2694 char *sep, *operand1_str; 2695 enum hist_field_fn op_fn; 2696 bool combine_consts; 2697 2698 if (*n_subexprs > 3) { 2699 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2700 return ERR_PTR(-EINVAL); 2701 } 2702 2703 field_op = contains_operator(str, &sep); 2704 2705 if (field_op == FIELD_OP_NONE) 2706 return parse_atom(hist_data, file, str, &flags, var_name); 2707 2708 if (field_op == FIELD_OP_UNARY_MINUS) 2709 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs); 2710 2711 /* Binary operator found, increment n_subexprs */ 2712 ++*n_subexprs; 2713 2714 /* Split the expression string at the root operator */ 2715 if (!sep) 2716 return ERR_PTR(-EINVAL); 2717 2718 *sep = '\0'; 2719 operand1_str = str; 2720 str = sep+1; 2721 2722 /* Binary operator requires both operands */ 2723 if (*operand1_str == '\0' || *str == '\0') 2724 return ERR_PTR(-EINVAL); 2725 2726 operand_flags = 0; 2727 2728 /* LHS of string is an expression e.g. a+b in a+b+c */ 2729 operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs); 2730 if (IS_ERR(operand1)) 2731 return ERR_CAST(operand1); 2732 2733 if (operand1->flags & HIST_FIELD_FL_STRING) { 2734 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str)); 2735 ret = -EINVAL; 2736 goto free_op1; 2737 } 2738 2739 /* RHS of string is another expression e.g. c in a+b+c */ 2740 operand_flags = 0; 2741 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2742 if (IS_ERR(operand2)) { 2743 ret = PTR_ERR(operand2); 2744 goto free_op1; 2745 } 2746 if (operand2->flags & HIST_FIELD_FL_STRING) { 2747 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2748 ret = -EINVAL; 2749 goto free_operands; 2750 } 2751 2752 switch (field_op) { 2753 case FIELD_OP_MINUS: 2754 op_fn = HIST_FIELD_FN_MINUS; 2755 break; 2756 case FIELD_OP_PLUS: 2757 op_fn = HIST_FIELD_FN_PLUS; 2758 break; 2759 case FIELD_OP_DIV: 2760 op_fn = HIST_FIELD_FN_DIV; 2761 break; 2762 case FIELD_OP_MULT: 2763 op_fn = HIST_FIELD_FN_MULT; 2764 break; 2765 default: 2766 ret = -EINVAL; 2767 goto free_operands; 2768 } 2769 2770 ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2); 2771 if (ret) 2772 goto free_operands; 2773 2774 operand_flags = var1 ? var1->flags : operand1->flags; 2775 operand2_flags = var2 ? var2->flags : operand2->flags; 2776 2777 /* 2778 * If both operands are constant, the expression can be 2779 * collapsed to a single constant. 2780 */ 2781 combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST; 2782 2783 flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR; 2784 2785 flags |= operand1->flags & 2786 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2787 2788 expr = create_hist_field(hist_data, NULL, flags, var_name); 2789 if (!expr) { 2790 ret = -ENOMEM; 2791 goto free_operands; 2792 } 2793 2794 operand1->read_once = true; 2795 operand2->read_once = true; 2796 2797 /* The operands are now owned and free'd by 'expr' */ 2798 expr->operands[0] = operand1; 2799 expr->operands[1] = operand2; 2800 2801 if (field_op == FIELD_OP_DIV && 2802 operand2_flags & HIST_FIELD_FL_CONST) { 2803 u64 divisor = var2 ? var2->constant : operand2->constant; 2804 2805 if (!divisor) { 2806 hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str)); 2807 ret = -EDOM; 2808 goto free_expr; 2809 } 2810 2811 /* 2812 * Copy the divisor here so we don't have to look it up 2813 * later if this is a var ref 2814 */ 2815 operand2->constant = divisor; 2816 op_fn = hist_field_get_div_fn(operand2); 2817 } 2818 2819 expr->fn_num = op_fn; 2820 2821 if (combine_consts) { 2822 if (var1) 2823 expr->operands[0] = var1; 2824 if (var2) 2825 expr->operands[1] = var2; 2826 2827 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL); 2828 expr->fn_num = HIST_FIELD_FN_CONST; 2829 2830 expr->operands[0] = NULL; 2831 expr->operands[1] = NULL; 2832 2833 /* 2834 * var refs won't be destroyed immediately 2835 * See: destroy_hist_field() 2836 */ 2837 destroy_hist_field(operand2, 0); 2838 destroy_hist_field(operand1, 0); 2839 2840 expr->name = expr_str(expr, 0); 2841 } else { 2842 /* The operand sizes should be the same, so just pick one */ 2843 expr->size = operand1->size; 2844 expr->is_signed = operand1->is_signed; 2845 2846 expr->operator = field_op; 2847 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2848 if (!expr->type) { 2849 ret = -ENOMEM; 2850 goto free_expr; 2851 } 2852 2853 expr->name = expr_str(expr, 0); 2854 } 2855 2856 return expr; 2857 2858 free_operands: 2859 destroy_hist_field(operand2, 0); 2860 free_op1: 2861 destroy_hist_field(operand1, 0); 2862 return ERR_PTR(ret); 2863 2864 free_expr: 2865 destroy_hist_field(expr, 0); 2866 return ERR_PTR(ret); 2867 } 2868 2869 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2870 struct trace_event_file *file) 2871 { 2872 struct event_trigger_data *test; 2873 2874 lockdep_assert_held(&event_mutex); 2875 2876 list_for_each_entry(test, &file->triggers, list) { 2877 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2878 if (test->private_data == hist_data) 2879 return test->filter_str; 2880 } 2881 } 2882 2883 return NULL; 2884 } 2885 2886 static struct event_command trigger_hist_cmd; 2887 static int event_hist_trigger_parse(struct event_command *cmd_ops, 2888 struct trace_event_file *file, 2889 char *glob, char *cmd, 2890 char *param_and_filter); 2891 2892 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2893 struct hist_trigger_data *hist_data, 2894 unsigned int n_keys) 2895 { 2896 struct hist_field *target_hist_field, *hist_field; 2897 unsigned int n, i, j; 2898 2899 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2900 return false; 2901 2902 i = hist_data->n_vals; 2903 j = target_hist_data->n_vals; 2904 2905 for (n = 0; n < n_keys; n++) { 2906 hist_field = hist_data->fields[i + n]; 2907 target_hist_field = target_hist_data->fields[j + n]; 2908 2909 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2910 return false; 2911 if (hist_field->size != target_hist_field->size) 2912 return false; 2913 if (hist_field->is_signed != target_hist_field->is_signed) 2914 return false; 2915 } 2916 2917 return true; 2918 } 2919 2920 static struct hist_trigger_data * 2921 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2922 struct trace_event_file *file) 2923 { 2924 struct hist_trigger_data *hist_data; 2925 struct event_trigger_data *test; 2926 unsigned int n_keys; 2927 2928 lockdep_assert_held(&event_mutex); 2929 2930 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2931 2932 list_for_each_entry(test, &file->triggers, list) { 2933 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2934 hist_data = test->private_data; 2935 2936 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2937 return hist_data; 2938 } 2939 } 2940 2941 return NULL; 2942 } 2943 2944 static struct trace_event_file *event_file(struct trace_array *tr, 2945 char *system, char *event_name) 2946 { 2947 struct trace_event_file *file; 2948 2949 file = __find_event_file(tr, system, event_name); 2950 if (!file) 2951 return ERR_PTR(-EINVAL); 2952 2953 return file; 2954 } 2955 2956 static struct hist_field * 2957 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2958 char *system, char *event_name, char *field_name) 2959 { 2960 struct hist_field *event_var; 2961 char *synthetic_name; 2962 struct seq_buf s; 2963 2964 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2965 if (!synthetic_name) 2966 return ERR_PTR(-ENOMEM); 2967 2968 seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); 2969 seq_buf_printf(&s, "synthetic_%s", field_name); 2970 2971 /* Terminate synthetic_name with a NUL. */ 2972 seq_buf_str(&s); 2973 2974 if (seq_buf_has_overflowed(&s)) { 2975 kfree(synthetic_name); 2976 return ERR_PTR(-E2BIG); 2977 } 2978 2979 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2980 2981 kfree(synthetic_name); 2982 2983 return event_var; 2984 } 2985 2986 /** 2987 * create_field_var_hist - Automatically create a histogram and var for a field 2988 * @target_hist_data: The target hist trigger 2989 * @subsys_name: Optional subsystem name 2990 * @event_name: Optional event name 2991 * @field_name: The name of the field (and the resulting variable) 2992 * 2993 * Hist trigger actions fetch data from variables, not directly from 2994 * events. However, for convenience, users are allowed to directly 2995 * specify an event field in an action, which will be automatically 2996 * converted into a variable on their behalf. 2997 * 2998 * If a user specifies a field on an event that isn't the event the 2999 * histogram currently being defined (the target event histogram), the 3000 * only way that can be accomplished is if a new hist trigger is 3001 * created and the field variable defined on that. 3002 * 3003 * This function creates a new histogram compatible with the target 3004 * event (meaning a histogram with the same key as the target 3005 * histogram), and creates a variable for the specified field, but 3006 * with 'synthetic_' prepended to the variable name in order to avoid 3007 * collision with normal field variables. 3008 * 3009 * Return: The variable created for the field. 3010 */ 3011 static struct hist_field * 3012 create_field_var_hist(struct hist_trigger_data *target_hist_data, 3013 char *subsys_name, char *event_name, char *field_name) 3014 { 3015 struct trace_array *tr = target_hist_data->event_file->tr; 3016 struct hist_trigger_data *hist_data; 3017 unsigned int i, n, first = true; 3018 struct field_var_hist *var_hist; 3019 struct trace_event_file *file; 3020 struct hist_field *key_field; 3021 struct hist_field *event_var; 3022 char *saved_filter; 3023 struct seq_buf s; 3024 char *cmd; 3025 int ret; 3026 3027 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 3028 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3029 return ERR_PTR(-EINVAL); 3030 } 3031 3032 file = event_file(tr, subsys_name, event_name); 3033 3034 if (IS_ERR(file)) { 3035 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3036 ret = PTR_ERR(file); 3037 return ERR_PTR(ret); 3038 } 3039 3040 /* 3041 * Look for a histogram compatible with target. We'll use the 3042 * found histogram specification to create a new matching 3043 * histogram with our variable on it. target_hist_data is not 3044 * yet a registered histogram so we can't use that. 3045 */ 3046 hist_data = find_compatible_hist(target_hist_data, file); 3047 if (!hist_data) { 3048 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3049 return ERR_PTR(-EINVAL); 3050 } 3051 3052 /* See if a synthetic field variable has already been created */ 3053 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3054 event_name, field_name); 3055 if (!IS_ERR_OR_NULL(event_var)) 3056 return event_var; 3057 3058 var_hist = kzalloc_obj(*var_hist); 3059 if (!var_hist) 3060 return ERR_PTR(-ENOMEM); 3061 3062 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3063 if (!cmd) { 3064 kfree(var_hist); 3065 return ERR_PTR(-ENOMEM); 3066 } 3067 3068 seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL); 3069 3070 /* Use the same keys as the compatible histogram */ 3071 seq_buf_puts(&s, "keys="); 3072 3073 for_each_hist_key_field(i, hist_data) { 3074 key_field = hist_data->fields[i]; 3075 if (!first) 3076 seq_buf_putc(&s, ','); 3077 seq_buf_puts(&s, key_field->field->name); 3078 first = false; 3079 } 3080 3081 /* Create the synthetic field variable specification */ 3082 seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name); 3083 3084 /* Use the same filter as the compatible histogram */ 3085 saved_filter = find_trigger_filter(hist_data, file); 3086 if (saved_filter) 3087 seq_buf_printf(&s, " if %s", saved_filter); 3088 3089 /* Terminate cmd with a NUL. */ 3090 seq_buf_str(&s); 3091 3092 if (seq_buf_has_overflowed(&s)) { 3093 kfree(cmd); 3094 kfree(var_hist); 3095 return ERR_PTR(-E2BIG); 3096 } 3097 3098 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3099 if (!var_hist->cmd) { 3100 kfree(cmd); 3101 kfree(var_hist); 3102 return ERR_PTR(-ENOMEM); 3103 } 3104 3105 /* Save the compatible histogram information */ 3106 var_hist->hist_data = hist_data; 3107 3108 /* Create the new histogram with our variable */ 3109 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 3110 "", "hist", cmd); 3111 if (ret) { 3112 kfree(cmd); 3113 kfree(var_hist->cmd); 3114 kfree(var_hist); 3115 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3116 return ERR_PTR(ret); 3117 } 3118 3119 kfree(cmd); 3120 3121 /* If we can't find the variable, something went wrong */ 3122 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3123 event_name, field_name); 3124 if (IS_ERR_OR_NULL(event_var)) { 3125 kfree(var_hist->cmd); 3126 kfree(var_hist); 3127 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3128 return ERR_PTR(-EINVAL); 3129 } 3130 3131 n = target_hist_data->n_field_var_hists; 3132 target_hist_data->field_var_hists[n] = var_hist; 3133 target_hist_data->n_field_var_hists++; 3134 3135 return event_var; 3136 } 3137 3138 static struct hist_field * 3139 find_target_event_var(struct hist_trigger_data *hist_data, 3140 char *subsys_name, char *event_name, char *var_name) 3141 { 3142 struct trace_event_file *file = hist_data->event_file; 3143 struct hist_field *hist_field = NULL; 3144 3145 if (subsys_name) { 3146 struct trace_event_call *call; 3147 3148 if (!event_name) 3149 return NULL; 3150 3151 call = file->event_call; 3152 3153 if (strcmp(subsys_name, call->class->system) != 0) 3154 return NULL; 3155 3156 if (strcmp(event_name, trace_event_name(call)) != 0) 3157 return NULL; 3158 } 3159 3160 hist_field = find_var_field(hist_data, var_name); 3161 3162 return hist_field; 3163 } 3164 3165 static inline void __update_field_vars(struct tracing_map_elt *elt, 3166 struct trace_buffer *buffer, 3167 struct ring_buffer_event *rbe, 3168 void *rec, 3169 struct field_var **field_vars, 3170 unsigned int n_field_vars, 3171 unsigned int field_var_str_start) 3172 { 3173 struct hist_elt_data *elt_data = elt->private_data; 3174 unsigned int i, j, var_idx; 3175 u64 var_val; 3176 3177 /* Make sure stacktrace can fit in the string variable length */ 3178 BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) > STR_VAR_LEN_MAX); 3179 3180 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3181 struct field_var *field_var = field_vars[i]; 3182 struct hist_field *var = field_var->var; 3183 struct hist_field *val = field_var->val; 3184 3185 var_val = hist_fn_call(val, elt, buffer, rbe, rec); 3186 var_idx = var->var.idx; 3187 3188 if (val->flags & (HIST_FIELD_FL_STRING | 3189 HIST_FIELD_FL_STACKTRACE)) { 3190 char *str = elt_data->field_var_str[j++]; 3191 char *val_str = (char *)(uintptr_t)var_val; 3192 unsigned int size; 3193 3194 if (val->flags & HIST_FIELD_FL_STRING) { 3195 size = min(val->size, STR_VAR_LEN_MAX); 3196 strscpy(str, val_str, size); 3197 } else { 3198 char *stack_start = str + sizeof(unsigned long); 3199 int e; 3200 3201 e = stack_trace_save((void *)stack_start, 3202 HIST_STACKTRACE_DEPTH, 3203 HIST_STACKTRACE_SKIP); 3204 if (e < HIST_STACKTRACE_DEPTH - 1) 3205 ((unsigned long *)stack_start)[e] = 0; 3206 *((unsigned long *)str) = e; 3207 } 3208 var_val = (u64)(uintptr_t)str; 3209 } 3210 tracing_map_set_var(elt, var_idx, var_val); 3211 } 3212 } 3213 3214 static void update_field_vars(struct hist_trigger_data *hist_data, 3215 struct tracing_map_elt *elt, 3216 struct trace_buffer *buffer, 3217 struct ring_buffer_event *rbe, 3218 void *rec) 3219 { 3220 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars, 3221 hist_data->n_field_vars, 0); 3222 } 3223 3224 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3225 struct tracing_map_elt *elt, 3226 struct trace_buffer *buffer, void *rec, 3227 struct ring_buffer_event *rbe, void *key, 3228 struct action_data *data, u64 *var_ref_vals) 3229 { 3230 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars, 3231 hist_data->n_save_vars, hist_data->n_field_var_str); 3232 } 3233 3234 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3235 struct trace_event_file *file, 3236 char *name, int size, const char *type) 3237 { 3238 struct hist_field *var; 3239 int idx; 3240 3241 if (find_var(hist_data, file, name) && !hist_data->remove) { 3242 var = ERR_PTR(-EINVAL); 3243 goto out; 3244 } 3245 3246 var = kzalloc_obj(struct hist_field); 3247 if (!var) { 3248 var = ERR_PTR(-ENOMEM); 3249 goto out; 3250 } 3251 3252 idx = tracing_map_add_var(hist_data->map); 3253 if (idx < 0) { 3254 kfree(var); 3255 var = ERR_PTR(-EINVAL); 3256 goto out; 3257 } 3258 3259 var->ref = 1; 3260 var->flags = HIST_FIELD_FL_VAR; 3261 var->var.idx = idx; 3262 var->var.hist_data = var->hist_data = hist_data; 3263 var->size = size; 3264 var->var.name = kstrdup(name, GFP_KERNEL); 3265 var->type = kstrdup_const(type, GFP_KERNEL); 3266 if (!var->var.name || !var->type) { 3267 kfree_const(var->type); 3268 kfree(var->var.name); 3269 kfree(var); 3270 var = ERR_PTR(-ENOMEM); 3271 } 3272 out: 3273 return var; 3274 } 3275 3276 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3277 struct trace_event_file *file, 3278 char *field_name) 3279 { 3280 struct hist_field *val = NULL, *var = NULL; 3281 unsigned long flags = HIST_FIELD_FL_VAR; 3282 struct trace_array *tr = file->tr; 3283 struct field_var *field_var; 3284 int ret = 0; 3285 3286 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3287 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3288 ret = -EINVAL; 3289 goto err; 3290 } 3291 3292 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3293 if (IS_ERR(val)) { 3294 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3295 ret = PTR_ERR(val); 3296 goto err; 3297 } 3298 3299 var = create_var(hist_data, file, field_name, val->size, val->type); 3300 if (IS_ERR(var)) { 3301 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3302 destroy_hist_field(val, 0); 3303 ret = PTR_ERR(var); 3304 goto err; 3305 } 3306 3307 field_var = kzalloc_obj(struct field_var); 3308 if (!field_var) { 3309 destroy_hist_field(val, 0); 3310 kfree_const(var->type); 3311 kfree(var->var.name); 3312 kfree(var); 3313 ret = -ENOMEM; 3314 goto err; 3315 } 3316 3317 field_var->var = var; 3318 field_var->val = val; 3319 out: 3320 return field_var; 3321 err: 3322 field_var = ERR_PTR(ret); 3323 goto out; 3324 } 3325 3326 /** 3327 * create_target_field_var - Automatically create a variable for a field 3328 * @target_hist_data: The target hist trigger 3329 * @subsys_name: Optional subsystem name 3330 * @event_name: Optional event name 3331 * @var_name: The name of the field (and the resulting variable) 3332 * 3333 * Hist trigger actions fetch data from variables, not directly from 3334 * events. However, for convenience, users are allowed to directly 3335 * specify an event field in an action, which will be automatically 3336 * converted into a variable on their behalf. 3337 * 3338 * This function creates a field variable with the name var_name on 3339 * the hist trigger currently being defined on the target event. If 3340 * subsys_name and event_name are specified, this function simply 3341 * verifies that they do in fact match the target event subsystem and 3342 * event name. 3343 * 3344 * Return: The variable created for the field. 3345 */ 3346 static struct field_var * 3347 create_target_field_var(struct hist_trigger_data *target_hist_data, 3348 char *subsys_name, char *event_name, char *var_name) 3349 { 3350 struct trace_event_file *file = target_hist_data->event_file; 3351 3352 if (subsys_name) { 3353 struct trace_event_call *call; 3354 3355 if (!event_name) 3356 return NULL; 3357 3358 call = file->event_call; 3359 3360 if (strcmp(subsys_name, call->class->system) != 0) 3361 return NULL; 3362 3363 if (strcmp(event_name, trace_event_name(call)) != 0) 3364 return NULL; 3365 } 3366 3367 return create_field_var(target_hist_data, file, var_name); 3368 } 3369 3370 static bool check_track_val_max(u64 track_val, u64 var_val) 3371 { 3372 if (var_val <= track_val) 3373 return false; 3374 3375 return true; 3376 } 3377 3378 static bool check_track_val_changed(u64 track_val, u64 var_val) 3379 { 3380 if (var_val == track_val) 3381 return false; 3382 3383 return true; 3384 } 3385 3386 static u64 get_track_val(struct hist_trigger_data *hist_data, 3387 struct tracing_map_elt *elt, 3388 struct action_data *data) 3389 { 3390 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3391 u64 track_val; 3392 3393 track_val = tracing_map_read_var(elt, track_var_idx); 3394 3395 return track_val; 3396 } 3397 3398 static void save_track_val(struct hist_trigger_data *hist_data, 3399 struct tracing_map_elt *elt, 3400 struct action_data *data, u64 var_val) 3401 { 3402 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3403 3404 tracing_map_set_var(elt, track_var_idx, var_val); 3405 } 3406 3407 static void save_track_data(struct hist_trigger_data *hist_data, 3408 struct tracing_map_elt *elt, 3409 struct trace_buffer *buffer, void *rec, 3410 struct ring_buffer_event *rbe, void *key, 3411 struct action_data *data, u64 *var_ref_vals) 3412 { 3413 if (data->track_data.save_data) 3414 data->track_data.save_data(hist_data, elt, buffer, rec, rbe, 3415 key, data, var_ref_vals); 3416 } 3417 3418 static bool check_track_val(struct tracing_map_elt *elt, 3419 struct action_data *data, 3420 u64 var_val) 3421 { 3422 struct hist_trigger_data *hist_data; 3423 u64 track_val; 3424 3425 hist_data = data->track_data.track_var->hist_data; 3426 track_val = get_track_val(hist_data, elt, data); 3427 3428 return data->track_data.check_val(track_val, var_val); 3429 } 3430 3431 #ifdef CONFIG_TRACER_SNAPSHOT 3432 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3433 { 3434 /* called with tr->max_lock held */ 3435 struct track_data *track_data = tr->cond_snapshot->cond_data; 3436 struct hist_elt_data *elt_data, *track_elt_data; 3437 struct snapshot_context *context = cond_data; 3438 struct action_data *action; 3439 u64 track_val; 3440 3441 if (!track_data) 3442 return false; 3443 3444 action = track_data->action_data; 3445 3446 track_val = get_track_val(track_data->hist_data, context->elt, 3447 track_data->action_data); 3448 3449 if (!action->track_data.check_val(track_data->track_val, track_val)) 3450 return false; 3451 3452 track_data->track_val = track_val; 3453 memcpy(track_data->key, context->key, track_data->key_len); 3454 3455 elt_data = context->elt->private_data; 3456 track_elt_data = track_data->elt.private_data; 3457 if (elt_data->comm) 3458 strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3459 3460 track_data->updated = true; 3461 3462 return true; 3463 } 3464 3465 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3466 struct tracing_map_elt *elt, 3467 struct trace_buffer *buffer, void *rec, 3468 struct ring_buffer_event *rbe, void *key, 3469 struct action_data *data, 3470 u64 *var_ref_vals) 3471 { 3472 struct trace_event_file *file = hist_data->event_file; 3473 struct snapshot_context context; 3474 3475 context.elt = elt; 3476 context.key = key; 3477 3478 tracing_snapshot_cond(file->tr, &context); 3479 } 3480 3481 static void hist_trigger_print_key(struct seq_file *m, 3482 struct hist_trigger_data *hist_data, 3483 void *key, 3484 struct tracing_map_elt *elt); 3485 3486 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3487 { 3488 unsigned int i; 3489 3490 if (!hist_data->n_actions) 3491 return NULL; 3492 3493 for (i = 0; i < hist_data->n_actions; i++) { 3494 struct action_data *data = hist_data->actions[i]; 3495 3496 if (data->action == ACTION_SNAPSHOT) 3497 return data; 3498 } 3499 3500 return NULL; 3501 } 3502 3503 static void track_data_snapshot_print(struct seq_file *m, 3504 struct hist_trigger_data *hist_data) 3505 { 3506 struct trace_event_file *file = hist_data->event_file; 3507 struct track_data *track_data; 3508 struct action_data *action; 3509 3510 track_data = tracing_cond_snapshot_data(file->tr); 3511 if (!track_data) 3512 return; 3513 3514 if (!track_data->updated) 3515 return; 3516 3517 action = snapshot_action(hist_data); 3518 if (!action) 3519 return; 3520 3521 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3522 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3523 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3524 action->track_data.var_str, track_data->track_val); 3525 3526 seq_puts(m, "\ttriggered by event with key: "); 3527 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3528 seq_putc(m, '\n'); 3529 } 3530 #else 3531 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3532 { 3533 return false; 3534 } 3535 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3536 struct tracing_map_elt *elt, 3537 struct trace_buffer *buffer, void *rec, 3538 struct ring_buffer_event *rbe, void *key, 3539 struct action_data *data, 3540 u64 *var_ref_vals) {} 3541 static void track_data_snapshot_print(struct seq_file *m, 3542 struct hist_trigger_data *hist_data) {} 3543 #endif /* CONFIG_TRACER_SNAPSHOT */ 3544 3545 static void track_data_print(struct seq_file *m, 3546 struct hist_trigger_data *hist_data, 3547 struct tracing_map_elt *elt, 3548 struct action_data *data) 3549 { 3550 u64 track_val = get_track_val(hist_data, elt, data); 3551 unsigned int i, save_var_idx; 3552 3553 if (data->handler == HANDLER_ONMAX) 3554 seq_printf(m, "\n\tmax: %10llu", track_val); 3555 else if (data->handler == HANDLER_ONCHANGE) 3556 seq_printf(m, "\n\tchanged: %10llu", track_val); 3557 3558 if (data->action == ACTION_SNAPSHOT) 3559 return; 3560 3561 for (i = 0; i < hist_data->n_save_vars; i++) { 3562 struct hist_field *save_val = hist_data->save_vars[i]->val; 3563 struct hist_field *save_var = hist_data->save_vars[i]->var; 3564 u64 val; 3565 3566 save_var_idx = save_var->var.idx; 3567 3568 val = tracing_map_read_var(elt, save_var_idx); 3569 3570 if (save_val->flags & HIST_FIELD_FL_STRING) { 3571 seq_printf(m, " %s: %-32s", save_var->var.name, 3572 (char *)(uintptr_t)(val)); 3573 } else 3574 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3575 } 3576 } 3577 3578 static void ontrack_action(struct hist_trigger_data *hist_data, 3579 struct tracing_map_elt *elt, 3580 struct trace_buffer *buffer, void *rec, 3581 struct ring_buffer_event *rbe, void *key, 3582 struct action_data *data, u64 *var_ref_vals) 3583 { 3584 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3585 3586 if (check_track_val(elt, data, var_val)) { 3587 save_track_val(hist_data, elt, data, var_val); 3588 save_track_data(hist_data, elt, buffer, rec, rbe, 3589 key, data, var_ref_vals); 3590 } 3591 } 3592 3593 static void action_data_destroy(struct action_data *data) 3594 { 3595 unsigned int i; 3596 3597 lockdep_assert_held(&event_mutex); 3598 3599 kfree(data->action_name); 3600 3601 for (i = 0; i < data->n_params; i++) 3602 kfree(data->params[i]); 3603 3604 if (data->synth_event) 3605 data->synth_event->ref--; 3606 3607 kfree(data->synth_event_name); 3608 3609 kfree(data); 3610 } 3611 3612 static void track_data_destroy(struct hist_trigger_data *hist_data, 3613 struct action_data *data) 3614 { 3615 struct trace_event_file *file = hist_data->event_file; 3616 3617 destroy_hist_field(data->track_data.track_var, 0); 3618 3619 if (data->action == ACTION_SNAPSHOT) { 3620 struct track_data *track_data; 3621 3622 track_data = tracing_cond_snapshot_data(file->tr); 3623 if (track_data && track_data->hist_data == hist_data) { 3624 tracing_snapshot_cond_disable(file->tr); 3625 track_data_free(track_data); 3626 } 3627 } 3628 3629 kfree(data->track_data.var_str); 3630 3631 action_data_destroy(data); 3632 } 3633 3634 static int action_create(struct hist_trigger_data *hist_data, 3635 struct action_data *data); 3636 3637 static int track_data_create(struct hist_trigger_data *hist_data, 3638 struct action_data *data) 3639 { 3640 struct hist_field *var_field, *ref_field, *track_var = NULL; 3641 struct trace_event_file *file = hist_data->event_file; 3642 struct trace_array *tr = file->tr; 3643 char *track_data_var_str; 3644 int ret = 0; 3645 3646 track_data_var_str = data->track_data.var_str; 3647 if (track_data_var_str[0] != '$') { 3648 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3649 return -EINVAL; 3650 } 3651 track_data_var_str++; 3652 3653 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3654 if (!var_field) { 3655 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3656 return -EINVAL; 3657 } 3658 3659 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3660 if (!ref_field) 3661 return -ENOMEM; 3662 3663 data->track_data.var_ref = ref_field; 3664 3665 if (data->handler == HANDLER_ONMAX) 3666 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3667 if (IS_ERR(track_var)) { 3668 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3669 ret = PTR_ERR(track_var); 3670 goto out; 3671 } 3672 3673 if (data->handler == HANDLER_ONCHANGE) 3674 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3675 if (IS_ERR(track_var)) { 3676 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3677 ret = PTR_ERR(track_var); 3678 goto out; 3679 } 3680 data->track_data.track_var = track_var; 3681 3682 ret = action_create(hist_data, data); 3683 out: 3684 return ret; 3685 } 3686 3687 static int parse_action_params(struct trace_array *tr, char *params, 3688 struct action_data *data) 3689 { 3690 char *param, *saved_param; 3691 bool first_param = true; 3692 int ret = 0; 3693 3694 while (params) { 3695 if (data->n_params >= SYNTH_FIELDS_MAX) { 3696 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3697 ret = -EINVAL; 3698 goto out; 3699 } 3700 3701 param = strsep(¶ms, ","); 3702 if (!param) { 3703 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3704 ret = -EINVAL; 3705 goto out; 3706 } 3707 3708 param = strstrip(param); 3709 if (strlen(param) < 2) { 3710 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3711 ret = -EINVAL; 3712 goto out; 3713 } 3714 3715 saved_param = kstrdup(param, GFP_KERNEL); 3716 if (!saved_param) { 3717 ret = -ENOMEM; 3718 goto out; 3719 } 3720 3721 if (first_param && data->use_trace_keyword) { 3722 data->synth_event_name = saved_param; 3723 first_param = false; 3724 continue; 3725 } 3726 first_param = false; 3727 3728 data->params[data->n_params++] = saved_param; 3729 } 3730 out: 3731 return ret; 3732 } 3733 3734 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3735 enum handler_id handler) 3736 { 3737 char *action_name; 3738 int ret = 0; 3739 3740 strsep(&str, "."); 3741 if (!str) { 3742 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3743 ret = -EINVAL; 3744 goto out; 3745 } 3746 3747 action_name = strsep(&str, "("); 3748 if (!action_name || !str) { 3749 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3750 ret = -EINVAL; 3751 goto out; 3752 } 3753 3754 if (str_has_prefix(action_name, "save")) { 3755 char *params = strsep(&str, ")"); 3756 3757 if (!params) { 3758 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3759 ret = -EINVAL; 3760 goto out; 3761 } 3762 3763 ret = parse_action_params(tr, params, data); 3764 if (ret) 3765 goto out; 3766 3767 if (handler == HANDLER_ONMAX) 3768 data->track_data.check_val = check_track_val_max; 3769 else if (handler == HANDLER_ONCHANGE) 3770 data->track_data.check_val = check_track_val_changed; 3771 else { 3772 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3773 ret = -EINVAL; 3774 goto out; 3775 } 3776 3777 data->track_data.save_data = save_track_data_vars; 3778 data->fn = ontrack_action; 3779 data->action = ACTION_SAVE; 3780 } else if (str_has_prefix(action_name, "snapshot")) { 3781 char *params = strsep(&str, ")"); 3782 3783 if (!str) { 3784 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3785 ret = -EINVAL; 3786 goto out; 3787 } 3788 3789 if (handler == HANDLER_ONMAX) 3790 data->track_data.check_val = check_track_val_max; 3791 else if (handler == HANDLER_ONCHANGE) 3792 data->track_data.check_val = check_track_val_changed; 3793 else { 3794 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3795 ret = -EINVAL; 3796 goto out; 3797 } 3798 3799 data->track_data.save_data = save_track_data_snapshot; 3800 data->fn = ontrack_action; 3801 data->action = ACTION_SNAPSHOT; 3802 } else { 3803 char *params = strsep(&str, ")"); 3804 3805 if (str_has_prefix(action_name, "trace")) 3806 data->use_trace_keyword = true; 3807 3808 if (params) { 3809 ret = parse_action_params(tr, params, data); 3810 if (ret) 3811 goto out; 3812 } 3813 3814 if (handler == HANDLER_ONMAX) 3815 data->track_data.check_val = check_track_val_max; 3816 else if (handler == HANDLER_ONCHANGE) 3817 data->track_data.check_val = check_track_val_changed; 3818 3819 if (handler != HANDLER_ONMATCH) { 3820 data->track_data.save_data = action_trace; 3821 data->fn = ontrack_action; 3822 } else 3823 data->fn = action_trace; 3824 3825 data->action = ACTION_TRACE; 3826 } 3827 3828 data->action_name = kstrdup(action_name, GFP_KERNEL); 3829 if (!data->action_name) { 3830 ret = -ENOMEM; 3831 goto out; 3832 } 3833 3834 data->handler = handler; 3835 out: 3836 return ret; 3837 } 3838 3839 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3840 char *str, enum handler_id handler) 3841 { 3842 struct action_data *data; 3843 int ret = -EINVAL; 3844 char *var_str; 3845 3846 data = kzalloc_obj(*data); 3847 if (!data) 3848 return ERR_PTR(-ENOMEM); 3849 3850 var_str = strsep(&str, ")"); 3851 if (!var_str || !str) { 3852 ret = -EINVAL; 3853 goto free; 3854 } 3855 3856 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3857 if (!data->track_data.var_str) { 3858 ret = -ENOMEM; 3859 goto free; 3860 } 3861 3862 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3863 if (ret) 3864 goto free; 3865 out: 3866 return data; 3867 free: 3868 track_data_destroy(hist_data, data); 3869 data = ERR_PTR(ret); 3870 goto out; 3871 } 3872 3873 static void onmatch_destroy(struct action_data *data) 3874 { 3875 kfree(data->match_data.event); 3876 kfree(data->match_data.event_system); 3877 3878 action_data_destroy(data); 3879 } 3880 3881 static void destroy_field_var(struct field_var *field_var) 3882 { 3883 if (!field_var) 3884 return; 3885 3886 destroy_hist_field(field_var->var, 0); 3887 destroy_hist_field(field_var->val, 0); 3888 3889 kfree(field_var); 3890 } 3891 3892 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3893 { 3894 unsigned int i; 3895 3896 for (i = 0; i < hist_data->n_field_vars; i++) 3897 destroy_field_var(hist_data->field_vars[i]); 3898 3899 for (i = 0; i < hist_data->n_save_vars; i++) 3900 destroy_field_var(hist_data->save_vars[i]); 3901 } 3902 3903 static void save_field_var(struct hist_trigger_data *hist_data, 3904 struct field_var *field_var) 3905 { 3906 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3907 3908 /* Stack traces are saved in the string storage too */ 3909 if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 3910 hist_data->n_field_var_str++; 3911 } 3912 3913 3914 static int check_synth_field(struct synth_event *event, 3915 struct hist_field *hist_field, 3916 unsigned int field_pos) 3917 { 3918 struct synth_field *field; 3919 3920 if (field_pos >= event->n_fields) 3921 return -EINVAL; 3922 3923 field = event->fields[field_pos]; 3924 3925 /* 3926 * A dynamic string synth field can accept static or 3927 * dynamic. A static string synth field can only accept a 3928 * same-sized static string, which is checked for later. 3929 */ 3930 if (strstr(hist_field->type, "char[") && field->is_string 3931 && field->is_dynamic) 3932 return 0; 3933 3934 if (strstr(hist_field->type, "long[") && field->is_stack) 3935 return 0; 3936 3937 if (strcmp(field->type, hist_field->type) != 0) { 3938 if (field->size != hist_field->size || 3939 (!field->is_string && field->is_signed != hist_field->is_signed)) 3940 return -EINVAL; 3941 } 3942 3943 return 0; 3944 } 3945 3946 static struct hist_field * 3947 trace_action_find_var(struct hist_trigger_data *hist_data, 3948 struct action_data *data, 3949 char *system, char *event, char *var) 3950 { 3951 struct trace_array *tr = hist_data->event_file->tr; 3952 struct hist_field *hist_field; 3953 3954 var++; /* skip '$' */ 3955 3956 hist_field = find_target_event_var(hist_data, system, event, var); 3957 if (!hist_field) { 3958 if (!system && data->handler == HANDLER_ONMATCH) { 3959 system = data->match_data.event_system; 3960 event = data->match_data.event; 3961 } 3962 3963 hist_field = find_event_var(hist_data, system, event, var); 3964 } 3965 3966 if (!hist_field) 3967 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3968 3969 return hist_field; 3970 } 3971 3972 static struct hist_field * 3973 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3974 struct action_data *data, char *system, 3975 char *event, char *var) 3976 { 3977 struct hist_field *hist_field = NULL; 3978 struct field_var *field_var; 3979 3980 /* 3981 * First try to create a field var on the target event (the 3982 * currently being defined). This will create a variable for 3983 * unqualified fields on the target event, or if qualified, 3984 * target fields that have qualified names matching the target. 3985 */ 3986 field_var = create_target_field_var(hist_data, system, event, var); 3987 3988 if (field_var && !IS_ERR(field_var)) { 3989 save_field_var(hist_data, field_var); 3990 hist_field = field_var->var; 3991 } else { 3992 field_var = NULL; 3993 /* 3994 * If no explicit system.event is specified, default to 3995 * looking for fields on the onmatch(system.event.xxx) 3996 * event. 3997 */ 3998 if (!system && data->handler == HANDLER_ONMATCH) { 3999 system = data->match_data.event_system; 4000 event = data->match_data.event; 4001 } 4002 4003 if (!event) 4004 goto free; 4005 /* 4006 * At this point, we're looking at a field on another 4007 * event. Because we can't modify a hist trigger on 4008 * another event to add a variable for a field, we need 4009 * to create a new trigger on that event and create the 4010 * variable at the same time. 4011 */ 4012 hist_field = create_field_var_hist(hist_data, system, event, var); 4013 if (IS_ERR(hist_field)) 4014 goto free; 4015 } 4016 out: 4017 return hist_field; 4018 free: 4019 destroy_field_var(field_var); 4020 hist_field = NULL; 4021 goto out; 4022 } 4023 4024 static int trace_action_create(struct hist_trigger_data *hist_data, 4025 struct action_data *data) 4026 { 4027 struct trace_array *tr = hist_data->event_file->tr; 4028 char *event_name, *param, *system = NULL; 4029 struct hist_field *hist_field, *var_ref; 4030 unsigned int i; 4031 unsigned int field_pos = 0; 4032 struct synth_event *event; 4033 char *synth_event_name; 4034 int var_ref_idx, ret = 0; 4035 4036 lockdep_assert_held(&event_mutex); 4037 4038 /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */ 4039 if (data->n_params > SYNTH_FIELDS_MAX) 4040 return -EINVAL; 4041 4042 if (data->use_trace_keyword) 4043 synth_event_name = data->synth_event_name; 4044 else 4045 synth_event_name = data->action_name; 4046 4047 event = find_synth_event(synth_event_name); 4048 if (!event) { 4049 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4050 return -EINVAL; 4051 } 4052 4053 event->ref++; 4054 4055 for (i = 0; i < data->n_params; i++) { 4056 char *p; 4057 4058 p = param = kstrdup(data->params[i], GFP_KERNEL); 4059 if (!param) { 4060 ret = -ENOMEM; 4061 goto err; 4062 } 4063 4064 system = strsep(¶m, "."); 4065 if (!param) { 4066 param = (char *)system; 4067 system = event_name = NULL; 4068 } else { 4069 event_name = strsep(¶m, "."); 4070 if (!param) { 4071 kfree(p); 4072 ret = -EINVAL; 4073 goto err; 4074 } 4075 } 4076 4077 if (param[0] == '$') 4078 hist_field = trace_action_find_var(hist_data, data, 4079 system, event_name, 4080 param); 4081 else 4082 hist_field = trace_action_create_field_var(hist_data, 4083 data, 4084 system, 4085 event_name, 4086 param); 4087 4088 if (!hist_field) { 4089 kfree(p); 4090 ret = -EINVAL; 4091 goto err; 4092 } 4093 4094 if (check_synth_field(event, hist_field, field_pos) == 0) { 4095 var_ref = create_var_ref(hist_data, hist_field, 4096 system, event_name); 4097 if (!var_ref) { 4098 kfree(p); 4099 ret = -ENOMEM; 4100 goto err; 4101 } 4102 4103 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 4104 if (WARN_ON(var_ref_idx < 0)) { 4105 kfree(p); 4106 ret = var_ref_idx; 4107 goto err; 4108 } 4109 4110 data->var_ref_idx[i] = var_ref_idx; 4111 4112 field_pos++; 4113 kfree(p); 4114 continue; 4115 } 4116 4117 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4118 kfree(p); 4119 ret = -EINVAL; 4120 goto err; 4121 } 4122 4123 if (field_pos != event->n_fields) { 4124 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4125 ret = -EINVAL; 4126 goto err; 4127 } 4128 4129 data->synth_event = event; 4130 out: 4131 return ret; 4132 err: 4133 event->ref--; 4134 4135 goto out; 4136 } 4137 4138 static int action_create(struct hist_trigger_data *hist_data, 4139 struct action_data *data) 4140 { 4141 struct trace_event_file *file = hist_data->event_file; 4142 struct trace_array *tr = file->tr; 4143 struct track_data *track_data; 4144 struct field_var *field_var; 4145 unsigned int i; 4146 char *param; 4147 int ret = 0; 4148 4149 if (data->action == ACTION_TRACE) 4150 return trace_action_create(hist_data, data); 4151 4152 if (data->action == ACTION_SNAPSHOT) { 4153 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4154 if (IS_ERR(track_data)) { 4155 ret = PTR_ERR(track_data); 4156 goto out; 4157 } 4158 4159 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4160 cond_snapshot_update); 4161 if (ret) 4162 track_data_free(track_data); 4163 4164 goto out; 4165 } 4166 4167 if (data->action == ACTION_SAVE) { 4168 if (hist_data->n_save_vars) { 4169 ret = -EEXIST; 4170 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4171 goto out; 4172 } 4173 4174 for (i = 0; i < data->n_params; i++) { 4175 param = kstrdup(data->params[i], GFP_KERNEL); 4176 if (!param) { 4177 ret = -ENOMEM; 4178 goto out; 4179 } 4180 4181 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4182 if (IS_ERR(field_var)) { 4183 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4184 errpos(param)); 4185 ret = PTR_ERR(field_var); 4186 kfree(param); 4187 goto out; 4188 } 4189 4190 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4191 if (field_var->val->flags & 4192 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4193 hist_data->n_save_var_str++; 4194 kfree(param); 4195 } 4196 } 4197 out: 4198 return ret; 4199 } 4200 4201 static int onmatch_create(struct hist_trigger_data *hist_data, 4202 struct action_data *data) 4203 { 4204 return action_create(hist_data, data); 4205 } 4206 4207 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4208 { 4209 char *match_event, *match_event_system; 4210 struct action_data *data; 4211 int ret = -EINVAL; 4212 4213 data = kzalloc_obj(*data); 4214 if (!data) 4215 return ERR_PTR(-ENOMEM); 4216 4217 match_event = strsep(&str, ")"); 4218 if (!match_event || !str) { 4219 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4220 goto free; 4221 } 4222 4223 match_event_system = strsep(&match_event, "."); 4224 if (!match_event) { 4225 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4226 goto free; 4227 } 4228 4229 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4230 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4231 goto free; 4232 } 4233 4234 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4235 if (!data->match_data.event) { 4236 ret = -ENOMEM; 4237 goto free; 4238 } 4239 4240 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4241 if (!data->match_data.event_system) { 4242 ret = -ENOMEM; 4243 goto free; 4244 } 4245 4246 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4247 if (ret) 4248 goto free; 4249 out: 4250 return data; 4251 free: 4252 onmatch_destroy(data); 4253 data = ERR_PTR(ret); 4254 goto out; 4255 } 4256 4257 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4258 { 4259 hist_data->fields[HITCOUNT_IDX] = 4260 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4261 if (!hist_data->fields[HITCOUNT_IDX]) 4262 return -ENOMEM; 4263 4264 hist_data->n_vals++; 4265 hist_data->n_fields++; 4266 4267 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4268 return -EINVAL; 4269 4270 return 0; 4271 } 4272 4273 static int __create_val_field(struct hist_trigger_data *hist_data, 4274 unsigned int val_idx, 4275 struct trace_event_file *file, 4276 char *var_name, char *field_str, 4277 unsigned long flags) 4278 { 4279 struct hist_field *hist_field; 4280 int ret = 0, n_subexprs = 0; 4281 4282 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs); 4283 if (IS_ERR(hist_field)) { 4284 ret = PTR_ERR(hist_field); 4285 goto out; 4286 } 4287 4288 /* values and variables should not have some modifiers */ 4289 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4290 /* Variable */ 4291 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4292 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2)) 4293 goto err; 4294 } else { 4295 /* Value */ 4296 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4297 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | 4298 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | 4299 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) 4300 goto err; 4301 } 4302 4303 hist_data->fields[val_idx] = hist_field; 4304 4305 ++hist_data->n_vals; 4306 ++hist_data->n_fields; 4307 4308 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4309 ret = -EINVAL; 4310 out: 4311 return ret; 4312 err: 4313 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 4314 return -EINVAL; 4315 } 4316 4317 static int create_val_field(struct hist_trigger_data *hist_data, 4318 unsigned int val_idx, 4319 struct trace_event_file *file, 4320 char *field_str) 4321 { 4322 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4323 return -EINVAL; 4324 4325 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4326 } 4327 4328 static const char no_comm[] = "(no comm)"; 4329 4330 static u64 hist_field_execname(struct hist_field *hist_field, 4331 struct tracing_map_elt *elt, 4332 struct trace_buffer *buffer, 4333 struct ring_buffer_event *rbe, 4334 void *event) 4335 { 4336 struct hist_elt_data *elt_data; 4337 4338 if (WARN_ON_ONCE(!elt)) 4339 return (u64)(unsigned long)no_comm; 4340 4341 elt_data = elt->private_data; 4342 4343 if (WARN_ON_ONCE(!elt_data->comm)) 4344 return (u64)(unsigned long)no_comm; 4345 4346 return (u64)(unsigned long)(elt_data->comm); 4347 } 4348 4349 static u64 hist_field_stack(struct hist_field *hist_field, 4350 struct tracing_map_elt *elt, 4351 struct trace_buffer *buffer, 4352 struct ring_buffer_event *rbe, 4353 void *event) 4354 { 4355 u32 str_item = *(u32 *)(event + hist_field->field->offset); 4356 int str_loc = str_item & 0xffff; 4357 char *addr = (char *)(event + str_loc); 4358 4359 return (u64)(unsigned long)addr; 4360 } 4361 4362 static u64 hist_fn_call(struct hist_field *hist_field, 4363 struct tracing_map_elt *elt, 4364 struct trace_buffer *buffer, 4365 struct ring_buffer_event *rbe, 4366 void *event) 4367 { 4368 switch (hist_field->fn_num) { 4369 case HIST_FIELD_FN_VAR_REF: 4370 return hist_field_var_ref(hist_field, elt, buffer, rbe, event); 4371 case HIST_FIELD_FN_COUNTER: 4372 return hist_field_counter(hist_field, elt, buffer, rbe, event); 4373 case HIST_FIELD_FN_CONST: 4374 return hist_field_const(hist_field, elt, buffer, rbe, event); 4375 case HIST_FIELD_FN_LOG2: 4376 return hist_field_log2(hist_field, elt, buffer, rbe, event); 4377 case HIST_FIELD_FN_BUCKET: 4378 return hist_field_bucket(hist_field, elt, buffer, rbe, event); 4379 case HIST_FIELD_FN_TIMESTAMP: 4380 return hist_field_timestamp(hist_field, elt, buffer, rbe, event); 4381 case HIST_FIELD_FN_CPU: 4382 return hist_field_cpu(hist_field, elt, buffer, rbe, event); 4383 case HIST_FIELD_FN_COMM: 4384 return hist_field_comm(hist_field, elt, buffer, rbe, event); 4385 case HIST_FIELD_FN_STRING: 4386 return hist_field_string(hist_field, elt, buffer, rbe, event); 4387 case HIST_FIELD_FN_DYNSTRING: 4388 return hist_field_dynstring(hist_field, elt, buffer, rbe, event); 4389 case HIST_FIELD_FN_RELDYNSTRING: 4390 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event); 4391 case HIST_FIELD_FN_PSTRING: 4392 return hist_field_pstring(hist_field, elt, buffer, rbe, event); 4393 case HIST_FIELD_FN_S64: 4394 return hist_field_s64(hist_field, elt, buffer, rbe, event); 4395 case HIST_FIELD_FN_U64: 4396 return hist_field_u64(hist_field, elt, buffer, rbe, event); 4397 case HIST_FIELD_FN_S32: 4398 return hist_field_s32(hist_field, elt, buffer, rbe, event); 4399 case HIST_FIELD_FN_U32: 4400 return hist_field_u32(hist_field, elt, buffer, rbe, event); 4401 case HIST_FIELD_FN_S16: 4402 return hist_field_s16(hist_field, elt, buffer, rbe, event); 4403 case HIST_FIELD_FN_U16: 4404 return hist_field_u16(hist_field, elt, buffer, rbe, event); 4405 case HIST_FIELD_FN_S8: 4406 return hist_field_s8(hist_field, elt, buffer, rbe, event); 4407 case HIST_FIELD_FN_U8: 4408 return hist_field_u8(hist_field, elt, buffer, rbe, event); 4409 case HIST_FIELD_FN_UMINUS: 4410 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event); 4411 case HIST_FIELD_FN_MINUS: 4412 return hist_field_minus(hist_field, elt, buffer, rbe, event); 4413 case HIST_FIELD_FN_PLUS: 4414 return hist_field_plus(hist_field, elt, buffer, rbe, event); 4415 case HIST_FIELD_FN_DIV: 4416 return hist_field_div(hist_field, elt, buffer, rbe, event); 4417 case HIST_FIELD_FN_MULT: 4418 return hist_field_mult(hist_field, elt, buffer, rbe, event); 4419 case HIST_FIELD_FN_DIV_POWER2: 4420 return div_by_power_of_two(hist_field, elt, buffer, rbe, event); 4421 case HIST_FIELD_FN_DIV_NOT_POWER2: 4422 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event); 4423 case HIST_FIELD_FN_DIV_MULT_SHIFT: 4424 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event); 4425 case HIST_FIELD_FN_EXECNAME: 4426 return hist_field_execname(hist_field, elt, buffer, rbe, event); 4427 case HIST_FIELD_FN_STACK: 4428 return hist_field_stack(hist_field, elt, buffer, rbe, event); 4429 default: 4430 return 0; 4431 } 4432 } 4433 4434 /* Convert a var that points to common_pid.execname to a string */ 4435 static void update_var_execname(struct hist_field *hist_field) 4436 { 4437 hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR | 4438 HIST_FIELD_FL_EXECNAME; 4439 hist_field->size = MAX_FILTER_STR_VAL; 4440 hist_field->is_signed = 0; 4441 4442 kfree_const(hist_field->type); 4443 hist_field->type = "char[]"; 4444 4445 hist_field->fn_num = HIST_FIELD_FN_EXECNAME; 4446 } 4447 4448 static int create_var_field(struct hist_trigger_data *hist_data, 4449 unsigned int val_idx, 4450 struct trace_event_file *file, 4451 char *var_name, char *expr_str) 4452 { 4453 struct trace_array *tr = hist_data->event_file->tr; 4454 unsigned long flags = 0; 4455 int ret; 4456 4457 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4458 return -EINVAL; 4459 4460 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4461 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4462 return -EINVAL; 4463 } 4464 4465 flags |= HIST_FIELD_FL_VAR; 4466 hist_data->n_vars++; 4467 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4468 return -EINVAL; 4469 4470 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4471 4472 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME) 4473 update_var_execname(hist_data->fields[val_idx]); 4474 4475 if (!ret && hist_data->fields[val_idx]->flags & 4476 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4477 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; 4478 4479 return ret; 4480 } 4481 4482 static int create_val_fields(struct hist_trigger_data *hist_data, 4483 struct trace_event_file *file) 4484 { 4485 unsigned int i, j = 1, n_hitcount = 0; 4486 char *fields_str, *field_str; 4487 int ret; 4488 4489 ret = create_hitcount_val(hist_data); 4490 if (ret) 4491 goto out; 4492 4493 fields_str = hist_data->attrs->vals_str; 4494 if (!fields_str) 4495 goto out; 4496 4497 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4498 j < TRACING_MAP_VALS_MAX; i++) { 4499 field_str = strsep(&fields_str, ","); 4500 if (!field_str) 4501 break; 4502 4503 if (strcmp(field_str, "hitcount") == 0) { 4504 if (!n_hitcount++) 4505 continue; 4506 } 4507 4508 ret = create_val_field(hist_data, j++, file, field_str); 4509 if (ret) 4510 goto out; 4511 } 4512 4513 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4514 ret = -EINVAL; 4515 out: 4516 /* There is only raw hitcount but nohitcount suppresses it. */ 4517 if (j == 1 && hist_data->attrs->no_hitcount) { 4518 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0); 4519 ret = -ENOENT; 4520 } 4521 4522 return ret; 4523 } 4524 4525 static int create_key_field(struct hist_trigger_data *hist_data, 4526 unsigned int key_idx, 4527 unsigned int key_offset, 4528 struct trace_event_file *file, 4529 char *field_str) 4530 { 4531 struct trace_array *tr = hist_data->event_file->tr; 4532 struct hist_field *hist_field = NULL; 4533 unsigned long flags = 0; 4534 unsigned int key_size; 4535 int ret = 0, n_subexprs = 0; 4536 4537 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4538 return -EINVAL; 4539 4540 flags |= HIST_FIELD_FL_KEY; 4541 4542 if (strcmp(field_str, "stacktrace") == 0) { 4543 flags |= HIST_FIELD_FL_STACKTRACE; 4544 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4545 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4546 } else { 4547 hist_field = parse_expr(hist_data, file, field_str, flags, 4548 NULL, &n_subexprs); 4549 if (IS_ERR(hist_field)) { 4550 ret = PTR_ERR(hist_field); 4551 goto out; 4552 } 4553 4554 if (field_has_hist_vars(hist_field, 0)) { 4555 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4556 destroy_hist_field(hist_field, 0); 4557 ret = -EINVAL; 4558 goto out; 4559 } 4560 4561 key_size = hist_field->size; 4562 } 4563 4564 hist_data->fields[key_idx] = hist_field; 4565 4566 key_size = ALIGN(key_size, sizeof(u64)); 4567 hist_data->fields[key_idx]->size = key_size; 4568 hist_data->fields[key_idx]->offset = key_offset; 4569 4570 hist_data->key_size += key_size; 4571 4572 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4573 ret = -EINVAL; 4574 goto out; 4575 } 4576 4577 hist_data->n_keys++; 4578 hist_data->n_fields++; 4579 4580 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4581 return -EINVAL; 4582 4583 ret = key_size; 4584 out: 4585 return ret; 4586 } 4587 4588 static int create_key_fields(struct hist_trigger_data *hist_data, 4589 struct trace_event_file *file) 4590 { 4591 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4592 char *fields_str, *field_str; 4593 int ret = -EINVAL; 4594 4595 fields_str = hist_data->attrs->keys_str; 4596 if (!fields_str) 4597 goto out; 4598 4599 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4600 field_str = strsep(&fields_str, ","); 4601 if (!field_str) 4602 break; 4603 ret = create_key_field(hist_data, i, key_offset, 4604 file, field_str); 4605 if (ret < 0) 4606 goto out; 4607 key_offset += ret; 4608 } 4609 if (fields_str) { 4610 ret = -EINVAL; 4611 goto out; 4612 } 4613 ret = 0; 4614 out: 4615 return ret; 4616 } 4617 4618 static int create_var_fields(struct hist_trigger_data *hist_data, 4619 struct trace_event_file *file) 4620 { 4621 unsigned int i, j = hist_data->n_vals; 4622 int ret = 0; 4623 4624 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4625 4626 for (i = 0; i < n_vars; i++) { 4627 char *var_name = hist_data->attrs->var_defs.name[i]; 4628 char *expr = hist_data->attrs->var_defs.expr[i]; 4629 4630 ret = create_var_field(hist_data, j++, file, var_name, expr); 4631 if (ret) 4632 goto out; 4633 } 4634 out: 4635 return ret; 4636 } 4637 4638 static void free_var_defs(struct hist_trigger_data *hist_data) 4639 { 4640 unsigned int i; 4641 4642 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4643 kfree(hist_data->attrs->var_defs.name[i]); 4644 kfree(hist_data->attrs->var_defs.expr[i]); 4645 } 4646 4647 hist_data->attrs->var_defs.n_vars = 0; 4648 } 4649 4650 static int parse_var_defs(struct hist_trigger_data *hist_data) 4651 { 4652 struct trace_array *tr = hist_data->event_file->tr; 4653 char *s, *str, *var_name, *field_str; 4654 unsigned int i, j, n_vars = 0; 4655 int ret = 0; 4656 4657 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4658 str = hist_data->attrs->assignment_str[i]; 4659 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4660 field_str = strsep(&str, ","); 4661 if (!field_str) 4662 break; 4663 4664 var_name = strsep(&field_str, "="); 4665 if (!var_name || !field_str) { 4666 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4667 errpos(var_name)); 4668 ret = -EINVAL; 4669 goto free; 4670 } 4671 4672 if (n_vars == TRACING_MAP_VARS_MAX) { 4673 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4674 ret = -EINVAL; 4675 goto free; 4676 } 4677 4678 s = kstrdup(var_name, GFP_KERNEL); 4679 if (!s) { 4680 ret = -ENOMEM; 4681 goto free; 4682 } 4683 hist_data->attrs->var_defs.name[n_vars] = s; 4684 4685 s = kstrdup(field_str, GFP_KERNEL); 4686 if (!s) { 4687 kfree(hist_data->attrs->var_defs.name[n_vars]); 4688 hist_data->attrs->var_defs.name[n_vars] = NULL; 4689 ret = -ENOMEM; 4690 goto free; 4691 } 4692 hist_data->attrs->var_defs.expr[n_vars++] = s; 4693 4694 hist_data->attrs->var_defs.n_vars = n_vars; 4695 } 4696 } 4697 4698 return ret; 4699 free: 4700 free_var_defs(hist_data); 4701 4702 return ret; 4703 } 4704 4705 static int create_hist_fields(struct hist_trigger_data *hist_data, 4706 struct trace_event_file *file) 4707 { 4708 int ret; 4709 4710 ret = parse_var_defs(hist_data); 4711 if (ret) 4712 return ret; 4713 4714 ret = create_val_fields(hist_data, file); 4715 if (ret) 4716 goto out; 4717 4718 ret = create_var_fields(hist_data, file); 4719 if (ret) 4720 goto out; 4721 4722 ret = create_key_fields(hist_data, file); 4723 4724 out: 4725 free_var_defs(hist_data); 4726 4727 return ret; 4728 } 4729 4730 static int is_descending(struct trace_array *tr, const char *str) 4731 { 4732 if (!str) 4733 return 0; 4734 4735 if (strcmp(str, "descending") == 0) 4736 return 1; 4737 4738 if (strcmp(str, "ascending") == 0) 4739 return 0; 4740 4741 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 4742 4743 return -EINVAL; 4744 } 4745 4746 static int create_sort_keys(struct hist_trigger_data *hist_data) 4747 { 4748 struct trace_array *tr = hist_data->event_file->tr; 4749 char *fields_str = hist_data->attrs->sort_key_str; 4750 struct tracing_map_sort_key *sort_key; 4751 int descending, ret = 0; 4752 unsigned int i, j, k; 4753 4754 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4755 4756 if (!fields_str) 4757 goto out; 4758 4759 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4760 struct hist_field *hist_field; 4761 char *field_str, *field_name; 4762 const char *test_name; 4763 4764 sort_key = &hist_data->sort_keys[i]; 4765 4766 field_str = strsep(&fields_str, ","); 4767 if (!field_str) 4768 break; 4769 4770 if (!*field_str) { 4771 ret = -EINVAL; 4772 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4773 break; 4774 } 4775 4776 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4777 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 4778 ret = -EINVAL; 4779 break; 4780 } 4781 4782 field_name = strsep(&field_str, "."); 4783 if (!field_name || !*field_name) { 4784 ret = -EINVAL; 4785 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4786 break; 4787 } 4788 4789 if (strcmp(field_name, "hitcount") == 0) { 4790 descending = is_descending(tr, field_str); 4791 if (descending < 0) { 4792 ret = descending; 4793 break; 4794 } 4795 sort_key->descending = descending; 4796 continue; 4797 } 4798 4799 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4800 unsigned int idx; 4801 4802 hist_field = hist_data->fields[j]; 4803 if (hist_field->flags & HIST_FIELD_FL_VAR) 4804 continue; 4805 4806 idx = k++; 4807 4808 test_name = hist_field_name(hist_field, 0); 4809 4810 if (strcmp(field_name, test_name) == 0) { 4811 sort_key->field_idx = idx; 4812 descending = is_descending(tr, field_str); 4813 if (descending < 0) { 4814 ret = descending; 4815 goto out; 4816 } 4817 sort_key->descending = descending; 4818 break; 4819 } 4820 } 4821 if (j == hist_data->n_fields) { 4822 ret = -EINVAL; 4823 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4824 break; 4825 } 4826 } 4827 4828 hist_data->n_sort_keys = i; 4829 out: 4830 return ret; 4831 } 4832 4833 static void destroy_actions(struct hist_trigger_data *hist_data) 4834 { 4835 unsigned int i; 4836 4837 for (i = 0; i < hist_data->n_actions; i++) { 4838 struct action_data *data = hist_data->actions[i]; 4839 4840 if (data->handler == HANDLER_ONMATCH) 4841 onmatch_destroy(data); 4842 else if (data->handler == HANDLER_ONMAX || 4843 data->handler == HANDLER_ONCHANGE) 4844 track_data_destroy(hist_data, data); 4845 else 4846 kfree(data); 4847 } 4848 } 4849 4850 static int parse_actions(struct hist_trigger_data *hist_data) 4851 { 4852 struct trace_array *tr = hist_data->event_file->tr; 4853 struct action_data *data; 4854 unsigned int i; 4855 int ret = 0; 4856 char *str; 4857 int len; 4858 4859 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4860 enum handler_id hid = 0; 4861 char *action_str; 4862 4863 str = hist_data->attrs->action_str[i]; 4864 4865 if ((len = str_has_prefix(str, "onmatch("))) 4866 hid = HANDLER_ONMATCH; 4867 else if ((len = str_has_prefix(str, "onmax("))) 4868 hid = HANDLER_ONMAX; 4869 else if ((len = str_has_prefix(str, "onchange("))) 4870 hid = HANDLER_ONCHANGE; 4871 4872 action_str = str + len; 4873 4874 switch (hid) { 4875 case HANDLER_ONMATCH: 4876 data = onmatch_parse(tr, action_str); 4877 break; 4878 case HANDLER_ONMAX: 4879 case HANDLER_ONCHANGE: 4880 data = track_data_parse(hist_data, action_str, hid); 4881 break; 4882 default: 4883 data = ERR_PTR(-EINVAL); 4884 break; 4885 } 4886 4887 if (IS_ERR(data)) { 4888 ret = PTR_ERR(data); 4889 break; 4890 } 4891 4892 hist_data->actions[hist_data->n_actions++] = data; 4893 } 4894 4895 return ret; 4896 } 4897 4898 static int create_actions(struct hist_trigger_data *hist_data) 4899 { 4900 struct action_data *data; 4901 unsigned int i; 4902 int ret = 0; 4903 4904 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4905 data = hist_data->actions[i]; 4906 4907 if (data->handler == HANDLER_ONMATCH) { 4908 ret = onmatch_create(hist_data, data); 4909 if (ret) 4910 break; 4911 } else if (data->handler == HANDLER_ONMAX || 4912 data->handler == HANDLER_ONCHANGE) { 4913 ret = track_data_create(hist_data, data); 4914 if (ret) 4915 break; 4916 } else { 4917 ret = -EINVAL; 4918 break; 4919 } 4920 } 4921 4922 return ret; 4923 } 4924 4925 static void print_actions(struct seq_file *m, 4926 struct hist_trigger_data *hist_data, 4927 struct tracing_map_elt *elt) 4928 { 4929 unsigned int i; 4930 4931 for (i = 0; i < hist_data->n_actions; i++) { 4932 struct action_data *data = hist_data->actions[i]; 4933 4934 if (data->action == ACTION_SNAPSHOT) 4935 continue; 4936 4937 if (data->handler == HANDLER_ONMAX || 4938 data->handler == HANDLER_ONCHANGE) 4939 track_data_print(m, hist_data, elt, data); 4940 } 4941 } 4942 4943 static void print_action_spec(struct seq_file *m, 4944 struct hist_trigger_data *hist_data, 4945 struct action_data *data) 4946 { 4947 unsigned int i; 4948 4949 if (data->action == ACTION_SAVE) { 4950 for (i = 0; i < hist_data->n_save_vars; i++) { 4951 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4952 if (i < hist_data->n_save_vars - 1) 4953 seq_puts(m, ","); 4954 } 4955 } else if (data->action == ACTION_TRACE) { 4956 if (data->use_trace_keyword) 4957 seq_printf(m, "%s", data->synth_event_name); 4958 for (i = 0; i < data->n_params; i++) { 4959 if (i || data->use_trace_keyword) 4960 seq_puts(m, ","); 4961 seq_printf(m, "%s", data->params[i]); 4962 } 4963 } 4964 } 4965 4966 static void print_track_data_spec(struct seq_file *m, 4967 struct hist_trigger_data *hist_data, 4968 struct action_data *data) 4969 { 4970 if (data->handler == HANDLER_ONMAX) 4971 seq_puts(m, ":onmax("); 4972 else if (data->handler == HANDLER_ONCHANGE) 4973 seq_puts(m, ":onchange("); 4974 seq_printf(m, "%s", data->track_data.var_str); 4975 seq_printf(m, ").%s(", data->action_name); 4976 4977 print_action_spec(m, hist_data, data); 4978 4979 seq_puts(m, ")"); 4980 } 4981 4982 static void print_onmatch_spec(struct seq_file *m, 4983 struct hist_trigger_data *hist_data, 4984 struct action_data *data) 4985 { 4986 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4987 data->match_data.event); 4988 4989 seq_printf(m, "%s(", data->action_name); 4990 4991 print_action_spec(m, hist_data, data); 4992 4993 seq_puts(m, ")"); 4994 } 4995 4996 static bool actions_match(struct hist_trigger_data *hist_data, 4997 struct hist_trigger_data *hist_data_test) 4998 { 4999 unsigned int i, j; 5000 5001 if (hist_data->n_actions != hist_data_test->n_actions) 5002 return false; 5003 5004 for (i = 0; i < hist_data->n_actions; i++) { 5005 struct action_data *data = hist_data->actions[i]; 5006 struct action_data *data_test = hist_data_test->actions[i]; 5007 char *action_name, *action_name_test; 5008 5009 if (data->handler != data_test->handler) 5010 return false; 5011 if (data->action != data_test->action) 5012 return false; 5013 5014 if (data->n_params != data_test->n_params) 5015 return false; 5016 5017 for (j = 0; j < data->n_params; j++) { 5018 if (strcmp(data->params[j], data_test->params[j]) != 0) 5019 return false; 5020 } 5021 5022 if (data->use_trace_keyword) 5023 action_name = data->synth_event_name; 5024 else 5025 action_name = data->action_name; 5026 5027 if (data_test->use_trace_keyword) 5028 action_name_test = data_test->synth_event_name; 5029 else 5030 action_name_test = data_test->action_name; 5031 5032 if (strcmp(action_name, action_name_test) != 0) 5033 return false; 5034 5035 if (data->handler == HANDLER_ONMATCH) { 5036 if (strcmp(data->match_data.event_system, 5037 data_test->match_data.event_system) != 0) 5038 return false; 5039 if (strcmp(data->match_data.event, 5040 data_test->match_data.event) != 0) 5041 return false; 5042 } else if (data->handler == HANDLER_ONMAX || 5043 data->handler == HANDLER_ONCHANGE) { 5044 if (strcmp(data->track_data.var_str, 5045 data_test->track_data.var_str) != 0) 5046 return false; 5047 } 5048 } 5049 5050 return true; 5051 } 5052 5053 5054 static void print_actions_spec(struct seq_file *m, 5055 struct hist_trigger_data *hist_data) 5056 { 5057 unsigned int i; 5058 5059 for (i = 0; i < hist_data->n_actions; i++) { 5060 struct action_data *data = hist_data->actions[i]; 5061 5062 if (data->handler == HANDLER_ONMATCH) 5063 print_onmatch_spec(m, hist_data, data); 5064 else if (data->handler == HANDLER_ONMAX || 5065 data->handler == HANDLER_ONCHANGE) 5066 print_track_data_spec(m, hist_data, data); 5067 } 5068 } 5069 5070 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5071 { 5072 unsigned int i; 5073 5074 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5075 kfree(hist_data->field_var_hists[i]->cmd); 5076 kfree(hist_data->field_var_hists[i]); 5077 } 5078 } 5079 5080 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5081 { 5082 if (!hist_data) 5083 return; 5084 5085 destroy_hist_trigger_attrs(hist_data->attrs); 5086 destroy_hist_fields(hist_data); 5087 tracing_map_destroy(hist_data->map); 5088 5089 destroy_actions(hist_data); 5090 destroy_field_vars(hist_data); 5091 destroy_field_var_hists(hist_data); 5092 5093 kfree(hist_data); 5094 } 5095 5096 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5097 { 5098 struct tracing_map *map = hist_data->map; 5099 struct ftrace_event_field *field; 5100 struct hist_field *hist_field; 5101 int i, idx = 0; 5102 5103 for_each_hist_field(i, hist_data) { 5104 hist_field = hist_data->fields[i]; 5105 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5106 tracing_map_cmp_fn_t cmp_fn; 5107 5108 field = hist_field->field; 5109 5110 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5111 cmp_fn = tracing_map_cmp_none; 5112 else if (!field || hist_field->flags & HIST_FIELD_FL_CPU) 5113 cmp_fn = tracing_map_cmp_num(hist_field->size, 5114 hist_field->is_signed); 5115 else if (is_string_field(field)) 5116 cmp_fn = tracing_map_cmp_string; 5117 else 5118 cmp_fn = tracing_map_cmp_num(field->size, 5119 field->is_signed); 5120 idx = tracing_map_add_key_field(map, 5121 hist_field->offset, 5122 cmp_fn); 5123 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5124 idx = tracing_map_add_sum_field(map); 5125 5126 if (idx < 0) 5127 return idx; 5128 5129 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5130 idx = tracing_map_add_var(map); 5131 if (idx < 0) 5132 return idx; 5133 hist_field->var.idx = idx; 5134 hist_field->var.hist_data = hist_data; 5135 } 5136 } 5137 5138 return 0; 5139 } 5140 5141 static struct hist_trigger_data * 5142 create_hist_data(unsigned int map_bits, 5143 struct hist_trigger_attrs *attrs, 5144 struct trace_event_file *file, 5145 bool remove) 5146 { 5147 const struct tracing_map_ops *map_ops = NULL; 5148 struct hist_trigger_data *hist_data; 5149 int ret = 0; 5150 5151 hist_data = kzalloc_obj(*hist_data); 5152 if (!hist_data) 5153 return ERR_PTR(-ENOMEM); 5154 5155 hist_data->attrs = attrs; 5156 hist_data->remove = remove; 5157 hist_data->event_file = file; 5158 5159 ret = parse_actions(hist_data); 5160 if (ret) 5161 goto free; 5162 5163 ret = create_hist_fields(hist_data, file); 5164 if (ret) 5165 goto free; 5166 5167 ret = create_sort_keys(hist_data); 5168 if (ret) 5169 goto free; 5170 5171 map_ops = &hist_trigger_elt_data_ops; 5172 5173 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5174 map_ops, hist_data); 5175 if (IS_ERR(hist_data->map)) { 5176 ret = PTR_ERR(hist_data->map); 5177 hist_data->map = NULL; 5178 goto free; 5179 } 5180 5181 ret = create_tracing_map_fields(hist_data); 5182 if (ret) 5183 goto free; 5184 out: 5185 return hist_data; 5186 free: 5187 hist_data->attrs = NULL; 5188 5189 destroy_hist_data(hist_data); 5190 5191 hist_data = ERR_PTR(ret); 5192 5193 goto out; 5194 } 5195 5196 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5197 struct tracing_map_elt *elt, 5198 struct trace_buffer *buffer, void *rec, 5199 struct ring_buffer_event *rbe, 5200 u64 *var_ref_vals) 5201 { 5202 struct hist_elt_data *elt_data; 5203 struct hist_field *hist_field; 5204 unsigned int i, var_idx; 5205 u64 hist_val; 5206 5207 elt_data = elt->private_data; 5208 elt_data->var_ref_vals = var_ref_vals; 5209 5210 for_each_hist_val_field(i, hist_data) { 5211 hist_field = hist_data->fields[i]; 5212 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5213 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5214 var_idx = hist_field->var.idx; 5215 5216 if (hist_field->flags & 5217 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) { 5218 unsigned int str_start, var_str_idx, idx; 5219 char *str, *val_str; 5220 unsigned int size; 5221 5222 str_start = hist_data->n_field_var_str + 5223 hist_data->n_save_var_str; 5224 var_str_idx = hist_field->var_str_idx; 5225 idx = str_start + var_str_idx; 5226 5227 str = elt_data->field_var_str[idx]; 5228 val_str = (char *)(uintptr_t)hist_val; 5229 5230 if (hist_field->flags & HIST_FIELD_FL_STRING) { 5231 size = min(hist_field->size, STR_VAR_LEN_MAX); 5232 strscpy(str, val_str, size); 5233 } else { 5234 char *stack_start = str + sizeof(unsigned long); 5235 int e; 5236 5237 e = stack_trace_save((void *)stack_start, 5238 HIST_STACKTRACE_DEPTH, 5239 HIST_STACKTRACE_SKIP); 5240 if (e < HIST_STACKTRACE_DEPTH - 1) 5241 ((unsigned long *)stack_start)[e] = 0; 5242 *((unsigned long *)str) = e; 5243 } 5244 hist_val = (u64)(uintptr_t)str; 5245 } 5246 tracing_map_set_var(elt, var_idx, hist_val); 5247 continue; 5248 } 5249 tracing_map_update_sum(elt, i, hist_val); 5250 } 5251 5252 for_each_hist_key_field(i, hist_data) { 5253 hist_field = hist_data->fields[i]; 5254 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5255 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5256 var_idx = hist_field->var.idx; 5257 tracing_map_set_var(elt, var_idx, hist_val); 5258 } 5259 } 5260 5261 update_field_vars(hist_data, elt, buffer, rbe, rec); 5262 } 5263 5264 static inline void add_to_key(char *compound_key, void *key, 5265 struct hist_field *key_field, void *rec) 5266 { 5267 size_t size = key_field->size; 5268 5269 if (key_field->flags & HIST_FIELD_FL_STRING) { 5270 5271 if (key_field->flags & HIST_FIELD_FL_COMM) { 5272 size = strlen((char *)key); 5273 } else { 5274 struct ftrace_event_field *field; 5275 5276 field = key_field->field; 5277 if (field->filter_type == FILTER_DYN_STRING || 5278 field->filter_type == FILTER_RDYN_STRING) 5279 size = *(u32 *)(rec + field->offset) >> 16; 5280 else if (field->filter_type == FILTER_STATIC_STRING) 5281 size = field->size; 5282 } 5283 5284 /* ensure NULL-termination */ 5285 if (size > key_field->size - 1) 5286 size = key_field->size - 1; 5287 } 5288 memcpy(compound_key + key_field->offset, key, size); 5289 } 5290 5291 static void 5292 hist_trigger_actions(struct hist_trigger_data *hist_data, 5293 struct tracing_map_elt *elt, 5294 struct trace_buffer *buffer, void *rec, 5295 struct ring_buffer_event *rbe, void *key, 5296 u64 *var_ref_vals) 5297 { 5298 struct action_data *data; 5299 unsigned int i; 5300 5301 for (i = 0; i < hist_data->n_actions; i++) { 5302 data = hist_data->actions[i]; 5303 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals); 5304 } 5305 } 5306 5307 /* 5308 * The hist_pad structure is used to save information to create 5309 * a histogram from the histogram trigger. It's too big to store 5310 * on the stack, so when the histogram trigger is initialized 5311 * a percpu array of 4 hist_pad structures is allocated. 5312 * This will cover every context from normal, softirq, irq and NMI 5313 * in the very unlikely event that a trigger happens at each of 5314 * these contexts and interrupts a currently active trigger. 5315 */ 5316 struct hist_pad { 5317 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5318 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5319 char compound_key[HIST_KEY_SIZE_MAX]; 5320 }; 5321 5322 static struct hist_pad __percpu *hist_pads; 5323 static DEFINE_PER_CPU(int, hist_pad_cnt); 5324 static refcount_t hist_pad_ref; 5325 5326 /* One hist_pad for every context (normal, softirq, irq, NMI) */ 5327 #define MAX_HIST_CNT 4 5328 5329 static int alloc_hist_pad(void) 5330 { 5331 lockdep_assert_held(&event_mutex); 5332 5333 if (refcount_read(&hist_pad_ref)) { 5334 refcount_inc(&hist_pad_ref); 5335 return 0; 5336 } 5337 5338 hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT, 5339 __alignof__(struct hist_pad)); 5340 if (!hist_pads) 5341 return -ENOMEM; 5342 5343 refcount_set(&hist_pad_ref, 1); 5344 return 0; 5345 } 5346 5347 static void free_hist_pad(void) 5348 { 5349 lockdep_assert_held(&event_mutex); 5350 5351 if (!refcount_dec_and_test(&hist_pad_ref)) 5352 return; 5353 5354 free_percpu(hist_pads); 5355 hist_pads = NULL; 5356 } 5357 5358 static struct hist_pad *get_hist_pad(void) 5359 { 5360 struct hist_pad *hist_pad; 5361 int cnt; 5362 5363 if (WARN_ON_ONCE(!hist_pads)) 5364 return NULL; 5365 5366 preempt_disable(); 5367 5368 hist_pad = per_cpu_ptr(hist_pads, smp_processor_id()); 5369 5370 if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) { 5371 preempt_enable(); 5372 return NULL; 5373 } 5374 5375 cnt = this_cpu_inc_return(hist_pad_cnt) - 1; 5376 5377 return &hist_pad[cnt]; 5378 } 5379 5380 static void put_hist_pad(void) 5381 { 5382 this_cpu_dec(hist_pad_cnt); 5383 preempt_enable(); 5384 } 5385 5386 static void event_hist_trigger(struct event_trigger_data *data, 5387 struct trace_buffer *buffer, void *rec, 5388 struct ring_buffer_event *rbe) 5389 { 5390 struct hist_trigger_data *hist_data = data->private_data; 5391 bool use_compound_key = (hist_data->n_keys > 1); 5392 struct tracing_map_elt *elt = NULL; 5393 struct hist_field *key_field; 5394 struct hist_pad *hist_pad; 5395 u64 field_contents; 5396 void *key = NULL; 5397 unsigned int i; 5398 5399 if (unlikely(!rbe)) 5400 return; 5401 5402 hist_pad = get_hist_pad(); 5403 if (!hist_pad) 5404 return; 5405 5406 memset(hist_pad->compound_key, 0, hist_data->key_size); 5407 5408 for_each_hist_key_field(i, hist_data) { 5409 key_field = hist_data->fields[i]; 5410 5411 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5412 unsigned long *entries = hist_pad->entries; 5413 5414 memset(entries, 0, HIST_STACKTRACE_SIZE); 5415 if (key_field->field) { 5416 unsigned long *stack, n_entries; 5417 5418 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5419 stack = (unsigned long *)(long)field_contents; 5420 n_entries = *stack; 5421 memcpy(entries, ++stack, n_entries * sizeof(unsigned long)); 5422 } else { 5423 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5424 HIST_STACKTRACE_SKIP); 5425 } 5426 key = entries; 5427 } else { 5428 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5429 if (key_field->flags & HIST_FIELD_FL_STRING) { 5430 key = (void *)(unsigned long)field_contents; 5431 use_compound_key = true; 5432 } else 5433 key = (void *)&field_contents; 5434 } 5435 5436 if (use_compound_key) 5437 add_to_key(hist_pad->compound_key, key, key_field, rec); 5438 } 5439 5440 if (use_compound_key) 5441 key = hist_pad->compound_key; 5442 5443 if (hist_data->n_var_refs && 5444 !resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false)) 5445 goto out; 5446 5447 elt = tracing_map_insert(hist_data->map, key); 5448 if (!elt) 5449 goto out; 5450 5451 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals); 5452 5453 if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) { 5454 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, 5455 key, hist_pad->var_ref_vals); 5456 } 5457 5458 hist_poll_wakeup(); 5459 5460 out: 5461 put_hist_pad(); 5462 } 5463 5464 static void hist_trigger_stacktrace_print(struct seq_file *m, 5465 unsigned long *stacktrace_entries, 5466 unsigned int max_entries) 5467 { 5468 unsigned int spaces = 8; 5469 unsigned int i; 5470 5471 for (i = 0; i < max_entries; i++) { 5472 if (!stacktrace_entries[i]) 5473 return; 5474 5475 seq_printf(m, "%*c", 1 + spaces, ' '); 5476 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]); 5477 } 5478 } 5479 5480 static void hist_trigger_print_key(struct seq_file *m, 5481 struct hist_trigger_data *hist_data, 5482 void *key, 5483 struct tracing_map_elt *elt) 5484 { 5485 struct hist_field *key_field; 5486 bool multiline = false; 5487 const char *field_name; 5488 unsigned int i; 5489 u64 uval; 5490 5491 seq_puts(m, "{ "); 5492 5493 for_each_hist_key_field(i, hist_data) { 5494 key_field = hist_data->fields[i]; 5495 5496 if (i > hist_data->n_vals) 5497 seq_puts(m, ", "); 5498 5499 field_name = hist_field_name(key_field, 0); 5500 5501 if (key_field->flags & HIST_FIELD_FL_HEX) { 5502 uval = *(u64 *)(key + key_field->offset); 5503 seq_printf(m, "%s: %llx", field_name, uval); 5504 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5505 uval = *(u64 *)(key + key_field->offset); 5506 seq_printf(m, "%s: [%llx] %-45ps", field_name, 5507 uval, (void *)(uintptr_t)uval); 5508 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5509 uval = *(u64 *)(key + key_field->offset); 5510 seq_printf(m, "%s: [%llx] %-55pS", field_name, 5511 uval, (void *)(uintptr_t)uval); 5512 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5513 struct hist_elt_data *elt_data = elt->private_data; 5514 char *comm; 5515 5516 if (WARN_ON_ONCE(!elt_data)) 5517 return; 5518 5519 comm = elt_data->comm; 5520 5521 uval = *(u64 *)(key + key_field->offset); 5522 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5523 comm, uval); 5524 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5525 const char *syscall_name; 5526 5527 uval = *(u64 *)(key + key_field->offset); 5528 syscall_name = get_syscall_name(uval); 5529 if (!syscall_name) 5530 syscall_name = "unknown_syscall"; 5531 5532 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5533 syscall_name, uval); 5534 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5535 if (key_field->field) 5536 seq_printf(m, "%s.stacktrace", key_field->field->name); 5537 else 5538 seq_puts(m, "common_stacktrace:\n"); 5539 hist_trigger_stacktrace_print(m, 5540 key + key_field->offset, 5541 HIST_STACKTRACE_DEPTH); 5542 multiline = true; 5543 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5544 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5545 *(u64 *)(key + key_field->offset)); 5546 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) { 5547 unsigned long buckets = key_field->buckets; 5548 uval = *(u64 *)(key + key_field->offset); 5549 seq_printf(m, "%s: ~ %llu-%llu", field_name, 5550 uval, uval + buckets -1); 5551 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5552 seq_printf(m, "%s: %-50s", field_name, 5553 (char *)(key + key_field->offset)); 5554 } else { 5555 uval = *(u64 *)(key + key_field->offset); 5556 seq_printf(m, "%s: %10llu", field_name, uval); 5557 } 5558 } 5559 5560 if (!multiline) 5561 seq_puts(m, " "); 5562 5563 seq_puts(m, "}"); 5564 } 5565 5566 /* Get the 100 times of the percentage of @val in @total */ 5567 static inline unsigned int __get_percentage(u64 val, u64 total) 5568 { 5569 if (!total) 5570 goto div0; 5571 5572 if (val < (U64_MAX / 10000)) 5573 return (unsigned int)div64_ul(val * 10000, total); 5574 5575 total = div64_u64(total, 10000); 5576 if (!total) 5577 goto div0; 5578 5579 return (unsigned int)div64_ul(val, total); 5580 div0: 5581 return val ? UINT_MAX : 0; 5582 } 5583 5584 #define BAR_CHAR '#' 5585 5586 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max) 5587 { 5588 unsigned int len = __get_percentage(val, max); 5589 int i; 5590 5591 if (len == UINT_MAX) { 5592 snprintf(buf, size, "[ERROR]"); 5593 return buf; 5594 } 5595 5596 len = len * size / 10000; 5597 for (i = 0; i < len && i < size; i++) 5598 buf[i] = BAR_CHAR; 5599 while (i < size) 5600 buf[i++] = ' '; 5601 buf[size] = '\0'; 5602 5603 return buf; 5604 } 5605 5606 struct hist_val_stat { 5607 u64 max; 5608 u64 total; 5609 }; 5610 5611 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx, 5612 const char *field_name, unsigned long flags, 5613 struct hist_val_stat *stats, 5614 struct tracing_map_elt *elt) 5615 { 5616 u64 val = tracing_map_read_sum(elt, idx); 5617 unsigned int pc; 5618 char bar[21]; 5619 5620 if (flags & HIST_FIELD_FL_PERCENT) { 5621 pc = __get_percentage(val, stats[idx].total); 5622 if (pc == UINT_MAX) 5623 seq_printf(m, " %s (%%):[ERROR]", field_name); 5624 else 5625 seq_printf(m, " %s (%%): %3u.%02u", field_name, 5626 pc / 100, pc % 100); 5627 } else if (flags & HIST_FIELD_FL_GRAPH) { 5628 seq_printf(m, " %s: %20s", field_name, 5629 __fill_bar_str(bar, 20, val, stats[idx].max)); 5630 } else if (flags & HIST_FIELD_FL_HEX) { 5631 seq_printf(m, " %s: %10llx", field_name, val); 5632 } else { 5633 seq_printf(m, " %s: %10llu", field_name, val); 5634 } 5635 } 5636 5637 static void hist_trigger_entry_print(struct seq_file *m, 5638 struct hist_trigger_data *hist_data, 5639 struct hist_val_stat *stats, 5640 void *key, 5641 struct tracing_map_elt *elt) 5642 { 5643 const char *field_name; 5644 unsigned int i = HITCOUNT_IDX; 5645 unsigned long flags; 5646 5647 hist_trigger_print_key(m, hist_data, key, elt); 5648 5649 /* At first, show the raw hitcount if !nohitcount */ 5650 if (!hist_data->attrs->no_hitcount) 5651 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt); 5652 5653 for (i = 1; i < hist_data->n_vals; i++) { 5654 field_name = hist_field_name(hist_data->fields[i], 0); 5655 flags = hist_data->fields[i]->flags; 5656 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR) 5657 continue; 5658 5659 seq_puts(m, " "); 5660 hist_trigger_print_val(m, i, field_name, flags, stats, elt); 5661 } 5662 5663 print_actions(m, hist_data, elt); 5664 5665 seq_puts(m, "\n"); 5666 } 5667 5668 static int print_entries(struct seq_file *m, 5669 struct hist_trigger_data *hist_data) 5670 { 5671 struct tracing_map_sort_entry **sort_entries = NULL; 5672 struct tracing_map *map = hist_data->map; 5673 int i, j, n_entries; 5674 struct hist_val_stat *stats = NULL; 5675 u64 val; 5676 5677 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5678 hist_data->n_sort_keys, 5679 &sort_entries); 5680 if (n_entries < 0) 5681 return n_entries; 5682 5683 /* Calculate the max and the total for each field if needed. */ 5684 for (j = 0; j < hist_data->n_vals; j++) { 5685 if (!(hist_data->fields[j]->flags & 5686 (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH))) 5687 continue; 5688 if (!stats) { 5689 stats = kzalloc_objs(*stats, hist_data->n_vals); 5690 if (!stats) { 5691 n_entries = -ENOMEM; 5692 goto out; 5693 } 5694 } 5695 for (i = 0; i < n_entries; i++) { 5696 val = tracing_map_read_sum(sort_entries[i]->elt, j); 5697 stats[j].total += val; 5698 if (stats[j].max < val) 5699 stats[j].max = val; 5700 } 5701 } 5702 5703 for (i = 0; i < n_entries; i++) 5704 hist_trigger_entry_print(m, hist_data, stats, 5705 sort_entries[i]->key, 5706 sort_entries[i]->elt); 5707 5708 kfree(stats); 5709 out: 5710 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5711 5712 return n_entries; 5713 } 5714 5715 static void hist_trigger_show(struct seq_file *m, 5716 struct event_trigger_data *data, int n) 5717 { 5718 struct hist_trigger_data *hist_data; 5719 int n_entries; 5720 5721 if (n > 0) 5722 seq_puts(m, "\n\n"); 5723 5724 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5725 data->cmd_ops->print(m, data); 5726 seq_puts(m, "#\n\n"); 5727 5728 hist_data = data->private_data; 5729 n_entries = print_entries(m, hist_data); 5730 if (n_entries < 0) 5731 n_entries = 0; 5732 5733 track_data_snapshot_print(m, hist_data); 5734 5735 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5736 (u64)atomic64_read(&hist_data->map->hits), 5737 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5738 } 5739 5740 struct hist_file_data { 5741 struct file *file; 5742 u64 last_read; 5743 u64 last_act; 5744 }; 5745 5746 static u64 get_hist_hit_count(struct trace_event_file *event_file) 5747 { 5748 struct hist_trigger_data *hist_data; 5749 struct event_trigger_data *data; 5750 u64 ret = 0; 5751 5752 list_for_each_entry(data, &event_file->triggers, list) { 5753 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5754 hist_data = data->private_data; 5755 ret += atomic64_read(&hist_data->map->hits); 5756 } 5757 } 5758 return ret; 5759 } 5760 5761 static int hist_show(struct seq_file *m, void *v) 5762 { 5763 struct hist_file_data *hist_file = m->private; 5764 struct event_trigger_data *data; 5765 struct trace_event_file *event_file; 5766 int n = 0; 5767 5768 guard(mutex)(&event_mutex); 5769 5770 event_file = event_file_file(hist_file->file); 5771 if (unlikely(!event_file)) 5772 return -ENODEV; 5773 5774 list_for_each_entry(data, &event_file->triggers, list) { 5775 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5776 hist_trigger_show(m, data, n++); 5777 } 5778 hist_file->last_read = get_hist_hit_count(event_file); 5779 /* 5780 * Update last_act too so that poll()/POLLPRI can wait for the next 5781 * event after any syscall on hist file. 5782 */ 5783 hist_file->last_act = hist_file->last_read; 5784 5785 return 0; 5786 } 5787 5788 static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wait) 5789 { 5790 struct trace_event_file *event_file; 5791 struct seq_file *m = file->private_data; 5792 struct hist_file_data *hist_file = m->private; 5793 __poll_t ret = 0; 5794 u64 cnt; 5795 5796 guard(mutex)(&event_mutex); 5797 5798 event_file = event_file_file(file); 5799 if (!event_file) 5800 return EPOLLERR; 5801 5802 hist_poll_wait(file, wait); 5803 5804 cnt = get_hist_hit_count(event_file); 5805 if (hist_file->last_read != cnt) 5806 ret |= EPOLLIN | EPOLLRDNORM; 5807 if (hist_file->last_act != cnt) { 5808 hist_file->last_act = cnt; 5809 ret |= EPOLLPRI; 5810 } 5811 5812 return ret; 5813 } 5814 5815 static int event_hist_release(struct inode *inode, struct file *file) 5816 { 5817 struct seq_file *m = file->private_data; 5818 struct hist_file_data *hist_file = m->private; 5819 5820 kfree(hist_file); 5821 return tracing_single_release_file_tr(inode, file); 5822 } 5823 5824 static int event_hist_open(struct inode *inode, struct file *file) 5825 { 5826 struct trace_event_file *event_file; 5827 struct hist_file_data *hist_file; 5828 int ret; 5829 5830 ret = tracing_open_file_tr(inode, file); 5831 if (ret) 5832 return ret; 5833 5834 guard(mutex)(&event_mutex); 5835 5836 event_file = event_file_file(file); 5837 if (!event_file) { 5838 ret = -ENODEV; 5839 goto err; 5840 } 5841 5842 hist_file = kzalloc_obj(*hist_file); 5843 if (!hist_file) { 5844 ret = -ENOMEM; 5845 goto err; 5846 } 5847 5848 hist_file->file = file; 5849 hist_file->last_act = get_hist_hit_count(event_file); 5850 5851 ret = single_open(file, hist_show, hist_file); 5852 if (ret) { 5853 kfree(hist_file); 5854 goto err; 5855 } 5856 5857 return 0; 5858 err: 5859 tracing_release_file_tr(inode, file); 5860 return ret; 5861 } 5862 5863 const struct file_operations event_hist_fops = { 5864 .open = event_hist_open, 5865 .read = seq_read, 5866 .llseek = seq_lseek, 5867 .release = event_hist_release, 5868 .poll = event_hist_poll, 5869 }; 5870 5871 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 5872 5873 #undef C 5874 #define C(a, b) b 5875 5876 static const char * const field_funcs[] = { FIELD_FUNCS }; 5877 5878 static void hist_field_debug_show_flags(struct seq_file *m, 5879 unsigned long flags) 5880 { 5881 seq_puts(m, " flags:\n"); 5882 5883 if (flags & HIST_FIELD_FL_KEY) 5884 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 5885 else if (flags & HIST_FIELD_FL_HITCOUNT) 5886 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 5887 else if (flags & HIST_FIELD_FL_VAR) 5888 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 5889 else if (flags & HIST_FIELD_FL_VAR_REF) 5890 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 5891 else 5892 seq_puts(m, " VAL: normal u64 value\n"); 5893 5894 if (flags & HIST_FIELD_FL_ALIAS) 5895 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 5896 else if (flags & HIST_FIELD_FL_CONST) 5897 seq_puts(m, " HIST_FIELD_FL_CONST\n"); 5898 } 5899 5900 static int hist_field_debug_show(struct seq_file *m, 5901 struct hist_field *field, unsigned long flags) 5902 { 5903 if ((field->flags & flags) != flags) { 5904 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 5905 return -EINVAL; 5906 } 5907 5908 hist_field_debug_show_flags(m, field->flags); 5909 if (field->field) 5910 seq_printf(m, " ftrace_event_field name: %s\n", 5911 field->field->name); 5912 5913 if (field->flags & HIST_FIELD_FL_VAR) { 5914 seq_printf(m, " var.name: %s\n", field->var.name); 5915 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5916 field->var.idx); 5917 } 5918 5919 if (field->flags & HIST_FIELD_FL_CONST) 5920 seq_printf(m, " constant: %llu\n", field->constant); 5921 5922 if (field->flags & HIST_FIELD_FL_ALIAS) 5923 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5924 field->var_ref_idx); 5925 5926 if (field->flags & HIST_FIELD_FL_VAR_REF) { 5927 seq_printf(m, " name: %s\n", field->name); 5928 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5929 field->var.idx); 5930 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 5931 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5932 field->var_ref_idx); 5933 if (field->system) 5934 seq_printf(m, " system: %s\n", field->system); 5935 if (field->event_name) 5936 seq_printf(m, " event_name: %s\n", field->event_name); 5937 } 5938 5939 seq_printf(m, " type: %s\n", field->type); 5940 seq_printf(m, " size: %u\n", field->size); 5941 seq_printf(m, " is_signed: %u\n", field->is_signed); 5942 seq_printf(m, " function: hist_field_%s()\n", field_funcs[field->fn_num]); 5943 5944 return 0; 5945 } 5946 5947 static int field_var_debug_show(struct seq_file *m, 5948 struct field_var *field_var, unsigned int i, 5949 bool save_vars) 5950 { 5951 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 5952 struct hist_field *field; 5953 int ret = 0; 5954 5955 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 5956 5957 field = field_var->var; 5958 5959 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 5960 5961 hist_field_debug_show_flags(m, field->flags); 5962 seq_printf(m, " var.name: %s\n", field->var.name); 5963 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5964 field->var.idx); 5965 5966 field = field_var->val; 5967 5968 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 5969 if (field->field) 5970 seq_printf(m, " ftrace_event_field name: %s\n", 5971 field->field->name); 5972 else { 5973 ret = -EINVAL; 5974 goto out; 5975 } 5976 5977 seq_printf(m, " type: %s\n", field->type); 5978 seq_printf(m, " size: %u\n", field->size); 5979 seq_printf(m, " is_signed: %u\n", field->is_signed); 5980 out: 5981 return ret; 5982 } 5983 5984 static int hist_action_debug_show(struct seq_file *m, 5985 struct action_data *data, int i) 5986 { 5987 int ret = 0; 5988 5989 if (data->handler == HANDLER_ONMAX || 5990 data->handler == HANDLER_ONCHANGE) { 5991 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 5992 ret = hist_field_debug_show(m, data->track_data.var_ref, 5993 HIST_FIELD_FL_VAR_REF); 5994 if (ret) 5995 goto out; 5996 5997 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 5998 ret = hist_field_debug_show(m, data->track_data.track_var, 5999 HIST_FIELD_FL_VAR); 6000 if (ret) 6001 goto out; 6002 } 6003 6004 if (data->handler == HANDLER_ONMATCH) { 6005 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 6006 i, data->match_data.event_system); 6007 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 6008 i, data->match_data.event); 6009 } 6010 out: 6011 return ret; 6012 } 6013 6014 static int hist_actions_debug_show(struct seq_file *m, 6015 struct hist_trigger_data *hist_data) 6016 { 6017 int i, ret = 0; 6018 6019 if (hist_data->n_actions) 6020 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 6021 6022 for (i = 0; i < hist_data->n_actions; i++) { 6023 struct action_data *action = hist_data->actions[i]; 6024 6025 ret = hist_action_debug_show(m, action, i); 6026 if (ret) 6027 goto out; 6028 } 6029 6030 if (hist_data->n_save_vars) 6031 seq_puts(m, "\n save action variables (save() params):\n"); 6032 6033 for (i = 0; i < hist_data->n_save_vars; i++) { 6034 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 6035 if (ret) 6036 goto out; 6037 } 6038 out: 6039 return ret; 6040 } 6041 6042 static void hist_trigger_debug_show(struct seq_file *m, 6043 struct event_trigger_data *data, int n) 6044 { 6045 struct hist_trigger_data *hist_data; 6046 int i, ret; 6047 6048 if (n > 0) 6049 seq_puts(m, "\n\n"); 6050 6051 seq_puts(m, "# event histogram\n#\n# trigger info: "); 6052 data->cmd_ops->print(m, data); 6053 seq_puts(m, "#\n\n"); 6054 6055 hist_data = data->private_data; 6056 6057 seq_printf(m, "hist_data: %p\n\n", hist_data); 6058 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 6059 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 6060 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 6061 6062 seq_puts(m, "\n val fields:\n\n"); 6063 6064 seq_puts(m, " hist_data->fields[0]:\n"); 6065 ret = hist_field_debug_show(m, hist_data->fields[0], 6066 HIST_FIELD_FL_HITCOUNT); 6067 if (ret) 6068 return; 6069 6070 for (i = 1; i < hist_data->n_vals; i++) { 6071 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6072 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 6073 if (ret) 6074 return; 6075 } 6076 6077 seq_puts(m, "\n key fields:\n"); 6078 6079 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 6080 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6081 ret = hist_field_debug_show(m, hist_data->fields[i], 6082 HIST_FIELD_FL_KEY); 6083 if (ret) 6084 return; 6085 } 6086 6087 if (hist_data->n_var_refs) 6088 seq_puts(m, "\n variable reference fields:\n"); 6089 6090 for (i = 0; i < hist_data->n_var_refs; i++) { 6091 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 6092 ret = hist_field_debug_show(m, hist_data->var_refs[i], 6093 HIST_FIELD_FL_VAR_REF); 6094 if (ret) 6095 return; 6096 } 6097 6098 if (hist_data->n_field_vars) 6099 seq_puts(m, "\n field variables:\n"); 6100 6101 for (i = 0; i < hist_data->n_field_vars; i++) { 6102 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 6103 if (ret) 6104 return; 6105 } 6106 6107 ret = hist_actions_debug_show(m, hist_data); 6108 if (ret) 6109 return; 6110 } 6111 6112 static int hist_debug_show(struct seq_file *m, void *v) 6113 { 6114 struct event_trigger_data *data; 6115 struct trace_event_file *event_file; 6116 int n = 0; 6117 6118 guard(mutex)(&event_mutex); 6119 6120 event_file = event_file_file(m->private); 6121 if (unlikely(!event_file)) 6122 return -ENODEV; 6123 6124 list_for_each_entry(data, &event_file->triggers, list) { 6125 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 6126 hist_trigger_debug_show(m, data, n++); 6127 } 6128 return 0; 6129 } 6130 6131 static int event_hist_debug_open(struct inode *inode, struct file *file) 6132 { 6133 int ret; 6134 6135 ret = tracing_open_file_tr(inode, file); 6136 if (ret) 6137 return ret; 6138 6139 ret = single_open(file, hist_debug_show, file); 6140 if (ret) 6141 tracing_release_file_tr(inode, file); 6142 return ret; 6143 } 6144 6145 const struct file_operations event_hist_debug_fops = { 6146 .open = event_hist_debug_open, 6147 .read = seq_read, 6148 .llseek = seq_lseek, 6149 .release = tracing_single_release_file_tr, 6150 }; 6151 #endif 6152 6153 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 6154 { 6155 const char *field_name = hist_field_name(hist_field, 0); 6156 6157 if (hist_field->var.name) 6158 seq_printf(m, "%s=", hist_field->var.name); 6159 6160 if (hist_field->flags & HIST_FIELD_FL_CPU) 6161 seq_puts(m, "common_cpu"); 6162 if (hist_field->flags & HIST_FIELD_FL_COMM) 6163 seq_puts(m, "common_comm"); 6164 else if (hist_field->flags & HIST_FIELD_FL_CONST) 6165 seq_printf(m, "%llu", hist_field->constant); 6166 else if (field_name) { 6167 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 6168 hist_field->flags & HIST_FIELD_FL_ALIAS) 6169 if (!hist_field->system) 6170 seq_putc(m, '$'); 6171 seq_printf(m, "%s", field_name); 6172 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 6173 seq_puts(m, "common_timestamp"); 6174 6175 if (hist_field->flags) { 6176 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 6177 !(hist_field->flags & HIST_FIELD_FL_EXPR) && 6178 !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) { 6179 const char *flags = get_hist_field_flags(hist_field); 6180 6181 if (flags) 6182 seq_printf(m, ".%s", flags); 6183 } 6184 } 6185 if (hist_field->buckets) 6186 seq_printf(m, "=%ld", hist_field->buckets); 6187 } 6188 6189 static int event_hist_trigger_print(struct seq_file *m, 6190 struct event_trigger_data *data) 6191 { 6192 struct hist_trigger_data *hist_data = data->private_data; 6193 struct hist_field *field; 6194 bool have_var = false; 6195 bool show_val = false; 6196 unsigned int i; 6197 6198 seq_puts(m, HIST_PREFIX); 6199 6200 if (data->name) 6201 seq_printf(m, "%s:", data->name); 6202 6203 seq_puts(m, "keys="); 6204 6205 for_each_hist_key_field(i, hist_data) { 6206 field = hist_data->fields[i]; 6207 6208 if (i > hist_data->n_vals) 6209 seq_puts(m, ","); 6210 6211 if (field->flags & HIST_FIELD_FL_STACKTRACE) { 6212 if (field->field) 6213 seq_printf(m, "%s.stacktrace", field->field->name); 6214 else 6215 seq_puts(m, "common_stacktrace"); 6216 } else 6217 hist_field_print(m, field); 6218 } 6219 6220 seq_puts(m, ":vals="); 6221 6222 for_each_hist_val_field(i, hist_data) { 6223 field = hist_data->fields[i]; 6224 if (field->flags & HIST_FIELD_FL_VAR) { 6225 have_var = true; 6226 continue; 6227 } 6228 6229 if (i == HITCOUNT_IDX) { 6230 if (hist_data->attrs->no_hitcount) 6231 continue; 6232 seq_puts(m, "hitcount"); 6233 } else { 6234 if (show_val) 6235 seq_puts(m, ","); 6236 hist_field_print(m, field); 6237 } 6238 show_val = true; 6239 } 6240 6241 if (have_var) { 6242 unsigned int n = 0; 6243 6244 seq_puts(m, ":"); 6245 6246 for_each_hist_val_field(i, hist_data) { 6247 field = hist_data->fields[i]; 6248 6249 if (field->flags & HIST_FIELD_FL_VAR) { 6250 if (n++) 6251 seq_puts(m, ","); 6252 hist_field_print(m, field); 6253 } 6254 } 6255 } 6256 6257 seq_puts(m, ":sort="); 6258 6259 for (i = 0; i < hist_data->n_sort_keys; i++) { 6260 struct tracing_map_sort_key *sort_key; 6261 unsigned int idx, first_key_idx; 6262 6263 /* skip VAR vals */ 6264 first_key_idx = hist_data->n_vals - hist_data->n_vars; 6265 6266 sort_key = &hist_data->sort_keys[i]; 6267 idx = sort_key->field_idx; 6268 6269 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 6270 return -EINVAL; 6271 6272 if (i > 0) 6273 seq_puts(m, ","); 6274 6275 if (idx == HITCOUNT_IDX) 6276 seq_puts(m, "hitcount"); 6277 else { 6278 if (idx >= first_key_idx) 6279 idx += hist_data->n_vars; 6280 hist_field_print(m, hist_data->fields[idx]); 6281 } 6282 6283 if (sort_key->descending) 6284 seq_puts(m, ".descending"); 6285 } 6286 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 6287 if (hist_data->enable_timestamps) 6288 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 6289 if (hist_data->attrs->no_hitcount) 6290 seq_puts(m, ":nohitcount"); 6291 6292 print_actions_spec(m, hist_data); 6293 6294 if (data->filter_str) 6295 seq_printf(m, " if %s", data->filter_str); 6296 6297 if (data->paused) 6298 seq_puts(m, " [paused]"); 6299 else 6300 seq_puts(m, " [active]"); 6301 6302 seq_putc(m, '\n'); 6303 6304 return 0; 6305 } 6306 6307 static int event_hist_trigger_init(struct event_trigger_data *data) 6308 { 6309 struct hist_trigger_data *hist_data = data->private_data; 6310 6311 if (alloc_hist_pad() < 0) 6312 return -ENOMEM; 6313 6314 if (!data->ref && hist_data->attrs->name) 6315 save_named_trigger(hist_data->attrs->name, data); 6316 6317 data->ref++; 6318 6319 return 0; 6320 } 6321 6322 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 6323 { 6324 struct trace_event_file *file; 6325 unsigned int i; 6326 char *cmd; 6327 int ret; 6328 6329 for (i = 0; i < hist_data->n_field_var_hists; i++) { 6330 file = hist_data->field_var_hists[i]->hist_data->event_file; 6331 cmd = hist_data->field_var_hists[i]->cmd; 6332 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 6333 "!hist", "hist", cmd); 6334 WARN_ON_ONCE(ret < 0); 6335 } 6336 } 6337 6338 static void event_hist_trigger_free(struct event_trigger_data *data) 6339 { 6340 struct hist_trigger_data *hist_data = data->private_data; 6341 6342 if (WARN_ON_ONCE(data->ref <= 0)) 6343 return; 6344 6345 data->ref--; 6346 if (!data->ref) { 6347 if (data->name) 6348 del_named_trigger(data); 6349 6350 trigger_data_free(data); 6351 6352 remove_hist_vars(hist_data); 6353 6354 unregister_field_var_hists(hist_data); 6355 6356 destroy_hist_data(hist_data); 6357 } 6358 free_hist_pad(); 6359 } 6360 6361 static int event_hist_trigger_named_init(struct event_trigger_data *data) 6362 { 6363 int ret; 6364 6365 data->ref++; 6366 6367 save_named_trigger(data->named_data->name, data); 6368 6369 ret = event_hist_trigger_init(data->named_data); 6370 if (ret < 0) { 6371 kfree(data->cmd_ops); 6372 data->cmd_ops = &trigger_hist_cmd; 6373 } 6374 6375 return ret; 6376 } 6377 6378 static void event_hist_trigger_named_free(struct event_trigger_data *data) 6379 { 6380 if (WARN_ON_ONCE(data->ref <= 0)) 6381 return; 6382 6383 event_hist_trigger_free(data->named_data); 6384 6385 data->ref--; 6386 if (!data->ref) { 6387 struct event_command *cmd_ops = data->cmd_ops; 6388 6389 del_named_trigger(data); 6390 trigger_data_free(data); 6391 kfree(cmd_ops); 6392 } 6393 } 6394 6395 static void hist_clear(struct event_trigger_data *data) 6396 { 6397 struct hist_trigger_data *hist_data = data->private_data; 6398 6399 if (data->name) 6400 pause_named_trigger(data); 6401 6402 tracepoint_synchronize_unregister(); 6403 6404 tracing_map_clear(hist_data->map); 6405 6406 if (data->name) 6407 unpause_named_trigger(data); 6408 } 6409 6410 static bool compatible_field(struct ftrace_event_field *field, 6411 struct ftrace_event_field *test_field) 6412 { 6413 if (field == test_field) 6414 return true; 6415 if (field == NULL || test_field == NULL) 6416 return false; 6417 if (strcmp(field->name, test_field->name) != 0) 6418 return false; 6419 if (strcmp(field->type, test_field->type) != 0) 6420 return false; 6421 if (field->size != test_field->size) 6422 return false; 6423 if (field->is_signed != test_field->is_signed) 6424 return false; 6425 6426 return true; 6427 } 6428 6429 static bool hist_trigger_match(struct event_trigger_data *data, 6430 struct event_trigger_data *data_test, 6431 struct event_trigger_data *named_data, 6432 bool ignore_filter) 6433 { 6434 struct tracing_map_sort_key *sort_key, *sort_key_test; 6435 struct hist_trigger_data *hist_data, *hist_data_test; 6436 struct hist_field *key_field, *key_field_test; 6437 unsigned int i; 6438 6439 if (named_data && (named_data != data_test) && 6440 (named_data != data_test->named_data)) 6441 return false; 6442 6443 if (!named_data && is_named_trigger(data_test)) 6444 return false; 6445 6446 hist_data = data->private_data; 6447 hist_data_test = data_test->private_data; 6448 6449 if (hist_data->n_vals != hist_data_test->n_vals || 6450 hist_data->n_fields != hist_data_test->n_fields || 6451 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 6452 return false; 6453 6454 if (!ignore_filter) { 6455 if ((data->filter_str && !data_test->filter_str) || 6456 (!data->filter_str && data_test->filter_str)) 6457 return false; 6458 } 6459 6460 for_each_hist_field(i, hist_data) { 6461 key_field = hist_data->fields[i]; 6462 key_field_test = hist_data_test->fields[i]; 6463 6464 if (key_field->flags != key_field_test->flags) 6465 return false; 6466 if (!compatible_field(key_field->field, key_field_test->field)) 6467 return false; 6468 if (key_field->offset != key_field_test->offset) 6469 return false; 6470 if (key_field->size != key_field_test->size) 6471 return false; 6472 if (key_field->is_signed != key_field_test->is_signed) 6473 return false; 6474 if (!!key_field->var.name != !!key_field_test->var.name) 6475 return false; 6476 if (key_field->var.name && 6477 strcmp(key_field->var.name, key_field_test->var.name) != 0) 6478 return false; 6479 } 6480 6481 for (i = 0; i < hist_data->n_sort_keys; i++) { 6482 sort_key = &hist_data->sort_keys[i]; 6483 sort_key_test = &hist_data_test->sort_keys[i]; 6484 6485 if (sort_key->field_idx != sort_key_test->field_idx || 6486 sort_key->descending != sort_key_test->descending) 6487 return false; 6488 } 6489 6490 if (!ignore_filter && data->filter_str && 6491 (strcmp(data->filter_str, data_test->filter_str) != 0)) 6492 return false; 6493 6494 if (!actions_match(hist_data, hist_data_test)) 6495 return false; 6496 6497 return true; 6498 } 6499 6500 static bool existing_hist_update_only(char *glob, 6501 struct event_trigger_data *data, 6502 struct trace_event_file *file) 6503 { 6504 struct hist_trigger_data *hist_data = data->private_data; 6505 struct event_trigger_data *test, *named_data = NULL; 6506 bool updated = false; 6507 6508 if (!hist_data->attrs->pause && !hist_data->attrs->cont && 6509 !hist_data->attrs->clear) 6510 goto out; 6511 6512 if (hist_data->attrs->name) { 6513 named_data = find_named_trigger(hist_data->attrs->name); 6514 if (named_data) { 6515 if (!hist_trigger_match(data, named_data, named_data, 6516 true)) 6517 goto out; 6518 } 6519 } 6520 6521 if (hist_data->attrs->name && !named_data) 6522 goto out; 6523 6524 list_for_each_entry(test, &file->triggers, list) { 6525 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6526 if (!hist_trigger_match(data, test, named_data, false)) 6527 continue; 6528 if (hist_data->attrs->pause) 6529 test->paused = true; 6530 else if (hist_data->attrs->cont) 6531 test->paused = false; 6532 else if (hist_data->attrs->clear) 6533 hist_clear(test); 6534 updated = true; 6535 goto out; 6536 } 6537 } 6538 out: 6539 return updated; 6540 } 6541 6542 /* 6543 * Set or disable using the per CPU trace_buffer_event when possible. 6544 */ 6545 static int tracing_set_filter_buffering(struct trace_array *tr, bool set) 6546 { 6547 guard(mutex)(&trace_types_lock); 6548 6549 if (set && tr->no_filter_buffering_ref++) 6550 return 0; 6551 6552 if (!set) { 6553 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) 6554 return -EINVAL; 6555 6556 --tr->no_filter_buffering_ref; 6557 } 6558 6559 return 0; 6560 } 6561 6562 static int hist_register_trigger(char *glob, 6563 struct event_trigger_data *data, 6564 struct trace_event_file *file) 6565 { 6566 struct hist_trigger_data *hist_data = data->private_data; 6567 struct event_trigger_data *test, *named_data = NULL; 6568 struct trace_array *tr = file->tr; 6569 int ret = 0; 6570 6571 if (hist_data->attrs->name) { 6572 named_data = find_named_trigger(hist_data->attrs->name); 6573 if (named_data) { 6574 if (!hist_trigger_match(data, named_data, named_data, 6575 true)) { 6576 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 6577 ret = -EINVAL; 6578 goto out; 6579 } 6580 } 6581 } 6582 6583 if (hist_data->attrs->name && !named_data) 6584 goto new; 6585 6586 lockdep_assert_held(&event_mutex); 6587 6588 list_for_each_entry(test, &file->triggers, list) { 6589 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6590 if (hist_trigger_match(data, test, named_data, false)) { 6591 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 6592 ret = -EEXIST; 6593 goto out; 6594 } 6595 } 6596 } 6597 new: 6598 if (hist_data->attrs->cont || hist_data->attrs->clear) { 6599 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 6600 ret = -ENOENT; 6601 goto out; 6602 } 6603 6604 if (hist_data->attrs->pause) 6605 data->paused = true; 6606 6607 if (named_data) { 6608 struct event_command *cmd_ops; 6609 6610 data->private_data = named_data->private_data; 6611 set_named_trigger_data(data, named_data); 6612 /* Copy the command ops and update some of the functions */ 6613 cmd_ops = kmalloc_obj(*cmd_ops); 6614 if (!cmd_ops) { 6615 ret = -ENOMEM; 6616 goto out; 6617 } 6618 *cmd_ops = *data->cmd_ops; 6619 cmd_ops->init = event_hist_trigger_named_init; 6620 cmd_ops->free = event_hist_trigger_named_free; 6621 data->cmd_ops = cmd_ops; 6622 } 6623 6624 if (data->cmd_ops->init) { 6625 ret = data->cmd_ops->init(data); 6626 if (ret < 0) 6627 goto out; 6628 } 6629 6630 if (hist_data->enable_timestamps) { 6631 char *clock = hist_data->attrs->clock; 6632 6633 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 6634 if (ret) { 6635 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 6636 goto out; 6637 } 6638 6639 tracing_set_filter_buffering(file->tr, true); 6640 } 6641 6642 if (named_data) 6643 destroy_hist_data(hist_data); 6644 out: 6645 return ret; 6646 } 6647 6648 static int hist_trigger_enable(struct event_trigger_data *data, 6649 struct trace_event_file *file) 6650 { 6651 int ret = 0; 6652 6653 list_add_tail_rcu(&data->list, &file->triggers); 6654 6655 update_cond_flag(file); 6656 6657 if (trace_event_trigger_enable_disable(file, 1) < 0) { 6658 list_del_rcu(&data->list); 6659 update_cond_flag(file); 6660 ret--; 6661 } 6662 6663 return ret; 6664 } 6665 6666 static bool have_hist_trigger_match(struct event_trigger_data *data, 6667 struct trace_event_file *file) 6668 { 6669 struct hist_trigger_data *hist_data = data->private_data; 6670 struct event_trigger_data *test, *named_data = NULL; 6671 bool match = false; 6672 6673 lockdep_assert_held(&event_mutex); 6674 6675 if (hist_data->attrs->name) 6676 named_data = find_named_trigger(hist_data->attrs->name); 6677 6678 list_for_each_entry(test, &file->triggers, list) { 6679 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6680 if (hist_trigger_match(data, test, named_data, false)) { 6681 match = true; 6682 break; 6683 } 6684 } 6685 } 6686 6687 return match; 6688 } 6689 6690 static bool hist_trigger_check_refs(struct event_trigger_data *data, 6691 struct trace_event_file *file) 6692 { 6693 struct hist_trigger_data *hist_data = data->private_data; 6694 struct event_trigger_data *test, *named_data = NULL; 6695 6696 lockdep_assert_held(&event_mutex); 6697 6698 if (hist_data->attrs->name) 6699 named_data = find_named_trigger(hist_data->attrs->name); 6700 6701 list_for_each_entry(test, &file->triggers, list) { 6702 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6703 if (!hist_trigger_match(data, test, named_data, false)) 6704 continue; 6705 hist_data = test->private_data; 6706 if (check_var_refs(hist_data)) 6707 return true; 6708 break; 6709 } 6710 } 6711 6712 return false; 6713 } 6714 6715 static void hist_unregister_trigger(char *glob, 6716 struct event_trigger_data *data, 6717 struct trace_event_file *file) 6718 { 6719 struct event_trigger_data *test = NULL, *iter, *named_data = NULL; 6720 struct hist_trigger_data *hist_data = data->private_data; 6721 6722 lockdep_assert_held(&event_mutex); 6723 6724 if (hist_data->attrs->name) 6725 named_data = find_named_trigger(hist_data->attrs->name); 6726 6727 list_for_each_entry(iter, &file->triggers, list) { 6728 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6729 if (!hist_trigger_match(data, iter, named_data, false)) 6730 continue; 6731 test = iter; 6732 list_del_rcu(&test->list); 6733 trace_event_trigger_enable_disable(file, 0); 6734 update_cond_flag(file); 6735 break; 6736 } 6737 } 6738 6739 if (test && test->cmd_ops->free) 6740 test->cmd_ops->free(test); 6741 6742 if (hist_data->enable_timestamps) { 6743 if (!hist_data->remove || test) 6744 tracing_set_filter_buffering(file->tr, false); 6745 } 6746 } 6747 6748 static bool hist_file_check_refs(struct trace_event_file *file) 6749 { 6750 struct hist_trigger_data *hist_data; 6751 struct event_trigger_data *test; 6752 6753 lockdep_assert_held(&event_mutex); 6754 6755 list_for_each_entry(test, &file->triggers, list) { 6756 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6757 hist_data = test->private_data; 6758 if (check_var_refs(hist_data)) 6759 return true; 6760 } 6761 } 6762 6763 return false; 6764 } 6765 6766 static void hist_unreg_all(struct trace_event_file *file) 6767 { 6768 struct event_trigger_data *test, *n; 6769 struct hist_trigger_data *hist_data; 6770 struct synth_event *se; 6771 const char *se_name; 6772 6773 lockdep_assert_held(&event_mutex); 6774 6775 if (hist_file_check_refs(file)) 6776 return; 6777 6778 list_for_each_entry_safe(test, n, &file->triggers, list) { 6779 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6780 hist_data = test->private_data; 6781 list_del_rcu(&test->list); 6782 trace_event_trigger_enable_disable(file, 0); 6783 6784 se_name = trace_event_name(file->event_call); 6785 se = find_synth_event(se_name); 6786 if (se) 6787 se->ref--; 6788 6789 update_cond_flag(file); 6790 if (hist_data->enable_timestamps) 6791 tracing_set_filter_buffering(file->tr, false); 6792 if (test->cmd_ops->free) 6793 test->cmd_ops->free(test); 6794 } 6795 } 6796 } 6797 6798 static int event_hist_trigger_parse(struct event_command *cmd_ops, 6799 struct trace_event_file *file, 6800 char *glob, char *cmd, 6801 char *param_and_filter) 6802 { 6803 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6804 struct event_trigger_data *trigger_data; 6805 struct hist_trigger_attrs *attrs; 6806 struct hist_trigger_data *hist_data; 6807 char *param, *filter, *p, *start; 6808 struct synth_event *se; 6809 const char *se_name; 6810 bool remove; 6811 int ret = 0; 6812 6813 lockdep_assert_held(&event_mutex); 6814 6815 if (WARN_ON(!glob)) 6816 return -EINVAL; 6817 6818 if (glob[0]) { 6819 hist_err_clear(); 6820 last_cmd_set(file, param_and_filter); 6821 } 6822 6823 remove = event_trigger_check_remove(glob); 6824 6825 if (event_trigger_empty_param(param_and_filter)) 6826 return -EINVAL; 6827 6828 /* 6829 * separate the trigger from the filter (k:v [if filter]) 6830 * allowing for whitespace in the trigger 6831 */ 6832 p = param = param_and_filter; 6833 do { 6834 p = strstr(p, "if"); 6835 if (!p) 6836 break; 6837 if (p == param_and_filter) 6838 return -EINVAL; 6839 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6840 p++; 6841 continue; 6842 } 6843 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1) 6844 return -EINVAL; 6845 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6846 p++; 6847 continue; 6848 } 6849 break; 6850 } while (1); 6851 6852 if (!p) 6853 filter = NULL; 6854 else { 6855 *(p - 1) = '\0'; 6856 filter = strstrip(p); 6857 param = strstrip(param); 6858 } 6859 6860 /* 6861 * To simplify arithmetic expression parsing, replace occurrences of 6862 * '.sym-offset' modifier with '.symXoffset' 6863 */ 6864 start = strstr(param, ".sym-offset"); 6865 while (start) { 6866 *(start + 4) = 'X'; 6867 start = strstr(start + 11, ".sym-offset"); 6868 } 6869 6870 attrs = parse_hist_trigger_attrs(file->tr, param); 6871 if (IS_ERR(attrs)) 6872 return PTR_ERR(attrs); 6873 6874 if (attrs->map_bits) 6875 hist_trigger_bits = attrs->map_bits; 6876 6877 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6878 if (IS_ERR(hist_data)) { 6879 destroy_hist_trigger_attrs(attrs); 6880 return PTR_ERR(hist_data); 6881 } 6882 6883 trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data); 6884 if (!trigger_data) { 6885 ret = -ENOMEM; 6886 goto out_free; 6887 } 6888 6889 ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data); 6890 if (ret < 0) 6891 goto out_free; 6892 6893 if (remove) { 6894 if (!have_hist_trigger_match(trigger_data, file)) 6895 goto out_free; 6896 6897 if (hist_trigger_check_refs(trigger_data, file)) { 6898 ret = -EBUSY; 6899 goto out_free; 6900 } 6901 6902 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6903 se_name = trace_event_name(file->event_call); 6904 se = find_synth_event(se_name); 6905 if (se) 6906 se->ref--; 6907 ret = 0; 6908 goto out_free; 6909 } 6910 6911 if (existing_hist_update_only(glob, trigger_data, file)) 6912 goto out_free; 6913 6914 if (!get_named_trigger_data(trigger_data)) { 6915 6916 ret = create_actions(hist_data); 6917 if (ret) 6918 goto out_free; 6919 6920 if (has_hist_vars(hist_data) || hist_data->n_var_refs) { 6921 ret = save_hist_vars(hist_data); 6922 if (ret) 6923 goto out_free; 6924 } 6925 6926 ret = tracing_map_init(hist_data->map); 6927 if (ret) 6928 goto out_free; 6929 } 6930 6931 ret = event_trigger_register(cmd_ops, file, glob, trigger_data); 6932 if (ret < 0) 6933 goto out_free; 6934 6935 ret = hist_trigger_enable(trigger_data, file); 6936 if (ret) 6937 goto out_unreg; 6938 6939 se_name = trace_event_name(file->event_call); 6940 se = find_synth_event(se_name); 6941 if (se) 6942 se->ref++; 6943 out: 6944 if (ret == 0 && glob[0]) 6945 hist_err_clear(); 6946 6947 return ret; 6948 out_unreg: 6949 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6950 out_free: 6951 remove_hist_vars(hist_data); 6952 6953 trigger_data_free(trigger_data); 6954 6955 destroy_hist_data(hist_data); 6956 goto out; 6957 } 6958 6959 static struct event_command trigger_hist_cmd = { 6960 .name = "hist", 6961 .trigger_type = ETT_EVENT_HIST, 6962 .flags = EVENT_CMD_FL_NEEDS_REC, 6963 .parse = event_hist_trigger_parse, 6964 .reg = hist_register_trigger, 6965 .unreg = hist_unregister_trigger, 6966 .unreg_all = hist_unreg_all, 6967 .set_filter = set_trigger_filter, 6968 .trigger = event_hist_trigger, 6969 .print = event_hist_trigger_print, 6970 .init = event_hist_trigger_init, 6971 .free = event_hist_trigger_free, 6972 }; 6973 6974 __init int register_trigger_hist_cmd(void) 6975 { 6976 int ret; 6977 6978 ret = register_event_command(&trigger_hist_cmd); 6979 WARN_ON(ret < 0); 6980 6981 return ret; 6982 } 6983 6984 static void 6985 hist_enable_trigger(struct event_trigger_data *data, 6986 struct trace_buffer *buffer, void *rec, 6987 struct ring_buffer_event *event) 6988 { 6989 struct enable_trigger_data *enable_data = data->private_data; 6990 struct event_trigger_data *test; 6991 6992 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 6993 lockdep_is_held(&event_mutex)) { 6994 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6995 if (enable_data->enable) 6996 test->paused = false; 6997 else 6998 test->paused = true; 6999 } 7000 } 7001 } 7002 7003 static void hist_enable_unreg_all(struct trace_event_file *file) 7004 { 7005 struct event_trigger_data *test, *n; 7006 7007 list_for_each_entry_safe(test, n, &file->triggers, list) { 7008 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 7009 list_del_rcu(&test->list); 7010 update_cond_flag(file); 7011 trace_event_trigger_enable_disable(file, 0); 7012 if (test->cmd_ops->free) 7013 test->cmd_ops->free(test); 7014 } 7015 } 7016 } 7017 7018 static struct event_command trigger_hist_enable_cmd = { 7019 .name = ENABLE_HIST_STR, 7020 .trigger_type = ETT_HIST_ENABLE, 7021 .parse = event_enable_trigger_parse, 7022 .reg = event_enable_register_trigger, 7023 .unreg = event_enable_unregister_trigger, 7024 .unreg_all = hist_enable_unreg_all, 7025 .set_filter = set_trigger_filter, 7026 .trigger = hist_enable_trigger, 7027 .count_func = event_trigger_count, 7028 .print = event_enable_trigger_print, 7029 .init = event_trigger_init, 7030 .free = event_enable_trigger_free, 7031 }; 7032 7033 static struct event_command trigger_hist_disable_cmd = { 7034 .name = DISABLE_HIST_STR, 7035 .trigger_type = ETT_HIST_ENABLE, 7036 .parse = event_enable_trigger_parse, 7037 .reg = event_enable_register_trigger, 7038 .unreg = event_enable_unregister_trigger, 7039 .unreg_all = hist_enable_unreg_all, 7040 .set_filter = set_trigger_filter, 7041 .trigger = hist_enable_trigger, 7042 .count_func = event_trigger_count, 7043 .print = event_enable_trigger_print, 7044 .init = event_trigger_init, 7045 .free = event_enable_trigger_free, 7046 }; 7047 7048 static __init void unregister_trigger_hist_enable_disable_cmds(void) 7049 { 7050 unregister_event_command(&trigger_hist_enable_cmd); 7051 unregister_event_command(&trigger_hist_disable_cmd); 7052 } 7053 7054 __init int register_trigger_hist_enable_disable_cmds(void) 7055 { 7056 int ret; 7057 7058 ret = register_event_command(&trigger_hist_enable_cmd); 7059 if (WARN_ON(ret < 0)) 7060 return ret; 7061 ret = register_event_command(&trigger_hist_disable_cmd); 7062 if (WARN_ON(ret < 0)) 7063 unregister_trigger_hist_enable_disable_cmds(); 7064 7065 return ret; 7066 } 7067