1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/seq_buf.h> 12 #include <linux/mutex.h> 13 #include <linux/slab.h> 14 #include <linux/stacktrace.h> 15 #include <linux/rculist.h> 16 #include <linux/tracefs.h> 17 18 /* for gfp flag names */ 19 #include <linux/trace_events.h> 20 #include <trace/events/mmflags.h> 21 22 #include "tracing_map.h" 23 #include "trace_synth.h" 24 25 #define ERRORS \ 26 C(NONE, "No error"), \ 27 C(DUPLICATE_VAR, "Variable already defined"), \ 28 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 29 C(TOO_MANY_VARS, "Too many variables defined"), \ 30 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 31 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 32 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 33 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 34 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 35 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 36 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 37 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 38 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 39 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 40 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 41 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 42 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 43 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 44 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 45 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 46 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 47 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 48 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 49 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 50 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 51 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 52 C(TOO_MANY_PARAMS, "Too many action params"), \ 53 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 54 C(INVALID_PARAM, "Invalid action param"), \ 55 C(ACTION_NOT_FOUND, "No action found"), \ 56 C(NO_SAVE_PARAMS, "No params found for save()"), \ 57 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 58 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 59 C(NO_CLOSING_PAREN, "No closing paren found"), \ 60 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 61 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 62 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 63 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 64 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 65 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 66 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 67 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 68 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 69 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \ 70 C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \ 71 C(EXPECT_NUMBER, "Expecting numeric literal"), \ 72 C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \ 73 C(DIVISION_BY_ZERO, "Division by zero"), \ 74 C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"), 75 76 #undef C 77 #define C(a, b) HIST_ERR_##a 78 79 enum { ERRORS }; 80 81 #undef C 82 #define C(a, b) b 83 84 static const char *err_text[] = { ERRORS }; 85 86 struct hist_field; 87 88 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 89 struct tracing_map_elt *elt, 90 struct trace_buffer *buffer, 91 struct ring_buffer_event *rbe, 92 void *event); 93 94 #define HIST_FIELD_OPERANDS_MAX 2 95 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 96 #define HIST_ACTIONS_MAX 8 97 #define HIST_DIV_SHIFT 20 /* For optimizing division by constants */ 98 99 enum field_op_id { 100 FIELD_OP_NONE, 101 FIELD_OP_PLUS, 102 FIELD_OP_MINUS, 103 FIELD_OP_UNARY_MINUS, 104 FIELD_OP_DIV, 105 FIELD_OP_MULT, 106 }; 107 108 #define FIELD_FUNCS \ 109 C(NOP, "nop"), \ 110 C(VAR_REF, "var_ref"), \ 111 C(COUNTER, "counter"), \ 112 C(CONST, "const"), \ 113 C(LOG2, "log2"), \ 114 C(BUCKET, "bucket"), \ 115 C(TIMESTAMP, "timestamp"), \ 116 C(CPU, "cpu"), \ 117 C(COMM, "comm"), \ 118 C(STRING, "string"), \ 119 C(DYNSTRING, "dynstring"), \ 120 C(RELDYNSTRING, "reldynstring"), \ 121 C(PSTRING, "pstring"), \ 122 C(S64, "s64"), \ 123 C(U64, "u64"), \ 124 C(S32, "s32"), \ 125 C(U32, "u32"), \ 126 C(S16, "s16"), \ 127 C(U16, "u16"), \ 128 C(S8, "s8"), \ 129 C(U8, "u8"), \ 130 C(UMINUS, "uminus"), \ 131 C(MINUS, "minus"), \ 132 C(PLUS, "plus"), \ 133 C(DIV, "div"), \ 134 C(MULT, "mult"), \ 135 C(DIV_POWER2, "div_power2"), \ 136 C(DIV_NOT_POWER2, "div_not_power2"), \ 137 C(DIV_MULT_SHIFT, "div_mult_shift"), \ 138 C(EXECNAME, "execname"), \ 139 C(STACK, "stack"), 140 141 #undef C 142 #define C(a, b) HIST_FIELD_FN_##a 143 144 enum hist_field_fn { 145 FIELD_FUNCS 146 }; 147 148 /* 149 * A hist_var (histogram variable) contains variable information for 150 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 151 * flag set. A hist_var has a variable name e.g. ts0, and is 152 * associated with a given histogram trigger, as specified by 153 * hist_data. The hist_var idx is the unique index assigned to the 154 * variable by the hist trigger's tracing_map. The idx is what is 155 * used to set a variable's value and, by a variable reference, to 156 * retrieve it. 157 */ 158 struct hist_var { 159 char *name; 160 struct hist_trigger_data *hist_data; 161 unsigned int idx; 162 }; 163 164 struct hist_field { 165 struct ftrace_event_field *field; 166 unsigned long flags; 167 unsigned long buckets; 168 const char *type; 169 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 170 struct hist_trigger_data *hist_data; 171 enum hist_field_fn fn_num; 172 unsigned int size; 173 unsigned int offset; 174 unsigned int is_signed; 175 176 /* 177 * Variable fields contain variable-specific info in var. 178 */ 179 struct hist_var var; 180 enum field_op_id operator; 181 char *system; 182 char *event_name; 183 184 /* 185 * The name field is used for EXPR and VAR_REF fields. VAR 186 * fields contain the variable name in var.name. 187 */ 188 char *name; 189 190 /* 191 * When a histogram trigger is hit, if it has any references 192 * to variables, the values of those variables are collected 193 * into a var_ref_vals array by resolve_var_refs(). The 194 * current value of each variable is read from the tracing_map 195 * using the hist field's hist_var.idx and entered into the 196 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 197 */ 198 unsigned int var_ref_idx; 199 bool read_once; 200 201 unsigned int var_str_idx; 202 203 /* Numeric literals are represented as u64 */ 204 u64 constant; 205 /* Used to optimize division by constants */ 206 u64 div_multiplier; 207 }; 208 209 static u64 hist_fn_call(struct hist_field *hist_field, 210 struct tracing_map_elt *elt, 211 struct trace_buffer *buffer, 212 struct ring_buffer_event *rbe, 213 void *event); 214 215 static u64 hist_field_const(struct hist_field *field, 216 struct tracing_map_elt *elt, 217 struct trace_buffer *buffer, 218 struct ring_buffer_event *rbe, 219 void *event) 220 { 221 return field->constant; 222 } 223 224 static u64 hist_field_counter(struct hist_field *field, 225 struct tracing_map_elt *elt, 226 struct trace_buffer *buffer, 227 struct ring_buffer_event *rbe, 228 void *event) 229 { 230 return 1; 231 } 232 233 static u64 hist_field_string(struct hist_field *hist_field, 234 struct tracing_map_elt *elt, 235 struct trace_buffer *buffer, 236 struct ring_buffer_event *rbe, 237 void *event) 238 { 239 char *addr = (char *)(event + hist_field->field->offset); 240 241 return (u64)(unsigned long)addr; 242 } 243 244 static u64 hist_field_dynstring(struct hist_field *hist_field, 245 struct tracing_map_elt *elt, 246 struct trace_buffer *buffer, 247 struct ring_buffer_event *rbe, 248 void *event) 249 { 250 u32 str_item = *(u32 *)(event + hist_field->field->offset); 251 int str_loc = str_item & 0xffff; 252 char *addr = (char *)(event + str_loc); 253 254 return (u64)(unsigned long)addr; 255 } 256 257 static u64 hist_field_reldynstring(struct hist_field *hist_field, 258 struct tracing_map_elt *elt, 259 struct trace_buffer *buffer, 260 struct ring_buffer_event *rbe, 261 void *event) 262 { 263 u32 *item = event + hist_field->field->offset; 264 u32 str_item = *item; 265 int str_loc = str_item & 0xffff; 266 char *addr = (char *)&item[1] + str_loc; 267 268 return (u64)(unsigned long)addr; 269 } 270 271 static u64 hist_field_pstring(struct hist_field *hist_field, 272 struct tracing_map_elt *elt, 273 struct trace_buffer *buffer, 274 struct ring_buffer_event *rbe, 275 void *event) 276 { 277 char **addr = (char **)(event + hist_field->field->offset); 278 279 return (u64)(unsigned long)*addr; 280 } 281 282 static u64 hist_field_log2(struct hist_field *hist_field, 283 struct tracing_map_elt *elt, 284 struct trace_buffer *buffer, 285 struct ring_buffer_event *rbe, 286 void *event) 287 { 288 struct hist_field *operand = hist_field->operands[0]; 289 290 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 291 292 return (u64) ilog2(roundup_pow_of_two(val)); 293 } 294 295 static u64 hist_field_bucket(struct hist_field *hist_field, 296 struct tracing_map_elt *elt, 297 struct trace_buffer *buffer, 298 struct ring_buffer_event *rbe, 299 void *event) 300 { 301 struct hist_field *operand = hist_field->operands[0]; 302 unsigned long buckets = hist_field->buckets; 303 304 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 305 306 if (WARN_ON_ONCE(!buckets)) 307 return val; 308 309 if (val >= LONG_MAX) 310 val = div64_ul(val, buckets); 311 else 312 val = (u64)((unsigned long)val / buckets); 313 return val * buckets; 314 } 315 316 static u64 hist_field_plus(struct hist_field *hist_field, 317 struct tracing_map_elt *elt, 318 struct trace_buffer *buffer, 319 struct ring_buffer_event *rbe, 320 void *event) 321 { 322 struct hist_field *operand1 = hist_field->operands[0]; 323 struct hist_field *operand2 = hist_field->operands[1]; 324 325 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 326 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 327 328 return val1 + val2; 329 } 330 331 static u64 hist_field_minus(struct hist_field *hist_field, 332 struct tracing_map_elt *elt, 333 struct trace_buffer *buffer, 334 struct ring_buffer_event *rbe, 335 void *event) 336 { 337 struct hist_field *operand1 = hist_field->operands[0]; 338 struct hist_field *operand2 = hist_field->operands[1]; 339 340 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 341 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 342 343 return val1 - val2; 344 } 345 346 static u64 hist_field_div(struct hist_field *hist_field, 347 struct tracing_map_elt *elt, 348 struct trace_buffer *buffer, 349 struct ring_buffer_event *rbe, 350 void *event) 351 { 352 struct hist_field *operand1 = hist_field->operands[0]; 353 struct hist_field *operand2 = hist_field->operands[1]; 354 355 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 356 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 357 358 /* Return -1 for the undefined case */ 359 if (!val2) 360 return -1; 361 362 /* Use shift if the divisor is a power of 2 */ 363 if (!(val2 & (val2 - 1))) 364 return val1 >> __ffs64(val2); 365 366 return div64_u64(val1, val2); 367 } 368 369 static u64 div_by_power_of_two(struct hist_field *hist_field, 370 struct tracing_map_elt *elt, 371 struct trace_buffer *buffer, 372 struct ring_buffer_event *rbe, 373 void *event) 374 { 375 struct hist_field *operand1 = hist_field->operands[0]; 376 struct hist_field *operand2 = hist_field->operands[1]; 377 378 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 379 380 return val1 >> __ffs64(operand2->constant); 381 } 382 383 static u64 div_by_not_power_of_two(struct hist_field *hist_field, 384 struct tracing_map_elt *elt, 385 struct trace_buffer *buffer, 386 struct ring_buffer_event *rbe, 387 void *event) 388 { 389 struct hist_field *operand1 = hist_field->operands[0]; 390 struct hist_field *operand2 = hist_field->operands[1]; 391 392 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 393 394 return div64_u64(val1, operand2->constant); 395 } 396 397 static u64 div_by_mult_and_shift(struct hist_field *hist_field, 398 struct tracing_map_elt *elt, 399 struct trace_buffer *buffer, 400 struct ring_buffer_event *rbe, 401 void *event) 402 { 403 struct hist_field *operand1 = hist_field->operands[0]; 404 struct hist_field *operand2 = hist_field->operands[1]; 405 406 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 407 408 /* 409 * If the divisor is a constant, do a multiplication and shift instead. 410 * 411 * Choose Z = some power of 2. If Y <= Z, then: 412 * X / Y = (X * (Z / Y)) / Z 413 * 414 * (Z / Y) is a constant (mult) which is calculated at parse time, so: 415 * X / Y = (X * mult) / Z 416 * 417 * The division by Z can be replaced by a shift since Z is a power of 2: 418 * X / Y = (X * mult) >> HIST_DIV_SHIFT 419 * 420 * As long, as X < Z the results will not be off by more than 1. 421 */ 422 if (val1 < (1 << HIST_DIV_SHIFT)) { 423 u64 mult = operand2->div_multiplier; 424 425 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT; 426 } 427 428 return div64_u64(val1, operand2->constant); 429 } 430 431 static u64 hist_field_mult(struct hist_field *hist_field, 432 struct tracing_map_elt *elt, 433 struct trace_buffer *buffer, 434 struct ring_buffer_event *rbe, 435 void *event) 436 { 437 struct hist_field *operand1 = hist_field->operands[0]; 438 struct hist_field *operand2 = hist_field->operands[1]; 439 440 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 441 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 442 443 return val1 * val2; 444 } 445 446 static u64 hist_field_unary_minus(struct hist_field *hist_field, 447 struct tracing_map_elt *elt, 448 struct trace_buffer *buffer, 449 struct ring_buffer_event *rbe, 450 void *event) 451 { 452 struct hist_field *operand = hist_field->operands[0]; 453 454 s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event); 455 u64 val = (u64)-sval; 456 457 return val; 458 } 459 460 #define DEFINE_HIST_FIELD_FN(type) \ 461 static u64 hist_field_##type(struct hist_field *hist_field, \ 462 struct tracing_map_elt *elt, \ 463 struct trace_buffer *buffer, \ 464 struct ring_buffer_event *rbe, \ 465 void *event) \ 466 { \ 467 type *addr = (type *)(event + hist_field->field->offset); \ 468 \ 469 return (u64)(unsigned long)*addr; \ 470 } 471 472 DEFINE_HIST_FIELD_FN(s64); 473 DEFINE_HIST_FIELD_FN(u64); 474 DEFINE_HIST_FIELD_FN(s32); 475 DEFINE_HIST_FIELD_FN(u32); 476 DEFINE_HIST_FIELD_FN(s16); 477 DEFINE_HIST_FIELD_FN(u16); 478 DEFINE_HIST_FIELD_FN(s8); 479 DEFINE_HIST_FIELD_FN(u8); 480 481 #define for_each_hist_field(i, hist_data) \ 482 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 483 484 #define for_each_hist_val_field(i, hist_data) \ 485 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 486 487 #define for_each_hist_key_field(i, hist_data) \ 488 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 489 490 #define HITCOUNT_IDX 0 491 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 492 493 enum hist_field_flags { 494 HIST_FIELD_FL_HITCOUNT = 1 << 0, 495 HIST_FIELD_FL_KEY = 1 << 1, 496 HIST_FIELD_FL_STRING = 1 << 2, 497 HIST_FIELD_FL_HEX = 1 << 3, 498 HIST_FIELD_FL_SYM = 1 << 4, 499 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 500 HIST_FIELD_FL_EXECNAME = 1 << 6, 501 HIST_FIELD_FL_SYSCALL = 1 << 7, 502 HIST_FIELD_FL_STACKTRACE = 1 << 8, 503 HIST_FIELD_FL_LOG2 = 1 << 9, 504 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 505 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 506 HIST_FIELD_FL_VAR = 1 << 12, 507 HIST_FIELD_FL_EXPR = 1 << 13, 508 HIST_FIELD_FL_VAR_REF = 1 << 14, 509 HIST_FIELD_FL_CPU = 1 << 15, 510 HIST_FIELD_FL_ALIAS = 1 << 16, 511 HIST_FIELD_FL_BUCKET = 1 << 17, 512 HIST_FIELD_FL_CONST = 1 << 18, 513 HIST_FIELD_FL_PERCENT = 1 << 19, 514 HIST_FIELD_FL_GRAPH = 1 << 20, 515 HIST_FIELD_FL_COMM = 1 << 21, 516 }; 517 518 struct var_defs { 519 unsigned int n_vars; 520 char *name[TRACING_MAP_VARS_MAX]; 521 char *expr[TRACING_MAP_VARS_MAX]; 522 }; 523 524 struct hist_trigger_attrs { 525 char *keys_str; 526 char *vals_str; 527 char *sort_key_str; 528 char *name; 529 char *clock; 530 bool pause; 531 bool cont; 532 bool clear; 533 bool ts_in_usecs; 534 bool no_hitcount; 535 unsigned int map_bits; 536 537 char *assignment_str[TRACING_MAP_VARS_MAX]; 538 unsigned int n_assignments; 539 540 char *action_str[HIST_ACTIONS_MAX]; 541 unsigned int n_actions; 542 543 struct var_defs var_defs; 544 }; 545 546 struct field_var { 547 struct hist_field *var; 548 struct hist_field *val; 549 }; 550 551 struct field_var_hist { 552 struct hist_trigger_data *hist_data; 553 char *cmd; 554 }; 555 556 struct hist_trigger_data { 557 struct hist_field *fields[HIST_FIELDS_MAX]; 558 unsigned int n_vals; 559 unsigned int n_keys; 560 unsigned int n_fields; 561 unsigned int n_vars; 562 unsigned int n_var_str; 563 unsigned int key_size; 564 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 565 unsigned int n_sort_keys; 566 struct trace_event_file *event_file; 567 struct hist_trigger_attrs *attrs; 568 struct tracing_map *map; 569 bool enable_timestamps; 570 bool remove; 571 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 572 unsigned int n_var_refs; 573 574 struct action_data *actions[HIST_ACTIONS_MAX]; 575 unsigned int n_actions; 576 577 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 578 unsigned int n_field_vars; 579 unsigned int n_field_var_str; 580 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 581 unsigned int n_field_var_hists; 582 583 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 584 unsigned int n_save_vars; 585 unsigned int n_save_var_str; 586 }; 587 588 struct action_data; 589 590 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 591 struct tracing_map_elt *elt, 592 struct trace_buffer *buffer, void *rec, 593 struct ring_buffer_event *rbe, void *key, 594 struct action_data *data, u64 *var_ref_vals); 595 596 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 597 598 enum handler_id { 599 HANDLER_ONMATCH = 1, 600 HANDLER_ONMAX, 601 HANDLER_ONCHANGE, 602 }; 603 604 enum action_id { 605 ACTION_SAVE = 1, 606 ACTION_TRACE, 607 ACTION_SNAPSHOT, 608 }; 609 610 struct action_data { 611 enum handler_id handler; 612 enum action_id action; 613 char *action_name; 614 action_fn_t fn; 615 616 unsigned int n_params; 617 char *params[SYNTH_FIELDS_MAX]; 618 619 /* 620 * When a histogram trigger is hit, the values of any 621 * references to variables, including variables being passed 622 * as parameters to synthetic events, are collected into a 623 * var_ref_vals array. This var_ref_idx array is an array of 624 * indices into the var_ref_vals array, one for each synthetic 625 * event param, and is passed to the synthetic event 626 * invocation. 627 */ 628 unsigned int var_ref_idx[SYNTH_FIELDS_MAX]; 629 struct synth_event *synth_event; 630 bool use_trace_keyword; 631 char *synth_event_name; 632 633 union { 634 struct { 635 char *event; 636 char *event_system; 637 } match_data; 638 639 struct { 640 /* 641 * var_str contains the $-unstripped variable 642 * name referenced by var_ref, and used when 643 * printing the action. Because var_ref 644 * creation is deferred to create_actions(), 645 * we need a per-action way to save it until 646 * then, thus var_str. 647 */ 648 char *var_str; 649 650 /* 651 * var_ref refers to the variable being 652 * tracked e.g onmax($var). 653 */ 654 struct hist_field *var_ref; 655 656 /* 657 * track_var contains the 'invisible' tracking 658 * variable created to keep the current 659 * e.g. max value. 660 */ 661 struct hist_field *track_var; 662 663 check_track_val_fn_t check_val; 664 action_fn_t save_data; 665 } track_data; 666 }; 667 }; 668 669 struct track_data { 670 u64 track_val; 671 bool updated; 672 673 unsigned int key_len; 674 void *key; 675 struct tracing_map_elt elt; 676 677 struct action_data *action_data; 678 struct hist_trigger_data *hist_data; 679 }; 680 681 struct hist_elt_data { 682 char *comm; 683 u64 *var_ref_vals; 684 int n_field_var_str; 685 char *field_var_str[] __counted_by(n_field_var_str); 686 }; 687 688 struct snapshot_context { 689 struct tracing_map_elt *elt; 690 void *key; 691 }; 692 693 /* 694 * Returns the specific division function to use if the divisor 695 * is constant. This avoids extra branches when the trigger is hit. 696 */ 697 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor) 698 { 699 u64 div = divisor->constant; 700 701 if (!(div & (div - 1))) 702 return HIST_FIELD_FN_DIV_POWER2; 703 704 /* If the divisor is too large, do a regular division */ 705 if (div > (1 << HIST_DIV_SHIFT)) 706 return HIST_FIELD_FN_DIV_NOT_POWER2; 707 708 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div); 709 return HIST_FIELD_FN_DIV_MULT_SHIFT; 710 } 711 712 static void track_data_free(struct track_data *track_data) 713 { 714 struct hist_elt_data *elt_data; 715 716 if (!track_data) 717 return; 718 719 kfree(track_data->key); 720 721 elt_data = track_data->elt.private_data; 722 if (elt_data) { 723 kfree(elt_data->comm); 724 kfree(elt_data); 725 } 726 727 kfree(track_data); 728 } 729 730 static struct track_data *track_data_alloc(unsigned int key_len, 731 struct action_data *action_data, 732 struct hist_trigger_data *hist_data) 733 { 734 struct track_data *data = kzalloc_obj(*data); 735 struct hist_elt_data *elt_data; 736 737 if (!data) 738 return ERR_PTR(-ENOMEM); 739 740 data->key = kzalloc(key_len, GFP_KERNEL); 741 if (!data->key) { 742 track_data_free(data); 743 return ERR_PTR(-ENOMEM); 744 } 745 746 data->key_len = key_len; 747 data->action_data = action_data; 748 data->hist_data = hist_data; 749 750 elt_data = kzalloc_obj(*elt_data); 751 if (!elt_data) { 752 track_data_free(data); 753 return ERR_PTR(-ENOMEM); 754 } 755 756 data->elt.private_data = elt_data; 757 758 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 759 if (!elt_data->comm) { 760 track_data_free(data); 761 return ERR_PTR(-ENOMEM); 762 } 763 764 return data; 765 } 766 767 #define HIST_PREFIX "hist:" 768 769 static char *last_cmd; 770 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 771 772 static int errpos(char *str) 773 { 774 if (!str || !last_cmd) 775 return 0; 776 777 return err_pos(last_cmd, str); 778 } 779 780 static void last_cmd_set(struct trace_event_file *file, char *str) 781 { 782 const char *system = NULL, *name = NULL; 783 struct trace_event_call *call; 784 785 if (!str) 786 return; 787 788 kfree(last_cmd); 789 790 last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str); 791 if (!last_cmd) 792 return; 793 794 if (file) { 795 call = file->event_call; 796 system = call->class->system; 797 if (system) { 798 name = trace_event_name(call); 799 if (!name) 800 system = NULL; 801 } 802 } 803 804 if (system) 805 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name); 806 } 807 808 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos) 809 { 810 if (!last_cmd) 811 return; 812 813 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 814 err_type, err_pos); 815 } 816 817 static void hist_err_clear(void) 818 { 819 if (last_cmd) 820 last_cmd[0] = '\0'; 821 last_cmd_loc[0] = '\0'; 822 } 823 824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 825 unsigned int *var_ref_idx); 826 827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 828 unsigned int *var_ref_idx) 829 { 830 struct tracepoint *tp = event->tp; 831 832 if (unlikely(static_key_enabled(&tp->key))) { 833 struct tracepoint_func *probe_func_ptr; 834 synth_probe_func_t probe_func; 835 void *__data; 836 837 if (!(cpu_online(raw_smp_processor_id()))) 838 return; 839 840 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 841 if (probe_func_ptr) { 842 do { 843 probe_func = probe_func_ptr->func; 844 __data = probe_func_ptr->data; 845 probe_func(__data, var_ref_vals, var_ref_idx); 846 } while ((++probe_func_ptr)->func); 847 } 848 } 849 } 850 851 static void action_trace(struct hist_trigger_data *hist_data, 852 struct tracing_map_elt *elt, 853 struct trace_buffer *buffer, void *rec, 854 struct ring_buffer_event *rbe, void *key, 855 struct action_data *data, u64 *var_ref_vals) 856 { 857 struct synth_event *event = data->synth_event; 858 859 trace_synth(event, var_ref_vals, data->var_ref_idx); 860 } 861 862 struct hist_var_data { 863 struct list_head list; 864 struct hist_trigger_data *hist_data; 865 }; 866 867 static u64 hist_field_timestamp(struct hist_field *hist_field, 868 struct tracing_map_elt *elt, 869 struct trace_buffer *buffer, 870 struct ring_buffer_event *rbe, 871 void *event) 872 { 873 struct hist_trigger_data *hist_data = hist_field->hist_data; 874 struct trace_array *tr = hist_data->event_file->tr; 875 876 u64 ts = ring_buffer_event_time_stamp(buffer, rbe); 877 878 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 879 ts = ns2usecs(ts); 880 881 return ts; 882 } 883 884 static u64 hist_field_cpu(struct hist_field *hist_field, 885 struct tracing_map_elt *elt, 886 struct trace_buffer *buffer, 887 struct ring_buffer_event *rbe, 888 void *event) 889 { 890 int cpu = smp_processor_id(); 891 892 return cpu; 893 } 894 895 static u64 hist_field_comm(struct hist_field *hist_field, 896 struct tracing_map_elt *elt, 897 struct trace_buffer *buffer, 898 struct ring_buffer_event *rbe, 899 void *event) 900 { 901 return (u64)(unsigned long)current->comm; 902 } 903 904 /** 905 * check_field_for_var_ref - Check if a VAR_REF field references a variable 906 * @hist_field: The VAR_REF field to check 907 * @var_data: The hist trigger that owns the variable 908 * @var_idx: The trigger variable identifier 909 * 910 * Check the given VAR_REF field to see whether or not it references 911 * the given variable associated with the given trigger. 912 * 913 * Return: The VAR_REF field if it does reference the variable, NULL if not 914 */ 915 static struct hist_field * 916 check_field_for_var_ref(struct hist_field *hist_field, 917 struct hist_trigger_data *var_data, 918 unsigned int var_idx) 919 { 920 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 921 922 if (hist_field && hist_field->var.idx == var_idx && 923 hist_field->var.hist_data == var_data) 924 return hist_field; 925 926 return NULL; 927 } 928 929 /** 930 * find_var_ref - Check if a trigger has a reference to a trigger variable 931 * @hist_data: The hist trigger that might have a reference to the variable 932 * @var_data: The hist trigger that owns the variable 933 * @var_idx: The trigger variable identifier 934 * 935 * Check the list of var_refs[] on the first hist trigger to see 936 * whether any of them are references to the variable on the second 937 * trigger. 938 * 939 * Return: The VAR_REF field referencing the variable if so, NULL if not 940 */ 941 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 942 struct hist_trigger_data *var_data, 943 unsigned int var_idx) 944 { 945 struct hist_field *hist_field; 946 unsigned int i; 947 948 for (i = 0; i < hist_data->n_var_refs; i++) { 949 hist_field = hist_data->var_refs[i]; 950 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 951 return hist_field; 952 } 953 954 return NULL; 955 } 956 957 /** 958 * find_any_var_ref - Check if there is a reference to a given trigger variable 959 * @hist_data: The hist trigger 960 * @var_idx: The trigger variable identifier 961 * 962 * Check to see whether the given variable is currently referenced by 963 * any other trigger. 964 * 965 * The trigger the variable is defined on is explicitly excluded - the 966 * assumption being that a self-reference doesn't prevent a trigger 967 * from being removed. 968 * 969 * Return: The VAR_REF field referencing the variable if so, NULL if not 970 */ 971 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 972 unsigned int var_idx) 973 { 974 struct trace_array *tr = hist_data->event_file->tr; 975 struct hist_field *found = NULL; 976 struct hist_var_data *var_data; 977 978 list_for_each_entry(var_data, &tr->hist_vars, list) { 979 if (var_data->hist_data == hist_data) 980 continue; 981 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 982 if (found) 983 break; 984 } 985 986 return found; 987 } 988 989 /** 990 * check_var_refs - Check if there is a reference to any of trigger's variables 991 * @hist_data: The hist trigger 992 * 993 * A trigger can define one or more variables. If any one of them is 994 * currently referenced by any other trigger, this function will 995 * determine that. 996 * 997 * Typically used to determine whether or not a trigger can be removed 998 * - if there are any references to a trigger's variables, it cannot. 999 * 1000 * Return: True if there is a reference to any of trigger's variables 1001 */ 1002 static bool check_var_refs(struct hist_trigger_data *hist_data) 1003 { 1004 struct hist_field *field; 1005 bool found = false; 1006 int i; 1007 1008 for_each_hist_field(i, hist_data) { 1009 field = hist_data->fields[i]; 1010 if (field && field->flags & HIST_FIELD_FL_VAR) { 1011 if (find_any_var_ref(hist_data, field->var.idx)) { 1012 found = true; 1013 break; 1014 } 1015 } 1016 } 1017 1018 return found; 1019 } 1020 1021 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1022 { 1023 struct trace_array *tr = hist_data->event_file->tr; 1024 struct hist_var_data *var_data, *found = NULL; 1025 1026 list_for_each_entry(var_data, &tr->hist_vars, list) { 1027 if (var_data->hist_data == hist_data) { 1028 found = var_data; 1029 break; 1030 } 1031 } 1032 1033 return found; 1034 } 1035 1036 static bool field_has_hist_vars(struct hist_field *hist_field, 1037 unsigned int level) 1038 { 1039 int i; 1040 1041 if (level > 3) 1042 return false; 1043 1044 if (!hist_field) 1045 return false; 1046 1047 if (hist_field->flags & HIST_FIELD_FL_VAR || 1048 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1049 return true; 1050 1051 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1052 struct hist_field *operand; 1053 1054 operand = hist_field->operands[i]; 1055 if (field_has_hist_vars(operand, level + 1)) 1056 return true; 1057 } 1058 1059 return false; 1060 } 1061 1062 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1063 { 1064 struct hist_field *hist_field; 1065 int i; 1066 1067 for_each_hist_field(i, hist_data) { 1068 hist_field = hist_data->fields[i]; 1069 if (field_has_hist_vars(hist_field, 0)) 1070 return true; 1071 } 1072 1073 return false; 1074 } 1075 1076 static int save_hist_vars(struct hist_trigger_data *hist_data) 1077 { 1078 struct trace_array *tr = hist_data->event_file->tr; 1079 struct hist_var_data *var_data; 1080 1081 var_data = find_hist_vars(hist_data); 1082 if (var_data) 1083 return 0; 1084 1085 if (tracing_check_open_get_tr(tr)) 1086 return -ENODEV; 1087 1088 var_data = kzalloc_obj(*var_data); 1089 if (!var_data) { 1090 trace_array_put(tr); 1091 return -ENOMEM; 1092 } 1093 1094 var_data->hist_data = hist_data; 1095 list_add(&var_data->list, &tr->hist_vars); 1096 1097 return 0; 1098 } 1099 1100 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1101 { 1102 struct trace_array *tr = hist_data->event_file->tr; 1103 struct hist_var_data *var_data; 1104 1105 var_data = find_hist_vars(hist_data); 1106 if (!var_data) 1107 return; 1108 1109 if (WARN_ON(check_var_refs(hist_data))) 1110 return; 1111 1112 list_del(&var_data->list); 1113 1114 kfree(var_data); 1115 1116 trace_array_put(tr); 1117 } 1118 1119 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1120 const char *var_name) 1121 { 1122 struct hist_field *hist_field, *found = NULL; 1123 int i; 1124 1125 for_each_hist_field(i, hist_data) { 1126 hist_field = hist_data->fields[i]; 1127 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1128 strcmp(hist_field->var.name, var_name) == 0) { 1129 found = hist_field; 1130 break; 1131 } 1132 } 1133 1134 return found; 1135 } 1136 1137 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1138 struct trace_event_file *file, 1139 const char *var_name) 1140 { 1141 struct hist_trigger_data *test_data; 1142 struct event_trigger_data *test; 1143 struct hist_field *hist_field; 1144 1145 lockdep_assert_held(&event_mutex); 1146 1147 hist_field = find_var_field(hist_data, var_name); 1148 if (hist_field) 1149 return hist_field; 1150 1151 list_for_each_entry(test, &file->triggers, list) { 1152 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1153 test_data = test->private_data; 1154 hist_field = find_var_field(test_data, var_name); 1155 if (hist_field) 1156 return hist_field; 1157 } 1158 } 1159 1160 return NULL; 1161 } 1162 1163 static struct trace_event_file *find_var_file(struct trace_array *tr, 1164 char *system, 1165 char *event_name, 1166 char *var_name) 1167 { 1168 struct hist_trigger_data *var_hist_data; 1169 struct hist_var_data *var_data; 1170 struct trace_event_file *file, *found = NULL; 1171 1172 if (system) 1173 return find_event_file(tr, system, event_name); 1174 1175 list_for_each_entry(var_data, &tr->hist_vars, list) { 1176 var_hist_data = var_data->hist_data; 1177 file = var_hist_data->event_file; 1178 if (file == found) 1179 continue; 1180 1181 if (find_var_field(var_hist_data, var_name)) { 1182 if (found) { 1183 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1184 return NULL; 1185 } 1186 1187 found = file; 1188 } 1189 } 1190 1191 return found; 1192 } 1193 1194 static struct hist_field *find_file_var(struct trace_event_file *file, 1195 const char *var_name) 1196 { 1197 struct hist_trigger_data *test_data; 1198 struct event_trigger_data *test; 1199 struct hist_field *hist_field; 1200 1201 lockdep_assert_held(&event_mutex); 1202 1203 list_for_each_entry(test, &file->triggers, list) { 1204 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1205 test_data = test->private_data; 1206 hist_field = find_var_field(test_data, var_name); 1207 if (hist_field) 1208 return hist_field; 1209 } 1210 } 1211 1212 return NULL; 1213 } 1214 1215 static struct hist_field * 1216 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1217 { 1218 struct trace_array *tr = hist_data->event_file->tr; 1219 struct hist_field *hist_field, *found = NULL; 1220 struct trace_event_file *file; 1221 unsigned int i; 1222 1223 for (i = 0; i < hist_data->n_actions; i++) { 1224 struct action_data *data = hist_data->actions[i]; 1225 1226 if (data->handler == HANDLER_ONMATCH) { 1227 char *system = data->match_data.event_system; 1228 char *event_name = data->match_data.event; 1229 1230 file = find_var_file(tr, system, event_name, var_name); 1231 if (!file) 1232 continue; 1233 hist_field = find_file_var(file, var_name); 1234 if (hist_field) { 1235 if (found) { 1236 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1237 errpos(var_name)); 1238 return ERR_PTR(-EINVAL); 1239 } 1240 1241 found = hist_field; 1242 } 1243 } 1244 } 1245 return found; 1246 } 1247 1248 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1249 char *system, 1250 char *event_name, 1251 char *var_name) 1252 { 1253 struct trace_array *tr = hist_data->event_file->tr; 1254 struct hist_field *hist_field = NULL; 1255 struct trace_event_file *file; 1256 1257 if (!system || !event_name) { 1258 hist_field = find_match_var(hist_data, var_name); 1259 if (IS_ERR(hist_field)) 1260 return NULL; 1261 if (hist_field) 1262 return hist_field; 1263 } 1264 1265 file = find_var_file(tr, system, event_name, var_name); 1266 if (!file) 1267 return NULL; 1268 1269 hist_field = find_file_var(file, var_name); 1270 1271 return hist_field; 1272 } 1273 1274 static u64 hist_field_var_ref(struct hist_field *hist_field, 1275 struct tracing_map_elt *elt, 1276 struct trace_buffer *buffer, 1277 struct ring_buffer_event *rbe, 1278 void *event) 1279 { 1280 struct hist_elt_data *elt_data; 1281 u64 var_val = 0; 1282 1283 if (WARN_ON_ONCE(!elt)) 1284 return var_val; 1285 1286 elt_data = elt->private_data; 1287 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1288 1289 return var_val; 1290 } 1291 1292 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1293 u64 *var_ref_vals, bool self) 1294 { 1295 struct hist_trigger_data *var_data; 1296 struct tracing_map_elt *var_elt; 1297 struct hist_field *hist_field; 1298 unsigned int i, var_idx; 1299 bool resolved = true; 1300 u64 var_val = 0; 1301 1302 for (i = 0; i < hist_data->n_var_refs; i++) { 1303 hist_field = hist_data->var_refs[i]; 1304 var_idx = hist_field->var.idx; 1305 var_data = hist_field->var.hist_data; 1306 1307 if (var_data == NULL) { 1308 resolved = false; 1309 break; 1310 } 1311 1312 if ((self && var_data != hist_data) || 1313 (!self && var_data == hist_data)) 1314 continue; 1315 1316 var_elt = tracing_map_lookup(var_data->map, key); 1317 if (!var_elt) { 1318 resolved = false; 1319 break; 1320 } 1321 1322 if (!tracing_map_var_set(var_elt, var_idx)) { 1323 resolved = false; 1324 break; 1325 } 1326 1327 if (self || !hist_field->read_once) 1328 var_val = tracing_map_read_var(var_elt, var_idx); 1329 else 1330 var_val = tracing_map_read_var_once(var_elt, var_idx); 1331 1332 var_ref_vals[i] = var_val; 1333 } 1334 1335 return resolved; 1336 } 1337 1338 static const char *hist_field_name(struct hist_field *field, 1339 unsigned int level) 1340 { 1341 const char *field_name = ""; 1342 1343 if (WARN_ON_ONCE(!field)) 1344 return field_name; 1345 1346 if (level > 1) 1347 return field_name; 1348 1349 if (field->field) 1350 field_name = field->field->name; 1351 else if (field->flags & HIST_FIELD_FL_LOG2 || 1352 field->flags & HIST_FIELD_FL_ALIAS || 1353 field->flags & HIST_FIELD_FL_BUCKET) 1354 field_name = hist_field_name(field->operands[0], ++level); 1355 else if (field->flags & HIST_FIELD_FL_CPU) 1356 field_name = "common_cpu"; 1357 else if (field->flags & HIST_FIELD_FL_COMM) 1358 field_name = "common_comm"; 1359 else if (field->flags & HIST_FIELD_FL_EXPR || 1360 field->flags & HIST_FIELD_FL_VAR_REF) { 1361 if (field->system) { 1362 static char full_name[MAX_FILTER_STR_VAL]; 1363 static char *fmt; 1364 int len; 1365 1366 fmt = field->flags & HIST_FIELD_FL_VAR_REF ? "%s.%s.$%s" : "%s.%s.%s"; 1367 1368 len = snprintf(full_name, sizeof(full_name), fmt, 1369 field->system, field->event_name, 1370 field->name); 1371 if (len < sizeof(full_name)) 1372 field_name = full_name; 1373 } else 1374 field_name = field->name; 1375 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1376 field_name = "common_timestamp"; 1377 else if (field->flags & HIST_FIELD_FL_STACKTRACE) { 1378 field_name = "common_stacktrace"; 1379 } else if (field->flags & HIST_FIELD_FL_HITCOUNT) 1380 field_name = "hitcount"; 1381 1382 if (field_name == NULL) 1383 field_name = ""; 1384 1385 return field_name; 1386 } 1387 1388 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed) 1389 { 1390 switch (field_size) { 1391 case 8: 1392 if (field_is_signed) 1393 return HIST_FIELD_FN_S64; 1394 else 1395 return HIST_FIELD_FN_U64; 1396 case 4: 1397 if (field_is_signed) 1398 return HIST_FIELD_FN_S32; 1399 else 1400 return HIST_FIELD_FN_U32; 1401 case 2: 1402 if (field_is_signed) 1403 return HIST_FIELD_FN_S16; 1404 else 1405 return HIST_FIELD_FN_U16; 1406 case 1: 1407 if (field_is_signed) 1408 return HIST_FIELD_FN_S8; 1409 else 1410 return HIST_FIELD_FN_U8; 1411 } 1412 1413 return HIST_FIELD_FN_NOP; 1414 } 1415 1416 static int parse_map_size(char *str) 1417 { 1418 unsigned long size, map_bits; 1419 int ret; 1420 1421 ret = kstrtoul(str, 0, &size); 1422 if (ret) 1423 goto out; 1424 1425 map_bits = ilog2(roundup_pow_of_two(size)); 1426 if (map_bits < TRACING_MAP_BITS_MIN || 1427 map_bits > TRACING_MAP_BITS_MAX) 1428 ret = -EINVAL; 1429 else 1430 ret = map_bits; 1431 out: 1432 return ret; 1433 } 1434 1435 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1436 { 1437 unsigned int i; 1438 1439 if (!attrs) 1440 return; 1441 1442 for (i = 0; i < attrs->n_assignments; i++) 1443 kfree(attrs->assignment_str[i]); 1444 1445 for (i = 0; i < attrs->n_actions; i++) 1446 kfree(attrs->action_str[i]); 1447 1448 kfree(attrs->name); 1449 kfree(attrs->sort_key_str); 1450 kfree(attrs->keys_str); 1451 kfree(attrs->vals_str); 1452 kfree(attrs->clock); 1453 kfree(attrs); 1454 } 1455 1456 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1457 { 1458 int ret = -EINVAL; 1459 1460 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1461 return ret; 1462 1463 if ((str_has_prefix(str, "onmatch(")) || 1464 (str_has_prefix(str, "onmax(")) || 1465 (str_has_prefix(str, "onchange("))) { 1466 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1467 if (!attrs->action_str[attrs->n_actions]) { 1468 ret = -ENOMEM; 1469 return ret; 1470 } 1471 attrs->n_actions++; 1472 ret = 0; 1473 } 1474 return ret; 1475 } 1476 1477 static int parse_assignment(struct trace_array *tr, 1478 char *str, struct hist_trigger_attrs *attrs) 1479 { 1480 int len, ret = 0; 1481 1482 if ((len = str_has_prefix(str, "key=")) || 1483 (len = str_has_prefix(str, "keys="))) { 1484 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1485 if (!attrs->keys_str) { 1486 ret = -ENOMEM; 1487 goto out; 1488 } 1489 } else if ((len = str_has_prefix(str, "val=")) || 1490 (len = str_has_prefix(str, "vals=")) || 1491 (len = str_has_prefix(str, "values="))) { 1492 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1493 if (!attrs->vals_str) { 1494 ret = -ENOMEM; 1495 goto out; 1496 } 1497 } else if ((len = str_has_prefix(str, "sort="))) { 1498 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1499 if (!attrs->sort_key_str) { 1500 ret = -ENOMEM; 1501 goto out; 1502 } 1503 } else if (str_has_prefix(str, "name=")) { 1504 attrs->name = kstrdup(str, GFP_KERNEL); 1505 if (!attrs->name) { 1506 ret = -ENOMEM; 1507 goto out; 1508 } 1509 } else if ((len = str_has_prefix(str, "clock="))) { 1510 str += len; 1511 1512 str = strstrip(str); 1513 attrs->clock = kstrdup(str, GFP_KERNEL); 1514 if (!attrs->clock) { 1515 ret = -ENOMEM; 1516 goto out; 1517 } 1518 } else if ((len = str_has_prefix(str, "size="))) { 1519 int map_bits = parse_map_size(str + len); 1520 1521 if (map_bits < 0) { 1522 ret = map_bits; 1523 goto out; 1524 } 1525 attrs->map_bits = map_bits; 1526 } else { 1527 char *assignment; 1528 1529 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1530 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1531 ret = -EINVAL; 1532 goto out; 1533 } 1534 1535 assignment = kstrdup(str, GFP_KERNEL); 1536 if (!assignment) { 1537 ret = -ENOMEM; 1538 goto out; 1539 } 1540 1541 attrs->assignment_str[attrs->n_assignments++] = assignment; 1542 } 1543 out: 1544 return ret; 1545 } 1546 1547 static struct hist_trigger_attrs * 1548 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1549 { 1550 struct hist_trigger_attrs *attrs; 1551 int ret = 0; 1552 1553 attrs = kzalloc_obj(*attrs); 1554 if (!attrs) 1555 return ERR_PTR(-ENOMEM); 1556 1557 while (trigger_str) { 1558 char *str = strsep(&trigger_str, ":"); 1559 char *rhs; 1560 1561 rhs = strchr(str, '='); 1562 if (rhs) { 1563 if (!strlen(++rhs)) { 1564 ret = -EINVAL; 1565 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1566 goto free; 1567 } 1568 ret = parse_assignment(tr, str, attrs); 1569 if (ret) 1570 goto free; 1571 } else if (strcmp(str, "nohitcount") == 0 || 1572 strcmp(str, "NOHC") == 0) 1573 attrs->no_hitcount = true; 1574 else if (strcmp(str, "pause") == 0) 1575 attrs->pause = true; 1576 else if ((strcmp(str, "cont") == 0) || 1577 (strcmp(str, "continue") == 0)) 1578 attrs->cont = true; 1579 else if (strcmp(str, "clear") == 0) 1580 attrs->clear = true; 1581 else { 1582 ret = parse_action(str, attrs); 1583 if (ret) 1584 goto free; 1585 } 1586 } 1587 1588 if (!attrs->keys_str) { 1589 ret = -EINVAL; 1590 goto free; 1591 } 1592 1593 if (!attrs->clock) { 1594 attrs->clock = kstrdup("global", GFP_KERNEL); 1595 if (!attrs->clock) { 1596 ret = -ENOMEM; 1597 goto free; 1598 } 1599 } 1600 1601 return attrs; 1602 free: 1603 destroy_hist_trigger_attrs(attrs); 1604 1605 return ERR_PTR(ret); 1606 } 1607 1608 static inline void save_comm(char *comm, struct task_struct *task) 1609 { 1610 if (!task->pid) { 1611 strcpy(comm, "<idle>"); 1612 return; 1613 } 1614 1615 if (WARN_ON_ONCE(task->pid < 0)) { 1616 strcpy(comm, "<XXX>"); 1617 return; 1618 } 1619 1620 strscpy(comm, task->comm, TASK_COMM_LEN); 1621 } 1622 1623 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1624 { 1625 unsigned int i; 1626 1627 for (i = 0; i < elt_data->n_field_var_str; i++) 1628 kfree(elt_data->field_var_str[i]); 1629 1630 kfree(elt_data->comm); 1631 kfree(elt_data); 1632 } 1633 1634 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1635 { 1636 struct hist_elt_data *elt_data = elt->private_data; 1637 1638 hist_elt_data_free(elt_data); 1639 } 1640 1641 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1642 { 1643 struct hist_trigger_data *hist_data = elt->map->private_data; 1644 unsigned int size = TASK_COMM_LEN; 1645 struct hist_elt_data *elt_data; 1646 struct hist_field *hist_field; 1647 unsigned int i, n_str; 1648 1649 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); 1650 1651 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + 1652 hist_data->n_var_str; 1653 if (n_str > SYNTH_FIELDS_MAX) 1654 return -EINVAL; 1655 1656 elt_data = kzalloc_flex(*elt_data, field_var_str, n_str); 1657 if (!elt_data) 1658 return -ENOMEM; 1659 1660 elt_data->n_field_var_str = n_str; 1661 1662 for_each_hist_field(i, hist_data) { 1663 hist_field = hist_data->fields[i]; 1664 1665 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) { 1666 elt_data->comm = kzalloc(size, GFP_KERNEL); 1667 if (!elt_data->comm) { 1668 kfree(elt_data); 1669 return -ENOMEM; 1670 } 1671 break; 1672 } 1673 } 1674 1675 size = STR_VAR_LEN_MAX; 1676 1677 for (i = 0; i < n_str; i++) { 1678 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1679 if (!elt_data->field_var_str[i]) { 1680 hist_elt_data_free(elt_data); 1681 return -ENOMEM; 1682 } 1683 } 1684 1685 elt->private_data = elt_data; 1686 1687 return 0; 1688 } 1689 1690 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1691 { 1692 struct hist_elt_data *elt_data = elt->private_data; 1693 1694 if (elt_data->comm) 1695 save_comm(elt_data->comm, current); 1696 } 1697 1698 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1699 .elt_alloc = hist_trigger_elt_data_alloc, 1700 .elt_free = hist_trigger_elt_data_free, 1701 .elt_init = hist_trigger_elt_data_init, 1702 }; 1703 1704 static const char *get_hist_field_flags(struct hist_field *hist_field) 1705 { 1706 const char *flags_str = NULL; 1707 1708 if (hist_field->flags & HIST_FIELD_FL_HEX) 1709 flags_str = "hex"; 1710 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1711 flags_str = "sym"; 1712 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1713 flags_str = "sym-offset"; 1714 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1715 flags_str = "execname"; 1716 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1717 flags_str = "syscall"; 1718 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1719 flags_str = "log2"; 1720 else if (hist_field->flags & HIST_FIELD_FL_BUCKET) 1721 flags_str = "buckets"; 1722 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1723 flags_str = "usecs"; 1724 else if (hist_field->flags & HIST_FIELD_FL_PERCENT) 1725 flags_str = "percent"; 1726 else if (hist_field->flags & HIST_FIELD_FL_GRAPH) 1727 flags_str = "graph"; 1728 else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 1729 flags_str = "stacktrace"; 1730 1731 return flags_str; 1732 } 1733 1734 static bool expr_field_str(struct hist_field *field, struct seq_buf *s) 1735 { 1736 const char *field_name; 1737 1738 if (field->flags & HIST_FIELD_FL_VAR_REF) { 1739 if (!field->system) 1740 seq_buf_putc(s, '$'); 1741 } else if (field->flags & HIST_FIELD_FL_CONST) 1742 seq_buf_printf(s, "%llu", field->constant); 1743 1744 field_name = hist_field_name(field, 0); 1745 if (!field_name) 1746 return false; 1747 1748 seq_buf_puts(s, field_name); 1749 1750 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1751 const char *flags_str = get_hist_field_flags(field); 1752 1753 if (flags_str) 1754 seq_buf_printf(s, ".%s", flags_str); 1755 } 1756 1757 return !seq_buf_has_overflowed(s); 1758 } 1759 1760 static char *expr_str(struct hist_field *field, unsigned int level) 1761 { 1762 char *expr __free(kfree) = NULL; 1763 struct seq_buf s; 1764 1765 if (level > 1) 1766 return ERR_PTR(-EINVAL); 1767 1768 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1769 if (!expr) 1770 return ERR_PTR(-ENOMEM); 1771 1772 seq_buf_init(&s, expr, MAX_FILTER_STR_VAL); 1773 1774 if (!field->operands[0]) { 1775 if (!expr_field_str(field, &s)) 1776 return ERR_PTR(-E2BIG); 1777 1778 return_ptr(expr); 1779 } 1780 1781 if (field->operator == FIELD_OP_UNARY_MINUS) { 1782 char *subexpr; 1783 1784 subexpr = expr_str(field->operands[0], ++level); 1785 if (IS_ERR(subexpr)) 1786 return subexpr; 1787 1788 seq_buf_printf(&s, "-(%s)", subexpr); 1789 kfree(subexpr); 1790 1791 if (seq_buf_has_overflowed(&s)) 1792 return ERR_PTR(-E2BIG); 1793 1794 return_ptr(expr); 1795 } 1796 1797 if (!expr_field_str(field->operands[0], &s)) 1798 return ERR_PTR(-E2BIG); 1799 1800 switch (field->operator) { 1801 case FIELD_OP_MINUS: 1802 seq_buf_putc(&s, '-'); 1803 break; 1804 case FIELD_OP_PLUS: 1805 seq_buf_putc(&s, '+'); 1806 break; 1807 case FIELD_OP_DIV: 1808 seq_buf_putc(&s, '/'); 1809 break; 1810 case FIELD_OP_MULT: 1811 seq_buf_putc(&s, '*'); 1812 break; 1813 default: 1814 return ERR_PTR(-EINVAL); 1815 } 1816 1817 if (seq_buf_has_overflowed(&s) || 1818 !expr_field_str(field->operands[1], &s)) 1819 return ERR_PTR(-E2BIG); 1820 1821 return_ptr(expr); 1822 } 1823 1824 /* 1825 * If field_op != FIELD_OP_NONE, *sep points to the root operator 1826 * of the expression tree to be evaluated. 1827 */ 1828 static int contains_operator(char *str, char **sep) 1829 { 1830 enum field_op_id field_op = FIELD_OP_NONE; 1831 char *minus_op, *plus_op, *div_op, *mult_op; 1832 1833 1834 /* 1835 * Report the last occurrence of the operators first, so that the 1836 * expression is evaluated left to right. This is important since 1837 * subtraction and division are not associative. 1838 * 1839 * e.g 1840 * 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2 1841 * 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2 1842 */ 1843 1844 /* 1845 * First, find lower precedence addition and subtraction 1846 * since the expression will be evaluated recursively. 1847 */ 1848 minus_op = strrchr(str, '-'); 1849 if (minus_op) { 1850 /* 1851 * Unary minus is not supported in sub-expressions. If 1852 * present, it is always the next root operator. 1853 */ 1854 if (minus_op == str) { 1855 field_op = FIELD_OP_UNARY_MINUS; 1856 goto out; 1857 } 1858 1859 field_op = FIELD_OP_MINUS; 1860 } 1861 1862 plus_op = strrchr(str, '+'); 1863 if (plus_op || minus_op) { 1864 /* 1865 * For operators of the same precedence use to rightmost as the 1866 * root, so that the expression is evaluated left to right. 1867 */ 1868 if (plus_op > minus_op) 1869 field_op = FIELD_OP_PLUS; 1870 goto out; 1871 } 1872 1873 /* 1874 * Multiplication and division have higher precedence than addition and 1875 * subtraction. 1876 */ 1877 div_op = strrchr(str, '/'); 1878 if (div_op) 1879 field_op = FIELD_OP_DIV; 1880 1881 mult_op = strrchr(str, '*'); 1882 /* 1883 * For operators of the same precedence use to rightmost as the 1884 * root, so that the expression is evaluated left to right. 1885 */ 1886 if (mult_op > div_op) 1887 field_op = FIELD_OP_MULT; 1888 1889 out: 1890 if (sep) { 1891 switch (field_op) { 1892 case FIELD_OP_UNARY_MINUS: 1893 case FIELD_OP_MINUS: 1894 *sep = minus_op; 1895 break; 1896 case FIELD_OP_PLUS: 1897 *sep = plus_op; 1898 break; 1899 case FIELD_OP_DIV: 1900 *sep = div_op; 1901 break; 1902 case FIELD_OP_MULT: 1903 *sep = mult_op; 1904 break; 1905 case FIELD_OP_NONE: 1906 default: 1907 *sep = NULL; 1908 break; 1909 } 1910 } 1911 1912 return field_op; 1913 } 1914 1915 static void __destroy_hist_field(struct hist_field *hist_field) 1916 { 1917 kfree(hist_field->var.name); 1918 kfree(hist_field->name); 1919 1920 /* Can likely be a const */ 1921 kfree_const(hist_field->type); 1922 1923 kfree(hist_field->system); 1924 kfree(hist_field->event_name); 1925 1926 kfree(hist_field); 1927 } 1928 1929 static void destroy_hist_field(struct hist_field *hist_field, 1930 unsigned int level) 1931 { 1932 unsigned int i; 1933 1934 if (level > 3) 1935 return; 1936 1937 if (!hist_field) 1938 return; 1939 1940 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1941 return; /* var refs will be destroyed separately */ 1942 1943 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1944 destroy_hist_field(hist_field->operands[i], level + 1); 1945 1946 __destroy_hist_field(hist_field); 1947 } 1948 1949 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1950 struct ftrace_event_field *field, 1951 unsigned long flags, 1952 char *var_name) 1953 { 1954 struct hist_field *hist_field; 1955 1956 if (field && is_function_field(field)) 1957 return NULL; 1958 1959 hist_field = kzalloc_obj(struct hist_field); 1960 if (!hist_field) 1961 return NULL; 1962 1963 hist_field->hist_data = hist_data; 1964 1965 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1966 goto out; /* caller will populate */ 1967 1968 if (flags & HIST_FIELD_FL_VAR_REF) { 1969 hist_field->fn_num = HIST_FIELD_FN_VAR_REF; 1970 goto out; 1971 } 1972 1973 if (flags & HIST_FIELD_FL_HITCOUNT) { 1974 hist_field->fn_num = HIST_FIELD_FN_COUNTER; 1975 hist_field->size = sizeof(u64); 1976 hist_field->type = "u64"; 1977 goto out; 1978 } 1979 1980 if (flags & HIST_FIELD_FL_CONST) { 1981 hist_field->fn_num = HIST_FIELD_FN_CONST; 1982 hist_field->size = sizeof(u64); 1983 hist_field->type = "u64"; 1984 goto out; 1985 } 1986 1987 if (flags & HIST_FIELD_FL_STACKTRACE) { 1988 if (field) 1989 hist_field->fn_num = HIST_FIELD_FN_STACK; 1990 else 1991 hist_field->fn_num = HIST_FIELD_FN_NOP; 1992 hist_field->size = HIST_STACKTRACE_SIZE; 1993 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL); 1994 if (!hist_field->type) 1995 goto free; 1996 goto out; 1997 } 1998 1999 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) { 2000 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET); 2001 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 : 2002 HIST_FIELD_FN_BUCKET; 2003 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2004 if (!hist_field->operands[0]) 2005 goto free; 2006 hist_field->size = hist_field->operands[0]->size; 2007 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL); 2008 if (!hist_field->type) 2009 goto free; 2010 goto out; 2011 } 2012 2013 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2014 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP; 2015 hist_field->size = sizeof(u64); 2016 hist_field->type = "u64"; 2017 goto out; 2018 } 2019 2020 if (flags & HIST_FIELD_FL_CPU) { 2021 hist_field->fn_num = HIST_FIELD_FN_CPU; 2022 hist_field->size = sizeof(int); 2023 hist_field->type = "unsigned int"; 2024 goto out; 2025 } 2026 2027 if (flags & HIST_FIELD_FL_COMM) { 2028 hist_field->fn_num = HIST_FIELD_FN_COMM; 2029 hist_field->size = MAX_FILTER_STR_VAL; 2030 hist_field->type = "char[]"; 2031 goto out; 2032 } 2033 2034 if (WARN_ON_ONCE(!field)) 2035 goto out; 2036 2037 /* Pointers to strings are just pointers and dangerous to dereference */ 2038 if (is_string_field(field) && 2039 (field->filter_type != FILTER_PTR_STRING)) { 2040 flags |= HIST_FIELD_FL_STRING; 2041 2042 hist_field->size = MAX_FILTER_STR_VAL; 2043 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2044 if (!hist_field->type) 2045 goto free; 2046 2047 if (field->filter_type == FILTER_STATIC_STRING) { 2048 hist_field->fn_num = HIST_FIELD_FN_STRING; 2049 hist_field->size = field->size; 2050 } else if (field->filter_type == FILTER_DYN_STRING) { 2051 hist_field->fn_num = HIST_FIELD_FN_DYNSTRING; 2052 } else if (field->filter_type == FILTER_RDYN_STRING) 2053 hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; 2054 else 2055 hist_field->fn_num = HIST_FIELD_FN_PSTRING; 2056 } else if (field->filter_type == FILTER_STACKTRACE) { 2057 flags |= HIST_FIELD_FL_STACKTRACE; 2058 2059 hist_field->size = MAX_FILTER_STR_VAL; 2060 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2061 if (!hist_field->type) 2062 goto free; 2063 2064 hist_field->fn_num = HIST_FIELD_FN_STACK; 2065 } else { 2066 hist_field->size = field->size; 2067 hist_field->is_signed = field->is_signed; 2068 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2069 if (!hist_field->type) 2070 goto free; 2071 2072 hist_field->fn_num = select_value_fn(field->size, 2073 field->is_signed); 2074 if (hist_field->fn_num == HIST_FIELD_FN_NOP) { 2075 destroy_hist_field(hist_field, 0); 2076 return NULL; 2077 } 2078 } 2079 out: 2080 hist_field->field = field; 2081 hist_field->flags = flags; 2082 2083 if (var_name) { 2084 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2085 if (!hist_field->var.name) 2086 goto free; 2087 } 2088 2089 return hist_field; 2090 free: 2091 destroy_hist_field(hist_field, 0); 2092 return NULL; 2093 } 2094 2095 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2096 { 2097 unsigned int i; 2098 2099 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2100 if (hist_data->fields[i]) { 2101 destroy_hist_field(hist_data->fields[i], 0); 2102 hist_data->fields[i] = NULL; 2103 } 2104 } 2105 2106 for (i = 0; i < hist_data->n_var_refs; i++) { 2107 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2108 __destroy_hist_field(hist_data->var_refs[i]); 2109 hist_data->var_refs[i] = NULL; 2110 } 2111 } 2112 2113 static int init_var_ref(struct hist_field *ref_field, 2114 struct hist_field *var_field, 2115 char *system, char *event_name) 2116 { 2117 int err = 0; 2118 2119 ref_field->var.idx = var_field->var.idx; 2120 ref_field->var.hist_data = var_field->hist_data; 2121 ref_field->size = var_field->size; 2122 ref_field->is_signed = var_field->is_signed; 2123 ref_field->flags |= var_field->flags & 2124 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2125 2126 if (system) { 2127 ref_field->system = kstrdup(system, GFP_KERNEL); 2128 if (!ref_field->system) 2129 return -ENOMEM; 2130 } 2131 2132 if (event_name) { 2133 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2134 if (!ref_field->event_name) { 2135 err = -ENOMEM; 2136 goto free; 2137 } 2138 } 2139 2140 if (var_field->var.name) { 2141 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2142 if (!ref_field->name) { 2143 err = -ENOMEM; 2144 goto free; 2145 } 2146 } else if (var_field->name) { 2147 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2148 if (!ref_field->name) { 2149 err = -ENOMEM; 2150 goto free; 2151 } 2152 } 2153 2154 ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL); 2155 if (!ref_field->type) { 2156 err = -ENOMEM; 2157 goto free; 2158 } 2159 out: 2160 return err; 2161 free: 2162 kfree(ref_field->system); 2163 ref_field->system = NULL; 2164 kfree(ref_field->event_name); 2165 ref_field->event_name = NULL; 2166 kfree(ref_field->name); 2167 ref_field->name = NULL; 2168 2169 goto out; 2170 } 2171 2172 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 2173 struct hist_field *var_field) 2174 { 2175 struct hist_field *ref_field; 2176 int i; 2177 2178 for (i = 0; i < hist_data->n_var_refs; i++) { 2179 ref_field = hist_data->var_refs[i]; 2180 if (ref_field->var.idx == var_field->var.idx && 2181 ref_field->var.hist_data == var_field->hist_data) 2182 return i; 2183 } 2184 2185 return -ENOENT; 2186 } 2187 2188 /** 2189 * create_var_ref - Create a variable reference and attach it to trigger 2190 * @hist_data: The trigger that will be referencing the variable 2191 * @var_field: The VAR field to create a reference to 2192 * @system: The optional system string 2193 * @event_name: The optional event_name string 2194 * 2195 * Given a variable hist_field, create a VAR_REF hist_field that 2196 * represents a reference to it. 2197 * 2198 * This function also adds the reference to the trigger that 2199 * now references the variable. 2200 * 2201 * Return: The VAR_REF field if successful, NULL if not 2202 */ 2203 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2204 struct hist_field *var_field, 2205 char *system, char *event_name) 2206 { 2207 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2208 struct hist_field *ref_field; 2209 int i; 2210 2211 /* Check if the variable already exists */ 2212 for (i = 0; i < hist_data->n_var_refs; i++) { 2213 ref_field = hist_data->var_refs[i]; 2214 if (ref_field->var.idx == var_field->var.idx && 2215 ref_field->var.hist_data == var_field->hist_data) 2216 return ref_field; 2217 } 2218 /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ 2219 if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) 2220 return NULL; 2221 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2222 if (ref_field) { 2223 if (init_var_ref(ref_field, var_field, system, event_name)) { 2224 __destroy_hist_field(ref_field); 2225 return NULL; 2226 } 2227 2228 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2229 ref_field->var_ref_idx = hist_data->n_var_refs++; 2230 } 2231 2232 return ref_field; 2233 } 2234 2235 static bool is_var_ref(char *var_name) 2236 { 2237 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2238 return false; 2239 2240 return true; 2241 } 2242 2243 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2244 char *var_name) 2245 { 2246 char *name, *field; 2247 unsigned int i; 2248 2249 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2250 name = hist_data->attrs->var_defs.name[i]; 2251 2252 if (strcmp(var_name, name) == 0) { 2253 field = hist_data->attrs->var_defs.expr[i]; 2254 if (contains_operator(field, NULL) || is_var_ref(field)) 2255 continue; 2256 return field; 2257 } 2258 } 2259 2260 return NULL; 2261 } 2262 2263 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2264 char *system, char *event_name, 2265 char *var_name) 2266 { 2267 struct trace_event_call *call; 2268 2269 if (system && event_name) { 2270 call = hist_data->event_file->event_call; 2271 2272 if (strcmp(system, call->class->system) != 0) 2273 return NULL; 2274 2275 if (strcmp(event_name, trace_event_name(call)) != 0) 2276 return NULL; 2277 } 2278 2279 if (!!system != !!event_name) 2280 return NULL; 2281 2282 if (!is_var_ref(var_name)) 2283 return NULL; 2284 2285 var_name++; 2286 2287 return field_name_from_var(hist_data, var_name); 2288 } 2289 2290 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2291 char *system, char *event_name, 2292 char *var_name) 2293 { 2294 struct hist_field *var_field = NULL, *ref_field = NULL; 2295 struct trace_array *tr = hist_data->event_file->tr; 2296 2297 if (!is_var_ref(var_name)) 2298 return NULL; 2299 2300 var_name++; 2301 2302 var_field = find_event_var(hist_data, system, event_name, var_name); 2303 if (var_field) 2304 ref_field = create_var_ref(hist_data, var_field, 2305 system, event_name); 2306 2307 if (!ref_field) 2308 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2309 2310 return ref_field; 2311 } 2312 2313 static struct ftrace_event_field * 2314 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2315 char *field_str, unsigned long *flags, unsigned long *buckets) 2316 { 2317 struct ftrace_event_field *field = NULL; 2318 char *field_name, *modifier, *str; 2319 struct trace_array *tr = file->tr; 2320 bool stack_modifier = false; 2321 2322 modifier = str = kstrdup(field_str, GFP_KERNEL); 2323 if (!modifier) 2324 return ERR_PTR(-ENOMEM); 2325 2326 field_name = strsep(&modifier, "."); 2327 if (modifier) { 2328 if (strcmp(modifier, "hex") == 0) 2329 *flags |= HIST_FIELD_FL_HEX; 2330 else if (strcmp(modifier, "sym") == 0) 2331 *flags |= HIST_FIELD_FL_SYM; 2332 /* 2333 * 'sym-offset' occurrences in the trigger string are modified 2334 * to 'symXoffset' to simplify arithmetic expression parsing. 2335 */ 2336 else if (strcmp(modifier, "symXoffset") == 0) 2337 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2338 else if ((strcmp(modifier, "execname") == 0) && 2339 (strcmp(field_name, "common_pid") == 0)) 2340 *flags |= HIST_FIELD_FL_EXECNAME; 2341 else if (strcmp(modifier, "syscall") == 0) 2342 *flags |= HIST_FIELD_FL_SYSCALL; 2343 else if (strcmp(modifier, "stacktrace") == 0) { 2344 *flags |= HIST_FIELD_FL_STACKTRACE; 2345 stack_modifier = true; 2346 } else if (strcmp(modifier, "log2") == 0) 2347 *flags |= HIST_FIELD_FL_LOG2; 2348 else if (strcmp(modifier, "usecs") == 0) 2349 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2350 else if (strncmp(modifier, "bucket", 6) == 0) { 2351 int ret; 2352 2353 modifier += 6; 2354 2355 if (*modifier == 's') 2356 modifier++; 2357 if (*modifier != '=') 2358 goto error; 2359 modifier++; 2360 ret = kstrtoul(modifier, 0, buckets); 2361 if (ret || !(*buckets)) 2362 goto error; 2363 *flags |= HIST_FIELD_FL_BUCKET; 2364 } else if (strncmp(modifier, "percent", 7) == 0) { 2365 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2366 goto error; 2367 *flags |= HIST_FIELD_FL_PERCENT; 2368 } else if (strncmp(modifier, "graph", 5) == 0) { 2369 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2370 goto error; 2371 *flags |= HIST_FIELD_FL_GRAPH; 2372 } else { 2373 error: 2374 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2375 field = ERR_PTR(-EINVAL); 2376 goto out; 2377 } 2378 } 2379 2380 if (strcmp(field_name, "common_timestamp") == 0) { 2381 *flags |= HIST_FIELD_FL_TIMESTAMP; 2382 hist_data->enable_timestamps = true; 2383 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2384 hist_data->attrs->ts_in_usecs = true; 2385 } else if (strcmp(field_name, "common_stacktrace") == 0) { 2386 *flags |= HIST_FIELD_FL_STACKTRACE; 2387 } else if (strcmp(field_name, "common_cpu") == 0) { 2388 *flags |= HIST_FIELD_FL_CPU; 2389 } else if (strcmp(field_name, "common_comm") == 0) { 2390 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2391 } else if (strcmp(field_name, "hitcount") == 0) 2392 *flags |= HIST_FIELD_FL_HITCOUNT; 2393 else { 2394 field = trace_find_event_field(file->event_call, field_name); 2395 if (!field || !field->size) { 2396 /* 2397 * For backward compatibility, if field_name 2398 * was "cpu" or "stacktrace", then we treat this 2399 * the same as common_cpu and common_stacktrace 2400 * respectively. This also works for "CPU", and 2401 * "STACKTRACE". 2402 */ 2403 if (field && field->filter_type == FILTER_CPU) { 2404 *flags |= HIST_FIELD_FL_CPU; 2405 } else if (field && field->filter_type == FILTER_STACKTRACE) { 2406 *flags |= HIST_FIELD_FL_STACKTRACE; 2407 field = NULL; 2408 } else if (field && field->filter_type == FILTER_COMM) { 2409 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2410 } else { 2411 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, 2412 errpos(field_name)); 2413 field = ERR_PTR(-EINVAL); 2414 goto out; 2415 } 2416 } 2417 } 2418 2419 if (stack_modifier && 2420 (!field || field->filter_type != FILTER_STACKTRACE)) { 2421 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 2422 field = ERR_PTR(-EINVAL); 2423 } 2424 out: 2425 kfree(str); 2426 2427 return field; 2428 } 2429 2430 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2431 struct hist_field *var_ref, 2432 char *var_name) 2433 { 2434 struct hist_field *alias = NULL; 2435 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2436 2437 alias = create_hist_field(hist_data, NULL, flags, var_name); 2438 if (!alias) 2439 return NULL; 2440 2441 alias->fn_num = var_ref->fn_num; 2442 alias->operands[0] = var_ref; 2443 2444 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2445 destroy_hist_field(alias, 0); 2446 return NULL; 2447 } 2448 2449 alias->var_ref_idx = var_ref->var_ref_idx; 2450 2451 return alias; 2452 } 2453 2454 static struct hist_field *parse_const(struct hist_trigger_data *hist_data, 2455 char *str, char *var_name, 2456 unsigned long *flags) 2457 { 2458 struct trace_array *tr = hist_data->event_file->tr; 2459 struct hist_field *field = NULL; 2460 u64 constant; 2461 2462 if (kstrtoull(str, 0, &constant)) { 2463 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str)); 2464 return NULL; 2465 } 2466 2467 *flags |= HIST_FIELD_FL_CONST; 2468 field = create_hist_field(hist_data, NULL, *flags, var_name); 2469 if (!field) 2470 return NULL; 2471 2472 field->constant = constant; 2473 2474 return field; 2475 } 2476 2477 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2478 struct trace_event_file *file, char *str, 2479 unsigned long *flags, char *var_name) 2480 { 2481 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2482 struct ftrace_event_field *field = NULL; 2483 struct hist_field *hist_field = NULL; 2484 unsigned long buckets = 0; 2485 int ret = 0; 2486 2487 if (isdigit(str[0])) { 2488 hist_field = parse_const(hist_data, str, var_name, flags); 2489 if (!hist_field) { 2490 ret = -EINVAL; 2491 goto out; 2492 } 2493 return hist_field; 2494 } 2495 2496 s = strchr(str, '.'); 2497 if (s) { 2498 s = strchr(++s, '.'); 2499 if (s) { 2500 ref_system = strsep(&str, "."); 2501 if (!str) { 2502 ret = -EINVAL; 2503 goto out; 2504 } 2505 ref_event = strsep(&str, "."); 2506 if (!str) { 2507 ret = -EINVAL; 2508 goto out; 2509 } 2510 ref_var = str; 2511 } 2512 } 2513 2514 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2515 if (!s) { 2516 hist_field = parse_var_ref(hist_data, ref_system, 2517 ref_event, ref_var); 2518 if (hist_field) { 2519 if (var_name) { 2520 hist_field = create_alias(hist_data, hist_field, var_name); 2521 if (!hist_field) { 2522 ret = -ENOMEM; 2523 goto out; 2524 } 2525 } 2526 return hist_field; 2527 } 2528 } else 2529 str = s; 2530 2531 field = parse_field(hist_data, file, str, flags, &buckets); 2532 if (IS_ERR(field)) { 2533 ret = PTR_ERR(field); 2534 goto out; 2535 } 2536 2537 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2538 if (!hist_field) { 2539 ret = -ENOMEM; 2540 goto out; 2541 } 2542 hist_field->buckets = buckets; 2543 2544 return hist_field; 2545 out: 2546 return ERR_PTR(ret); 2547 } 2548 2549 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2550 struct trace_event_file *file, 2551 char *str, unsigned long flags, 2552 char *var_name, unsigned int *n_subexprs); 2553 2554 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2555 struct trace_event_file *file, 2556 char *str, unsigned long flags, 2557 char *var_name, unsigned int *n_subexprs) 2558 { 2559 struct hist_field *operand1, *expr = NULL; 2560 unsigned long operand_flags; 2561 int ret = 0; 2562 char *s; 2563 2564 /* Unary minus operator, increment n_subexprs */ 2565 ++*n_subexprs; 2566 2567 /* we support only -(xxx) i.e. explicit parens required */ 2568 2569 if (*n_subexprs > 3) { 2570 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2571 ret = -EINVAL; 2572 goto free; 2573 } 2574 2575 str++; /* skip leading '-' */ 2576 2577 s = strchr(str, '('); 2578 if (s) 2579 str++; 2580 else { 2581 ret = -EINVAL; 2582 goto free; 2583 } 2584 2585 s = strrchr(str, ')'); 2586 if (s) { 2587 /* unary minus not supported in sub-expressions */ 2588 if (*(s+1) != '\0') { 2589 hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR, 2590 errpos(str)); 2591 ret = -EINVAL; 2592 goto free; 2593 } 2594 *s = '\0'; 2595 } 2596 else { 2597 ret = -EINVAL; /* no closing ')' */ 2598 goto free; 2599 } 2600 2601 flags |= HIST_FIELD_FL_EXPR; 2602 expr = create_hist_field(hist_data, NULL, flags, var_name); 2603 if (!expr) { 2604 ret = -ENOMEM; 2605 goto free; 2606 } 2607 2608 operand_flags = 0; 2609 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2610 if (IS_ERR(operand1)) { 2611 ret = PTR_ERR(operand1); 2612 goto free; 2613 } 2614 if (operand1->flags & HIST_FIELD_FL_STRING) { 2615 /* String type can not be the operand of unary operator. */ 2616 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2617 destroy_hist_field(operand1, 0); 2618 ret = -EINVAL; 2619 goto free; 2620 } 2621 2622 expr->flags |= operand1->flags & 2623 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2624 expr->fn_num = HIST_FIELD_FN_UMINUS; 2625 expr->operands[0] = operand1; 2626 expr->size = operand1->size; 2627 expr->is_signed = operand1->is_signed; 2628 expr->operator = FIELD_OP_UNARY_MINUS; 2629 expr->name = expr_str(expr, 0); 2630 if (IS_ERR(expr->name)) { 2631 ret = PTR_ERR(expr->name); 2632 expr->name = NULL; 2633 goto free; 2634 } 2635 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2636 if (!expr->type) { 2637 ret = -ENOMEM; 2638 goto free; 2639 } 2640 2641 return expr; 2642 free: 2643 destroy_hist_field(expr, 0); 2644 return ERR_PTR(ret); 2645 } 2646 2647 /* 2648 * If the operands are var refs, return pointers the 2649 * variable(s) referenced in var1 and var2, else NULL. 2650 */ 2651 static int check_expr_operands(struct trace_array *tr, 2652 struct hist_field *operand1, 2653 struct hist_field *operand2, 2654 struct hist_field **var1, 2655 struct hist_field **var2) 2656 { 2657 unsigned long operand1_flags = operand1->flags; 2658 unsigned long operand2_flags = operand2->flags; 2659 2660 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2661 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2662 struct hist_field *var; 2663 2664 var = find_var_field(operand1->var.hist_data, operand1->name); 2665 if (!var) 2666 return -EINVAL; 2667 operand1_flags = var->flags; 2668 *var1 = var; 2669 } 2670 2671 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2672 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2673 struct hist_field *var; 2674 2675 var = find_var_field(operand2->var.hist_data, operand2->name); 2676 if (!var) 2677 return -EINVAL; 2678 operand2_flags = var->flags; 2679 *var2 = var; 2680 } 2681 2682 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2683 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2684 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2685 return -EINVAL; 2686 } 2687 2688 return 0; 2689 } 2690 2691 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2692 struct trace_event_file *file, 2693 char *str, unsigned long flags, 2694 char *var_name, unsigned int *n_subexprs) 2695 { 2696 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2697 struct hist_field *var1 = NULL, *var2 = NULL; 2698 unsigned long operand_flags, operand2_flags; 2699 int field_op, ret = -EINVAL; 2700 char *sep, *operand1_str; 2701 enum hist_field_fn op_fn; 2702 bool combine_consts; 2703 2704 if (*n_subexprs > 3) { 2705 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2706 return ERR_PTR(-EINVAL); 2707 } 2708 2709 field_op = contains_operator(str, &sep); 2710 2711 if (field_op == FIELD_OP_NONE) 2712 return parse_atom(hist_data, file, str, &flags, var_name); 2713 2714 if (field_op == FIELD_OP_UNARY_MINUS) 2715 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs); 2716 2717 /* Binary operator found, increment n_subexprs */ 2718 ++*n_subexprs; 2719 2720 /* Split the expression string at the root operator */ 2721 if (!sep) 2722 return ERR_PTR(-EINVAL); 2723 2724 *sep = '\0'; 2725 operand1_str = str; 2726 str = sep+1; 2727 2728 /* Binary operator requires both operands */ 2729 if (*operand1_str == '\0' || *str == '\0') 2730 return ERR_PTR(-EINVAL); 2731 2732 operand_flags = 0; 2733 2734 /* LHS of string is an expression e.g. a+b in a+b+c */ 2735 operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs); 2736 if (IS_ERR(operand1)) 2737 return ERR_CAST(operand1); 2738 2739 if (operand1->flags & HIST_FIELD_FL_STRING) { 2740 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str)); 2741 ret = -EINVAL; 2742 goto free_op1; 2743 } 2744 2745 /* RHS of string is another expression e.g. c in a+b+c */ 2746 operand_flags = 0; 2747 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2748 if (IS_ERR(operand2)) { 2749 ret = PTR_ERR(operand2); 2750 goto free_op1; 2751 } 2752 if (operand2->flags & HIST_FIELD_FL_STRING) { 2753 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2754 ret = -EINVAL; 2755 goto free_operands; 2756 } 2757 2758 switch (field_op) { 2759 case FIELD_OP_MINUS: 2760 op_fn = HIST_FIELD_FN_MINUS; 2761 break; 2762 case FIELD_OP_PLUS: 2763 op_fn = HIST_FIELD_FN_PLUS; 2764 break; 2765 case FIELD_OP_DIV: 2766 op_fn = HIST_FIELD_FN_DIV; 2767 break; 2768 case FIELD_OP_MULT: 2769 op_fn = HIST_FIELD_FN_MULT; 2770 break; 2771 default: 2772 ret = -EINVAL; 2773 goto free_operands; 2774 } 2775 2776 ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2); 2777 if (ret) 2778 goto free_operands; 2779 2780 operand_flags = var1 ? var1->flags : operand1->flags; 2781 operand2_flags = var2 ? var2->flags : operand2->flags; 2782 2783 /* 2784 * If both operands are constant, the expression can be 2785 * collapsed to a single constant. 2786 */ 2787 combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST; 2788 2789 flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR; 2790 2791 flags |= operand1->flags & 2792 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2793 2794 expr = create_hist_field(hist_data, NULL, flags, var_name); 2795 if (!expr) { 2796 ret = -ENOMEM; 2797 goto free_operands; 2798 } 2799 2800 operand1->read_once = true; 2801 operand2->read_once = true; 2802 2803 /* The operands are now owned and free'd by 'expr' */ 2804 expr->operands[0] = operand1; 2805 expr->operands[1] = operand2; 2806 2807 if (field_op == FIELD_OP_DIV && 2808 operand2_flags & HIST_FIELD_FL_CONST) { 2809 u64 divisor = var2 ? var2->constant : operand2->constant; 2810 2811 if (!divisor) { 2812 hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str)); 2813 ret = -EDOM; 2814 goto free_expr; 2815 } 2816 2817 /* 2818 * Copy the divisor here so we don't have to look it up 2819 * later if this is a var ref 2820 */ 2821 operand2->constant = divisor; 2822 op_fn = hist_field_get_div_fn(operand2); 2823 } 2824 2825 expr->fn_num = op_fn; 2826 2827 if (combine_consts) { 2828 if (var1) 2829 expr->operands[0] = var1; 2830 if (var2) 2831 expr->operands[1] = var2; 2832 2833 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL); 2834 expr->fn_num = HIST_FIELD_FN_CONST; 2835 2836 expr->operands[0] = NULL; 2837 expr->operands[1] = NULL; 2838 2839 /* 2840 * var refs won't be destroyed immediately 2841 * See: destroy_hist_field() 2842 */ 2843 destroy_hist_field(operand2, 0); 2844 destroy_hist_field(operand1, 0); 2845 2846 expr->name = expr_str(expr, 0); 2847 if (IS_ERR(expr->name)) { 2848 ret = PTR_ERR(expr->name); 2849 expr->name = NULL; 2850 goto free_expr; 2851 } 2852 } else { 2853 /* The operand sizes should be the same, so just pick one */ 2854 expr->size = operand1->size; 2855 expr->is_signed = operand1->is_signed; 2856 2857 expr->operator = field_op; 2858 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2859 if (!expr->type) { 2860 ret = -ENOMEM; 2861 goto free_expr; 2862 } 2863 2864 expr->name = expr_str(expr, 0); 2865 if (IS_ERR(expr->name)) { 2866 ret = PTR_ERR(expr->name); 2867 expr->name = NULL; 2868 goto free_expr; 2869 } 2870 } 2871 2872 return expr; 2873 2874 free_operands: 2875 destroy_hist_field(operand2, 0); 2876 free_op1: 2877 destroy_hist_field(operand1, 0); 2878 return ERR_PTR(ret); 2879 2880 free_expr: 2881 destroy_hist_field(expr, 0); 2882 return ERR_PTR(ret); 2883 } 2884 2885 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2886 struct trace_event_file *file) 2887 { 2888 struct event_trigger_data *test; 2889 2890 lockdep_assert_held(&event_mutex); 2891 2892 list_for_each_entry(test, &file->triggers, list) { 2893 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2894 if (test->private_data == hist_data) 2895 return test->filter_str; 2896 } 2897 } 2898 2899 return NULL; 2900 } 2901 2902 static struct event_command trigger_hist_cmd; 2903 static int event_hist_trigger_parse(struct event_command *cmd_ops, 2904 struct trace_event_file *file, 2905 char *glob, char *cmd, 2906 char *param_and_filter); 2907 2908 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2909 struct hist_trigger_data *hist_data, 2910 unsigned int n_keys) 2911 { 2912 struct hist_field *target_hist_field, *hist_field; 2913 unsigned int n, i, j; 2914 2915 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2916 return false; 2917 2918 i = hist_data->n_vals; 2919 j = target_hist_data->n_vals; 2920 2921 for (n = 0; n < n_keys; n++) { 2922 hist_field = hist_data->fields[i + n]; 2923 target_hist_field = target_hist_data->fields[j + n]; 2924 2925 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2926 return false; 2927 if (hist_field->size != target_hist_field->size) 2928 return false; 2929 if (hist_field->is_signed != target_hist_field->is_signed) 2930 return false; 2931 } 2932 2933 return true; 2934 } 2935 2936 static struct hist_trigger_data * 2937 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2938 struct trace_event_file *file) 2939 { 2940 struct hist_trigger_data *hist_data; 2941 struct event_trigger_data *test; 2942 unsigned int n_keys; 2943 2944 lockdep_assert_held(&event_mutex); 2945 2946 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2947 2948 list_for_each_entry(test, &file->triggers, list) { 2949 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2950 hist_data = test->private_data; 2951 2952 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2953 return hist_data; 2954 } 2955 } 2956 2957 return NULL; 2958 } 2959 2960 static struct trace_event_file *event_file(struct trace_array *tr, 2961 char *system, char *event_name) 2962 { 2963 struct trace_event_file *file; 2964 2965 file = __find_event_file(tr, system, event_name); 2966 if (!file) 2967 return ERR_PTR(-EINVAL); 2968 2969 return file; 2970 } 2971 2972 static struct hist_field * 2973 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2974 char *system, char *event_name, char *field_name) 2975 { 2976 struct hist_field *event_var; 2977 char *synthetic_name; 2978 struct seq_buf s; 2979 2980 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2981 if (!synthetic_name) 2982 return ERR_PTR(-ENOMEM); 2983 2984 seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); 2985 seq_buf_printf(&s, "synthetic_%s", field_name); 2986 2987 /* Terminate synthetic_name with a NUL. */ 2988 seq_buf_str(&s); 2989 2990 if (seq_buf_has_overflowed(&s)) { 2991 kfree(synthetic_name); 2992 return ERR_PTR(-E2BIG); 2993 } 2994 2995 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2996 2997 kfree(synthetic_name); 2998 2999 return event_var; 3000 } 3001 3002 /** 3003 * create_field_var_hist - Automatically create a histogram and var for a field 3004 * @target_hist_data: The target hist trigger 3005 * @subsys_name: Optional subsystem name 3006 * @event_name: Optional event name 3007 * @field_name: The name of the field (and the resulting variable) 3008 * 3009 * Hist trigger actions fetch data from variables, not directly from 3010 * events. However, for convenience, users are allowed to directly 3011 * specify an event field in an action, which will be automatically 3012 * converted into a variable on their behalf. 3013 * 3014 * If a user specifies a field on an event that isn't the event the 3015 * histogram currently being defined (the target event histogram), the 3016 * only way that can be accomplished is if a new hist trigger is 3017 * created and the field variable defined on that. 3018 * 3019 * This function creates a new histogram compatible with the target 3020 * event (meaning a histogram with the same key as the target 3021 * histogram), and creates a variable for the specified field, but 3022 * with 'synthetic_' prepended to the variable name in order to avoid 3023 * collision with normal field variables. 3024 * 3025 * Return: The variable created for the field. 3026 */ 3027 static struct hist_field * 3028 create_field_var_hist(struct hist_trigger_data *target_hist_data, 3029 char *subsys_name, char *event_name, char *field_name) 3030 { 3031 struct trace_array *tr = target_hist_data->event_file->tr; 3032 struct hist_trigger_data *hist_data; 3033 unsigned int i, n, first = true; 3034 struct field_var_hist *var_hist; 3035 struct trace_event_file *file; 3036 struct hist_field *key_field; 3037 struct hist_field *event_var; 3038 char *saved_filter; 3039 struct seq_buf s; 3040 char *cmd; 3041 int ret; 3042 3043 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 3044 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3045 return ERR_PTR(-EINVAL); 3046 } 3047 3048 file = event_file(tr, subsys_name, event_name); 3049 3050 if (IS_ERR(file)) { 3051 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3052 ret = PTR_ERR(file); 3053 return ERR_PTR(ret); 3054 } 3055 3056 /* 3057 * Look for a histogram compatible with target. We'll use the 3058 * found histogram specification to create a new matching 3059 * histogram with our variable on it. target_hist_data is not 3060 * yet a registered histogram so we can't use that. 3061 */ 3062 hist_data = find_compatible_hist(target_hist_data, file); 3063 if (!hist_data) { 3064 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3065 return ERR_PTR(-EINVAL); 3066 } 3067 3068 /* See if a synthetic field variable has already been created */ 3069 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3070 event_name, field_name); 3071 if (!IS_ERR_OR_NULL(event_var)) 3072 return event_var; 3073 3074 var_hist = kzalloc_obj(*var_hist); 3075 if (!var_hist) 3076 return ERR_PTR(-ENOMEM); 3077 3078 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3079 if (!cmd) { 3080 kfree(var_hist); 3081 return ERR_PTR(-ENOMEM); 3082 } 3083 3084 seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL); 3085 3086 /* Use the same keys as the compatible histogram */ 3087 seq_buf_puts(&s, "keys="); 3088 3089 for_each_hist_key_field(i, hist_data) { 3090 key_field = hist_data->fields[i]; 3091 if (!first) 3092 seq_buf_putc(&s, ','); 3093 seq_buf_puts(&s, key_field->field->name); 3094 first = false; 3095 } 3096 3097 /* Create the synthetic field variable specification */ 3098 seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name); 3099 3100 /* Use the same filter as the compatible histogram */ 3101 saved_filter = find_trigger_filter(hist_data, file); 3102 if (saved_filter) 3103 seq_buf_printf(&s, " if %s", saved_filter); 3104 3105 /* Terminate cmd with a NUL. */ 3106 seq_buf_str(&s); 3107 3108 if (seq_buf_has_overflowed(&s)) { 3109 kfree(cmd); 3110 kfree(var_hist); 3111 return ERR_PTR(-E2BIG); 3112 } 3113 3114 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3115 if (!var_hist->cmd) { 3116 kfree(cmd); 3117 kfree(var_hist); 3118 return ERR_PTR(-ENOMEM); 3119 } 3120 3121 /* Save the compatible histogram information */ 3122 var_hist->hist_data = hist_data; 3123 3124 /* Create the new histogram with our variable */ 3125 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 3126 "", "hist", cmd); 3127 if (ret) { 3128 kfree(cmd); 3129 kfree(var_hist->cmd); 3130 kfree(var_hist); 3131 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3132 return ERR_PTR(ret); 3133 } 3134 3135 kfree(cmd); 3136 3137 /* If we can't find the variable, something went wrong */ 3138 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3139 event_name, field_name); 3140 if (IS_ERR_OR_NULL(event_var)) { 3141 kfree(var_hist->cmd); 3142 kfree(var_hist); 3143 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3144 return ERR_PTR(-EINVAL); 3145 } 3146 3147 n = target_hist_data->n_field_var_hists; 3148 target_hist_data->field_var_hists[n] = var_hist; 3149 target_hist_data->n_field_var_hists++; 3150 3151 return event_var; 3152 } 3153 3154 static struct hist_field * 3155 find_target_event_var(struct hist_trigger_data *hist_data, 3156 char *subsys_name, char *event_name, char *var_name) 3157 { 3158 struct trace_event_file *file = hist_data->event_file; 3159 struct hist_field *hist_field = NULL; 3160 3161 if (subsys_name) { 3162 struct trace_event_call *call; 3163 3164 if (!event_name) 3165 return NULL; 3166 3167 call = file->event_call; 3168 3169 if (strcmp(subsys_name, call->class->system) != 0) 3170 return NULL; 3171 3172 if (strcmp(event_name, trace_event_name(call)) != 0) 3173 return NULL; 3174 } 3175 3176 hist_field = find_var_field(hist_data, var_name); 3177 3178 return hist_field; 3179 } 3180 3181 static inline void __update_field_vars(struct tracing_map_elt *elt, 3182 struct trace_buffer *buffer, 3183 struct ring_buffer_event *rbe, 3184 void *rec, 3185 struct field_var **field_vars, 3186 unsigned int n_field_vars, 3187 unsigned int field_var_str_start) 3188 { 3189 struct hist_elt_data *elt_data = elt->private_data; 3190 unsigned int i, j, var_idx; 3191 u64 var_val; 3192 3193 /* Make sure stacktrace can fit in the string variable length */ 3194 BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) > STR_VAR_LEN_MAX); 3195 3196 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3197 struct field_var *field_var = field_vars[i]; 3198 struct hist_field *var = field_var->var; 3199 struct hist_field *val = field_var->val; 3200 3201 var_val = hist_fn_call(val, elt, buffer, rbe, rec); 3202 var_idx = var->var.idx; 3203 3204 if (val->flags & (HIST_FIELD_FL_STRING | 3205 HIST_FIELD_FL_STACKTRACE)) { 3206 char *str = elt_data->field_var_str[j++]; 3207 char *val_str = (char *)(uintptr_t)var_val; 3208 unsigned int size; 3209 3210 if (val->flags & HIST_FIELD_FL_STRING) { 3211 size = min(val->size, STR_VAR_LEN_MAX); 3212 strscpy(str, val_str, size); 3213 } else { 3214 char *stack_start = str + sizeof(unsigned long); 3215 int e; 3216 3217 e = stack_trace_save((void *)stack_start, 3218 HIST_STACKTRACE_DEPTH, 3219 HIST_STACKTRACE_SKIP); 3220 if (e < HIST_STACKTRACE_DEPTH - 1) 3221 ((unsigned long *)stack_start)[e] = 0; 3222 *((unsigned long *)str) = e; 3223 } 3224 var_val = (u64)(uintptr_t)str; 3225 } 3226 tracing_map_set_var(elt, var_idx, var_val); 3227 } 3228 } 3229 3230 static void update_field_vars(struct hist_trigger_data *hist_data, 3231 struct tracing_map_elt *elt, 3232 struct trace_buffer *buffer, 3233 struct ring_buffer_event *rbe, 3234 void *rec) 3235 { 3236 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars, 3237 hist_data->n_field_vars, 0); 3238 } 3239 3240 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3241 struct tracing_map_elt *elt, 3242 struct trace_buffer *buffer, void *rec, 3243 struct ring_buffer_event *rbe, void *key, 3244 struct action_data *data, u64 *var_ref_vals) 3245 { 3246 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars, 3247 hist_data->n_save_vars, hist_data->n_field_var_str); 3248 } 3249 3250 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3251 struct trace_event_file *file, 3252 char *name, int size, const char *type) 3253 { 3254 struct hist_field *var; 3255 int idx; 3256 3257 if (find_var(hist_data, file, name) && !hist_data->remove) { 3258 var = ERR_PTR(-EINVAL); 3259 goto out; 3260 } 3261 3262 var = kzalloc_obj(struct hist_field); 3263 if (!var) { 3264 var = ERR_PTR(-ENOMEM); 3265 goto out; 3266 } 3267 3268 idx = tracing_map_add_var(hist_data->map); 3269 if (idx < 0) { 3270 kfree(var); 3271 var = ERR_PTR(-EINVAL); 3272 goto out; 3273 } 3274 3275 var->flags = HIST_FIELD_FL_VAR; 3276 var->var.idx = idx; 3277 var->var.hist_data = var->hist_data = hist_data; 3278 var->size = size; 3279 var->var.name = kstrdup(name, GFP_KERNEL); 3280 var->type = kstrdup_const(type, GFP_KERNEL); 3281 if (!var->var.name || !var->type) { 3282 kfree_const(var->type); 3283 kfree(var->var.name); 3284 kfree(var); 3285 var = ERR_PTR(-ENOMEM); 3286 } 3287 out: 3288 return var; 3289 } 3290 3291 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3292 struct trace_event_file *file, 3293 char *field_name) 3294 { 3295 struct hist_field *val = NULL, *var = NULL; 3296 unsigned long flags = HIST_FIELD_FL_VAR; 3297 struct trace_array *tr = file->tr; 3298 struct field_var *field_var; 3299 int ret = 0; 3300 3301 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3302 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3303 ret = -EINVAL; 3304 goto err; 3305 } 3306 3307 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3308 if (IS_ERR(val)) { 3309 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3310 ret = PTR_ERR(val); 3311 goto err; 3312 } 3313 3314 var = create_var(hist_data, file, field_name, val->size, val->type); 3315 if (IS_ERR(var)) { 3316 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3317 destroy_hist_field(val, 0); 3318 ret = PTR_ERR(var); 3319 goto err; 3320 } 3321 3322 field_var = kzalloc_obj(struct field_var); 3323 if (!field_var) { 3324 destroy_hist_field(val, 0); 3325 kfree_const(var->type); 3326 kfree(var->var.name); 3327 kfree(var); 3328 ret = -ENOMEM; 3329 goto err; 3330 } 3331 3332 field_var->var = var; 3333 field_var->val = val; 3334 out: 3335 return field_var; 3336 err: 3337 field_var = ERR_PTR(ret); 3338 goto out; 3339 } 3340 3341 /** 3342 * create_target_field_var - Automatically create a variable for a field 3343 * @target_hist_data: The target hist trigger 3344 * @subsys_name: Optional subsystem name 3345 * @event_name: Optional event name 3346 * @var_name: The name of the field (and the resulting variable) 3347 * 3348 * Hist trigger actions fetch data from variables, not directly from 3349 * events. However, for convenience, users are allowed to directly 3350 * specify an event field in an action, which will be automatically 3351 * converted into a variable on their behalf. 3352 * 3353 * This function creates a field variable with the name var_name on 3354 * the hist trigger currently being defined on the target event. If 3355 * subsys_name and event_name are specified, this function simply 3356 * verifies that they do in fact match the target event subsystem and 3357 * event name. 3358 * 3359 * Return: The variable created for the field. 3360 */ 3361 static struct field_var * 3362 create_target_field_var(struct hist_trigger_data *target_hist_data, 3363 char *subsys_name, char *event_name, char *var_name) 3364 { 3365 struct trace_event_file *file = target_hist_data->event_file; 3366 3367 if (subsys_name) { 3368 struct trace_event_call *call; 3369 3370 if (!event_name) 3371 return NULL; 3372 3373 call = file->event_call; 3374 3375 if (strcmp(subsys_name, call->class->system) != 0) 3376 return NULL; 3377 3378 if (strcmp(event_name, trace_event_name(call)) != 0) 3379 return NULL; 3380 } 3381 3382 return create_field_var(target_hist_data, file, var_name); 3383 } 3384 3385 static bool check_track_val_max(u64 track_val, u64 var_val) 3386 { 3387 if (var_val <= track_val) 3388 return false; 3389 3390 return true; 3391 } 3392 3393 static bool check_track_val_changed(u64 track_val, u64 var_val) 3394 { 3395 if (var_val == track_val) 3396 return false; 3397 3398 return true; 3399 } 3400 3401 static u64 get_track_val(struct hist_trigger_data *hist_data, 3402 struct tracing_map_elt *elt, 3403 struct action_data *data) 3404 { 3405 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3406 u64 track_val; 3407 3408 track_val = tracing_map_read_var(elt, track_var_idx); 3409 3410 return track_val; 3411 } 3412 3413 static void save_track_val(struct hist_trigger_data *hist_data, 3414 struct tracing_map_elt *elt, 3415 struct action_data *data, u64 var_val) 3416 { 3417 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3418 3419 tracing_map_set_var(elt, track_var_idx, var_val); 3420 } 3421 3422 static void save_track_data(struct hist_trigger_data *hist_data, 3423 struct tracing_map_elt *elt, 3424 struct trace_buffer *buffer, void *rec, 3425 struct ring_buffer_event *rbe, void *key, 3426 struct action_data *data, u64 *var_ref_vals) 3427 { 3428 if (data->track_data.save_data) 3429 data->track_data.save_data(hist_data, elt, buffer, rec, rbe, 3430 key, data, var_ref_vals); 3431 } 3432 3433 static bool check_track_val(struct tracing_map_elt *elt, 3434 struct action_data *data, 3435 u64 var_val) 3436 { 3437 struct hist_trigger_data *hist_data; 3438 u64 track_val; 3439 3440 hist_data = data->track_data.track_var->hist_data; 3441 track_val = get_track_val(hist_data, elt, data); 3442 3443 return data->track_data.check_val(track_val, var_val); 3444 } 3445 3446 #ifdef CONFIG_TRACER_SNAPSHOT 3447 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3448 { 3449 /* called with tr->max_lock held */ 3450 struct track_data *track_data = tr->cond_snapshot->cond_data; 3451 struct hist_elt_data *elt_data, *track_elt_data; 3452 struct snapshot_context *context = cond_data; 3453 struct action_data *action; 3454 u64 track_val; 3455 3456 if (!track_data) 3457 return false; 3458 3459 action = track_data->action_data; 3460 3461 track_val = get_track_val(track_data->hist_data, context->elt, 3462 track_data->action_data); 3463 3464 if (!action->track_data.check_val(track_data->track_val, track_val)) 3465 return false; 3466 3467 track_data->track_val = track_val; 3468 memcpy(track_data->key, context->key, track_data->key_len); 3469 3470 elt_data = context->elt->private_data; 3471 track_elt_data = track_data->elt.private_data; 3472 if (elt_data->comm) 3473 strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3474 3475 track_data->updated = true; 3476 3477 return true; 3478 } 3479 3480 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3481 struct tracing_map_elt *elt, 3482 struct trace_buffer *buffer, void *rec, 3483 struct ring_buffer_event *rbe, void *key, 3484 struct action_data *data, 3485 u64 *var_ref_vals) 3486 { 3487 struct trace_event_file *file = hist_data->event_file; 3488 struct snapshot_context context; 3489 3490 context.elt = elt; 3491 context.key = key; 3492 3493 tracing_snapshot_cond(file->tr, &context); 3494 } 3495 3496 static void hist_trigger_print_key(struct seq_file *m, 3497 struct hist_trigger_data *hist_data, 3498 void *key, 3499 struct tracing_map_elt *elt); 3500 3501 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3502 { 3503 unsigned int i; 3504 3505 if (!hist_data->n_actions) 3506 return NULL; 3507 3508 for (i = 0; i < hist_data->n_actions; i++) { 3509 struct action_data *data = hist_data->actions[i]; 3510 3511 if (data->action == ACTION_SNAPSHOT) 3512 return data; 3513 } 3514 3515 return NULL; 3516 } 3517 3518 static void track_data_snapshot_print(struct seq_file *m, 3519 struct hist_trigger_data *hist_data) 3520 { 3521 struct trace_event_file *file = hist_data->event_file; 3522 struct track_data *track_data; 3523 struct action_data *action; 3524 3525 track_data = tracing_cond_snapshot_data(file->tr); 3526 if (!track_data) 3527 return; 3528 3529 if (!track_data->updated) 3530 return; 3531 3532 action = snapshot_action(hist_data); 3533 if (!action) 3534 return; 3535 3536 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3537 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3538 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3539 action->track_data.var_str, track_data->track_val); 3540 3541 seq_puts(m, "\ttriggered by event with key: "); 3542 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3543 seq_putc(m, '\n'); 3544 } 3545 #else 3546 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3547 { 3548 return false; 3549 } 3550 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3551 struct tracing_map_elt *elt, 3552 struct trace_buffer *buffer, void *rec, 3553 struct ring_buffer_event *rbe, void *key, 3554 struct action_data *data, 3555 u64 *var_ref_vals) {} 3556 static void track_data_snapshot_print(struct seq_file *m, 3557 struct hist_trigger_data *hist_data) {} 3558 #endif /* CONFIG_TRACER_SNAPSHOT */ 3559 3560 static void track_data_print(struct seq_file *m, 3561 struct hist_trigger_data *hist_data, 3562 struct tracing_map_elt *elt, 3563 struct action_data *data) 3564 { 3565 u64 track_val = get_track_val(hist_data, elt, data); 3566 unsigned int i, save_var_idx; 3567 3568 if (data->handler == HANDLER_ONMAX) 3569 seq_printf(m, "\n\tmax: %10llu", track_val); 3570 else if (data->handler == HANDLER_ONCHANGE) 3571 seq_printf(m, "\n\tchanged: %10llu", track_val); 3572 3573 if (data->action == ACTION_SNAPSHOT) 3574 return; 3575 3576 for (i = 0; i < hist_data->n_save_vars; i++) { 3577 struct hist_field *save_val = hist_data->save_vars[i]->val; 3578 struct hist_field *save_var = hist_data->save_vars[i]->var; 3579 u64 val; 3580 3581 save_var_idx = save_var->var.idx; 3582 3583 val = tracing_map_read_var(elt, save_var_idx); 3584 3585 if (save_val->flags & HIST_FIELD_FL_STRING) { 3586 seq_printf(m, " %s: %-32s", save_var->var.name, 3587 (char *)(uintptr_t)(val)); 3588 } else 3589 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3590 } 3591 } 3592 3593 static void ontrack_action(struct hist_trigger_data *hist_data, 3594 struct tracing_map_elt *elt, 3595 struct trace_buffer *buffer, void *rec, 3596 struct ring_buffer_event *rbe, void *key, 3597 struct action_data *data, u64 *var_ref_vals) 3598 { 3599 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3600 3601 if (check_track_val(elt, data, var_val)) { 3602 save_track_val(hist_data, elt, data, var_val); 3603 save_track_data(hist_data, elt, buffer, rec, rbe, 3604 key, data, var_ref_vals); 3605 } 3606 } 3607 3608 static void action_data_destroy(struct action_data *data) 3609 { 3610 unsigned int i; 3611 3612 lockdep_assert_held(&event_mutex); 3613 3614 kfree(data->action_name); 3615 3616 for (i = 0; i < data->n_params; i++) 3617 kfree(data->params[i]); 3618 3619 if (data->synth_event) 3620 data->synth_event->ref--; 3621 3622 kfree(data->synth_event_name); 3623 3624 kfree(data); 3625 } 3626 3627 static void track_data_destroy(struct hist_trigger_data *hist_data, 3628 struct action_data *data) 3629 { 3630 struct trace_event_file *file = hist_data->event_file; 3631 3632 destroy_hist_field(data->track_data.track_var, 0); 3633 3634 if (data->action == ACTION_SNAPSHOT) { 3635 struct track_data *track_data; 3636 3637 track_data = tracing_cond_snapshot_data(file->tr); 3638 if (track_data && track_data->hist_data == hist_data) { 3639 tracing_snapshot_cond_disable(file->tr); 3640 track_data_free(track_data); 3641 } 3642 } 3643 3644 kfree(data->track_data.var_str); 3645 3646 action_data_destroy(data); 3647 } 3648 3649 static int action_create(struct hist_trigger_data *hist_data, 3650 struct action_data *data); 3651 3652 static int track_data_create(struct hist_trigger_data *hist_data, 3653 struct action_data *data) 3654 { 3655 struct hist_field *var_field, *ref_field, *track_var = NULL; 3656 struct trace_event_file *file = hist_data->event_file; 3657 struct trace_array *tr = file->tr; 3658 char *track_data_var_str; 3659 int ret = 0; 3660 3661 track_data_var_str = data->track_data.var_str; 3662 if (track_data_var_str[0] != '$') { 3663 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3664 return -EINVAL; 3665 } 3666 track_data_var_str++; 3667 3668 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3669 if (!var_field) { 3670 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3671 return -EINVAL; 3672 } 3673 3674 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3675 if (!ref_field) 3676 return -ENOMEM; 3677 3678 data->track_data.var_ref = ref_field; 3679 3680 if (data->handler == HANDLER_ONMAX) 3681 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3682 if (IS_ERR(track_var)) { 3683 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3684 ret = PTR_ERR(track_var); 3685 goto out; 3686 } 3687 3688 if (data->handler == HANDLER_ONCHANGE) 3689 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3690 if (IS_ERR(track_var)) { 3691 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3692 ret = PTR_ERR(track_var); 3693 goto out; 3694 } 3695 data->track_data.track_var = track_var; 3696 3697 ret = action_create(hist_data, data); 3698 out: 3699 return ret; 3700 } 3701 3702 static int parse_action_params(struct trace_array *tr, char *params, 3703 struct action_data *data) 3704 { 3705 char *param, *saved_param; 3706 bool first_param = true; 3707 int ret = 0; 3708 3709 while (params) { 3710 if (data->n_params >= SYNTH_FIELDS_MAX) { 3711 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3712 ret = -EINVAL; 3713 goto out; 3714 } 3715 3716 param = strsep(¶ms, ","); 3717 if (!param) { 3718 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3719 ret = -EINVAL; 3720 goto out; 3721 } 3722 3723 param = strstrip(param); 3724 if (strlen(param) < 2) { 3725 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3726 ret = -EINVAL; 3727 goto out; 3728 } 3729 3730 saved_param = kstrdup(param, GFP_KERNEL); 3731 if (!saved_param) { 3732 ret = -ENOMEM; 3733 goto out; 3734 } 3735 3736 if (first_param && data->use_trace_keyword) { 3737 data->synth_event_name = saved_param; 3738 first_param = false; 3739 continue; 3740 } 3741 first_param = false; 3742 3743 data->params[data->n_params++] = saved_param; 3744 } 3745 out: 3746 return ret; 3747 } 3748 3749 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3750 enum handler_id handler) 3751 { 3752 char *action_name; 3753 int ret = 0; 3754 3755 strsep(&str, "."); 3756 if (!str) { 3757 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3758 ret = -EINVAL; 3759 goto out; 3760 } 3761 3762 action_name = strsep(&str, "("); 3763 if (!action_name || !str) { 3764 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3765 ret = -EINVAL; 3766 goto out; 3767 } 3768 3769 if (str_has_prefix(action_name, "save")) { 3770 char *params = strsep(&str, ")"); 3771 3772 if (!params) { 3773 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3774 ret = -EINVAL; 3775 goto out; 3776 } 3777 3778 ret = parse_action_params(tr, params, data); 3779 if (ret) 3780 goto out; 3781 3782 if (handler == HANDLER_ONMAX) 3783 data->track_data.check_val = check_track_val_max; 3784 else if (handler == HANDLER_ONCHANGE) 3785 data->track_data.check_val = check_track_val_changed; 3786 else { 3787 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3788 ret = -EINVAL; 3789 goto out; 3790 } 3791 3792 data->track_data.save_data = save_track_data_vars; 3793 data->fn = ontrack_action; 3794 data->action = ACTION_SAVE; 3795 } else if (str_has_prefix(action_name, "snapshot")) { 3796 char *params = strsep(&str, ")"); 3797 3798 if (!str) { 3799 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3800 ret = -EINVAL; 3801 goto out; 3802 } 3803 3804 if (handler == HANDLER_ONMAX) 3805 data->track_data.check_val = check_track_val_max; 3806 else if (handler == HANDLER_ONCHANGE) 3807 data->track_data.check_val = check_track_val_changed; 3808 else { 3809 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3810 ret = -EINVAL; 3811 goto out; 3812 } 3813 3814 data->track_data.save_data = save_track_data_snapshot; 3815 data->fn = ontrack_action; 3816 data->action = ACTION_SNAPSHOT; 3817 } else { 3818 char *params = strsep(&str, ")"); 3819 3820 if (str_has_prefix(action_name, "trace")) 3821 data->use_trace_keyword = true; 3822 3823 if (params) { 3824 ret = parse_action_params(tr, params, data); 3825 if (ret) 3826 goto out; 3827 } 3828 3829 if (handler == HANDLER_ONMAX) 3830 data->track_data.check_val = check_track_val_max; 3831 else if (handler == HANDLER_ONCHANGE) 3832 data->track_data.check_val = check_track_val_changed; 3833 3834 if (handler != HANDLER_ONMATCH) { 3835 data->track_data.save_data = action_trace; 3836 data->fn = ontrack_action; 3837 } else 3838 data->fn = action_trace; 3839 3840 data->action = ACTION_TRACE; 3841 } 3842 3843 data->action_name = kstrdup(action_name, GFP_KERNEL); 3844 if (!data->action_name) { 3845 ret = -ENOMEM; 3846 goto out; 3847 } 3848 3849 data->handler = handler; 3850 out: 3851 return ret; 3852 } 3853 3854 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3855 char *str, enum handler_id handler) 3856 { 3857 struct action_data *data; 3858 int ret = -EINVAL; 3859 char *var_str; 3860 3861 data = kzalloc_obj(*data); 3862 if (!data) 3863 return ERR_PTR(-ENOMEM); 3864 3865 var_str = strsep(&str, ")"); 3866 if (!var_str || !str) { 3867 ret = -EINVAL; 3868 goto free; 3869 } 3870 3871 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3872 if (!data->track_data.var_str) { 3873 ret = -ENOMEM; 3874 goto free; 3875 } 3876 3877 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3878 if (ret) 3879 goto free; 3880 out: 3881 return data; 3882 free: 3883 track_data_destroy(hist_data, data); 3884 data = ERR_PTR(ret); 3885 goto out; 3886 } 3887 3888 static void onmatch_destroy(struct action_data *data) 3889 { 3890 kfree(data->match_data.event); 3891 kfree(data->match_data.event_system); 3892 3893 action_data_destroy(data); 3894 } 3895 3896 static void destroy_field_var(struct field_var *field_var) 3897 { 3898 if (!field_var) 3899 return; 3900 3901 destroy_hist_field(field_var->var, 0); 3902 destroy_hist_field(field_var->val, 0); 3903 3904 kfree(field_var); 3905 } 3906 3907 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3908 { 3909 unsigned int i; 3910 3911 for (i = 0; i < hist_data->n_field_vars; i++) 3912 destroy_field_var(hist_data->field_vars[i]); 3913 3914 for (i = 0; i < hist_data->n_save_vars; i++) 3915 destroy_field_var(hist_data->save_vars[i]); 3916 } 3917 3918 static void save_field_var(struct hist_trigger_data *hist_data, 3919 struct field_var *field_var) 3920 { 3921 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3922 3923 /* Stack traces are saved in the string storage too */ 3924 if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 3925 hist_data->n_field_var_str++; 3926 } 3927 3928 3929 static int check_synth_field(struct synth_event *event, 3930 struct hist_field *hist_field, 3931 unsigned int field_pos) 3932 { 3933 struct synth_field *field; 3934 3935 if (field_pos >= event->n_fields) 3936 return -EINVAL; 3937 3938 field = event->fields[field_pos]; 3939 3940 /* 3941 * A dynamic string synth field can accept static or 3942 * dynamic. A static string synth field can only accept a 3943 * same-sized static string, which is checked for later. 3944 */ 3945 if (strstr(hist_field->type, "char[") && field->is_string 3946 && field->is_dynamic) 3947 return 0; 3948 3949 if (strstr(hist_field->type, "long[") && field->is_stack) 3950 return 0; 3951 3952 if (strcmp(field->type, hist_field->type) != 0) { 3953 if (field->size != hist_field->size || 3954 (!field->is_string && field->is_signed != hist_field->is_signed)) 3955 return -EINVAL; 3956 } 3957 3958 return 0; 3959 } 3960 3961 static struct hist_field * 3962 trace_action_find_var(struct hist_trigger_data *hist_data, 3963 struct action_data *data, 3964 char *system, char *event, char *var) 3965 { 3966 struct trace_array *tr = hist_data->event_file->tr; 3967 struct hist_field *hist_field; 3968 3969 var++; /* skip '$' */ 3970 3971 hist_field = find_target_event_var(hist_data, system, event, var); 3972 if (!hist_field) { 3973 if (!system && data->handler == HANDLER_ONMATCH) { 3974 system = data->match_data.event_system; 3975 event = data->match_data.event; 3976 } 3977 3978 hist_field = find_event_var(hist_data, system, event, var); 3979 } 3980 3981 if (!hist_field) 3982 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3983 3984 return hist_field; 3985 } 3986 3987 static struct hist_field * 3988 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3989 struct action_data *data, char *system, 3990 char *event, char *var) 3991 { 3992 struct hist_field *hist_field = NULL; 3993 struct field_var *field_var; 3994 3995 /* 3996 * First try to create a field var on the target event (the 3997 * currently being defined). This will create a variable for 3998 * unqualified fields on the target event, or if qualified, 3999 * target fields that have qualified names matching the target. 4000 */ 4001 field_var = create_target_field_var(hist_data, system, event, var); 4002 4003 if (field_var && !IS_ERR(field_var)) { 4004 save_field_var(hist_data, field_var); 4005 hist_field = field_var->var; 4006 } else { 4007 field_var = NULL; 4008 /* 4009 * If no explicit system.event is specified, default to 4010 * looking for fields on the onmatch(system.event.xxx) 4011 * event. 4012 */ 4013 if (!system && data->handler == HANDLER_ONMATCH) { 4014 system = data->match_data.event_system; 4015 event = data->match_data.event; 4016 } 4017 4018 if (!event) 4019 goto free; 4020 /* 4021 * At this point, we're looking at a field on another 4022 * event. Because we can't modify a hist trigger on 4023 * another event to add a variable for a field, we need 4024 * to create a new trigger on that event and create the 4025 * variable at the same time. 4026 */ 4027 hist_field = create_field_var_hist(hist_data, system, event, var); 4028 if (IS_ERR(hist_field)) 4029 goto free; 4030 } 4031 out: 4032 return hist_field; 4033 free: 4034 destroy_field_var(field_var); 4035 hist_field = NULL; 4036 goto out; 4037 } 4038 4039 static int trace_action_create(struct hist_trigger_data *hist_data, 4040 struct action_data *data) 4041 { 4042 struct trace_array *tr = hist_data->event_file->tr; 4043 char *event_name, *param, *system = NULL; 4044 struct hist_field *hist_field, *var_ref; 4045 unsigned int i; 4046 unsigned int field_pos = 0; 4047 struct synth_event *event; 4048 char *synth_event_name; 4049 int var_ref_idx, ret = 0; 4050 4051 lockdep_assert_held(&event_mutex); 4052 4053 /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */ 4054 if (data->n_params > SYNTH_FIELDS_MAX) 4055 return -EINVAL; 4056 4057 if (data->use_trace_keyword) 4058 synth_event_name = data->synth_event_name; 4059 else 4060 synth_event_name = data->action_name; 4061 4062 event = find_synth_event(synth_event_name); 4063 if (!event) { 4064 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4065 return -EINVAL; 4066 } 4067 4068 event->ref++; 4069 4070 for (i = 0; i < data->n_params; i++) { 4071 char *p; 4072 4073 p = param = kstrdup(data->params[i], GFP_KERNEL); 4074 if (!param) { 4075 ret = -ENOMEM; 4076 goto err; 4077 } 4078 4079 system = strsep(¶m, "."); 4080 if (!param) { 4081 param = (char *)system; 4082 system = event_name = NULL; 4083 } else { 4084 event_name = strsep(¶m, "."); 4085 if (!param) { 4086 kfree(p); 4087 ret = -EINVAL; 4088 goto err; 4089 } 4090 } 4091 4092 if (param[0] == '$') 4093 hist_field = trace_action_find_var(hist_data, data, 4094 system, event_name, 4095 param); 4096 else 4097 hist_field = trace_action_create_field_var(hist_data, 4098 data, 4099 system, 4100 event_name, 4101 param); 4102 4103 if (!hist_field) { 4104 kfree(p); 4105 ret = -EINVAL; 4106 goto err; 4107 } 4108 4109 if (check_synth_field(event, hist_field, field_pos) == 0) { 4110 var_ref = create_var_ref(hist_data, hist_field, 4111 system, event_name); 4112 if (!var_ref) { 4113 kfree(p); 4114 ret = -ENOMEM; 4115 goto err; 4116 } 4117 4118 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 4119 if (WARN_ON(var_ref_idx < 0)) { 4120 kfree(p); 4121 ret = var_ref_idx; 4122 goto err; 4123 } 4124 4125 data->var_ref_idx[i] = var_ref_idx; 4126 4127 field_pos++; 4128 kfree(p); 4129 continue; 4130 } 4131 4132 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4133 kfree(p); 4134 ret = -EINVAL; 4135 goto err; 4136 } 4137 4138 if (field_pos != event->n_fields) { 4139 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4140 ret = -EINVAL; 4141 goto err; 4142 } 4143 4144 data->synth_event = event; 4145 out: 4146 return ret; 4147 err: 4148 event->ref--; 4149 4150 goto out; 4151 } 4152 4153 static int action_create(struct hist_trigger_data *hist_data, 4154 struct action_data *data) 4155 { 4156 struct trace_event_file *file = hist_data->event_file; 4157 struct trace_array *tr = file->tr; 4158 struct track_data *track_data; 4159 struct field_var *field_var; 4160 unsigned int i; 4161 char *param; 4162 int ret = 0; 4163 4164 if (data->action == ACTION_TRACE) 4165 return trace_action_create(hist_data, data); 4166 4167 if (data->action == ACTION_SNAPSHOT) { 4168 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4169 if (IS_ERR(track_data)) { 4170 ret = PTR_ERR(track_data); 4171 goto out; 4172 } 4173 4174 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4175 cond_snapshot_update); 4176 if (ret) 4177 track_data_free(track_data); 4178 4179 goto out; 4180 } 4181 4182 if (data->action == ACTION_SAVE) { 4183 if (hist_data->n_save_vars) { 4184 ret = -EEXIST; 4185 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4186 goto out; 4187 } 4188 4189 for (i = 0; i < data->n_params; i++) { 4190 param = kstrdup(data->params[i], GFP_KERNEL); 4191 if (!param) { 4192 ret = -ENOMEM; 4193 goto out; 4194 } 4195 4196 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4197 if (IS_ERR(field_var)) { 4198 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4199 errpos(param)); 4200 ret = PTR_ERR(field_var); 4201 kfree(param); 4202 goto out; 4203 } 4204 4205 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4206 if (field_var->val->flags & 4207 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4208 hist_data->n_save_var_str++; 4209 kfree(param); 4210 } 4211 } 4212 out: 4213 return ret; 4214 } 4215 4216 static int onmatch_create(struct hist_trigger_data *hist_data, 4217 struct action_data *data) 4218 { 4219 return action_create(hist_data, data); 4220 } 4221 4222 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4223 { 4224 char *match_event, *match_event_system; 4225 struct action_data *data; 4226 int ret = -EINVAL; 4227 4228 data = kzalloc_obj(*data); 4229 if (!data) 4230 return ERR_PTR(-ENOMEM); 4231 4232 match_event = strsep(&str, ")"); 4233 if (!match_event || !str) { 4234 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4235 goto free; 4236 } 4237 4238 match_event_system = strsep(&match_event, "."); 4239 if (!match_event) { 4240 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4241 goto free; 4242 } 4243 4244 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4245 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4246 goto free; 4247 } 4248 4249 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4250 if (!data->match_data.event) { 4251 ret = -ENOMEM; 4252 goto free; 4253 } 4254 4255 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4256 if (!data->match_data.event_system) { 4257 ret = -ENOMEM; 4258 goto free; 4259 } 4260 4261 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4262 if (ret) 4263 goto free; 4264 out: 4265 return data; 4266 free: 4267 onmatch_destroy(data); 4268 data = ERR_PTR(ret); 4269 goto out; 4270 } 4271 4272 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4273 { 4274 hist_data->fields[HITCOUNT_IDX] = 4275 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4276 if (!hist_data->fields[HITCOUNT_IDX]) 4277 return -ENOMEM; 4278 4279 hist_data->n_vals++; 4280 hist_data->n_fields++; 4281 4282 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4283 return -EINVAL; 4284 4285 return 0; 4286 } 4287 4288 static int __create_val_field(struct hist_trigger_data *hist_data, 4289 unsigned int val_idx, 4290 struct trace_event_file *file, 4291 char *var_name, char *field_str, 4292 unsigned long flags) 4293 { 4294 struct hist_field *hist_field; 4295 int ret = 0, n_subexprs = 0; 4296 4297 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs); 4298 if (IS_ERR(hist_field)) { 4299 ret = PTR_ERR(hist_field); 4300 goto out; 4301 } 4302 4303 /* values and variables should not have some modifiers */ 4304 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4305 /* Variable */ 4306 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4307 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2)) 4308 goto err; 4309 } else { 4310 /* Value */ 4311 if (hist_field->flags & (HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | 4312 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | 4313 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) 4314 goto err; 4315 } 4316 4317 hist_data->fields[val_idx] = hist_field; 4318 4319 ++hist_data->n_vals; 4320 ++hist_data->n_fields; 4321 4322 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4323 ret = -EINVAL; 4324 out: 4325 return ret; 4326 err: 4327 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 4328 destroy_hist_field(hist_field, 0); 4329 return -EINVAL; 4330 } 4331 4332 static int create_val_field(struct hist_trigger_data *hist_data, 4333 unsigned int val_idx, 4334 struct trace_event_file *file, 4335 char *field_str) 4336 { 4337 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4338 return -EINVAL; 4339 4340 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4341 } 4342 4343 static const char no_comm[] = "(no comm)"; 4344 4345 static u64 hist_field_execname(struct hist_field *hist_field, 4346 struct tracing_map_elt *elt, 4347 struct trace_buffer *buffer, 4348 struct ring_buffer_event *rbe, 4349 void *event) 4350 { 4351 struct hist_elt_data *elt_data; 4352 4353 if (WARN_ON_ONCE(!elt)) 4354 return (u64)(unsigned long)no_comm; 4355 4356 elt_data = elt->private_data; 4357 4358 if (WARN_ON_ONCE(!elt_data->comm)) 4359 return (u64)(unsigned long)no_comm; 4360 4361 return (u64)(unsigned long)(elt_data->comm); 4362 } 4363 4364 static u64 hist_field_stack(struct hist_field *hist_field, 4365 struct tracing_map_elt *elt, 4366 struct trace_buffer *buffer, 4367 struct ring_buffer_event *rbe, 4368 void *event) 4369 { 4370 u32 str_item = *(u32 *)(event + hist_field->field->offset); 4371 int str_loc = str_item & 0xffff; 4372 char *addr = (char *)(event + str_loc); 4373 4374 return (u64)(unsigned long)addr; 4375 } 4376 4377 static u64 hist_fn_call(struct hist_field *hist_field, 4378 struct tracing_map_elt *elt, 4379 struct trace_buffer *buffer, 4380 struct ring_buffer_event *rbe, 4381 void *event) 4382 { 4383 switch (hist_field->fn_num) { 4384 case HIST_FIELD_FN_VAR_REF: 4385 return hist_field_var_ref(hist_field, elt, buffer, rbe, event); 4386 case HIST_FIELD_FN_COUNTER: 4387 return hist_field_counter(hist_field, elt, buffer, rbe, event); 4388 case HIST_FIELD_FN_CONST: 4389 return hist_field_const(hist_field, elt, buffer, rbe, event); 4390 case HIST_FIELD_FN_LOG2: 4391 return hist_field_log2(hist_field, elt, buffer, rbe, event); 4392 case HIST_FIELD_FN_BUCKET: 4393 return hist_field_bucket(hist_field, elt, buffer, rbe, event); 4394 case HIST_FIELD_FN_TIMESTAMP: 4395 return hist_field_timestamp(hist_field, elt, buffer, rbe, event); 4396 case HIST_FIELD_FN_CPU: 4397 return hist_field_cpu(hist_field, elt, buffer, rbe, event); 4398 case HIST_FIELD_FN_COMM: 4399 return hist_field_comm(hist_field, elt, buffer, rbe, event); 4400 case HIST_FIELD_FN_STRING: 4401 return hist_field_string(hist_field, elt, buffer, rbe, event); 4402 case HIST_FIELD_FN_DYNSTRING: 4403 return hist_field_dynstring(hist_field, elt, buffer, rbe, event); 4404 case HIST_FIELD_FN_RELDYNSTRING: 4405 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event); 4406 case HIST_FIELD_FN_PSTRING: 4407 return hist_field_pstring(hist_field, elt, buffer, rbe, event); 4408 case HIST_FIELD_FN_S64: 4409 return hist_field_s64(hist_field, elt, buffer, rbe, event); 4410 case HIST_FIELD_FN_U64: 4411 return hist_field_u64(hist_field, elt, buffer, rbe, event); 4412 case HIST_FIELD_FN_S32: 4413 return hist_field_s32(hist_field, elt, buffer, rbe, event); 4414 case HIST_FIELD_FN_U32: 4415 return hist_field_u32(hist_field, elt, buffer, rbe, event); 4416 case HIST_FIELD_FN_S16: 4417 return hist_field_s16(hist_field, elt, buffer, rbe, event); 4418 case HIST_FIELD_FN_U16: 4419 return hist_field_u16(hist_field, elt, buffer, rbe, event); 4420 case HIST_FIELD_FN_S8: 4421 return hist_field_s8(hist_field, elt, buffer, rbe, event); 4422 case HIST_FIELD_FN_U8: 4423 return hist_field_u8(hist_field, elt, buffer, rbe, event); 4424 case HIST_FIELD_FN_UMINUS: 4425 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event); 4426 case HIST_FIELD_FN_MINUS: 4427 return hist_field_minus(hist_field, elt, buffer, rbe, event); 4428 case HIST_FIELD_FN_PLUS: 4429 return hist_field_plus(hist_field, elt, buffer, rbe, event); 4430 case HIST_FIELD_FN_DIV: 4431 return hist_field_div(hist_field, elt, buffer, rbe, event); 4432 case HIST_FIELD_FN_MULT: 4433 return hist_field_mult(hist_field, elt, buffer, rbe, event); 4434 case HIST_FIELD_FN_DIV_POWER2: 4435 return div_by_power_of_two(hist_field, elt, buffer, rbe, event); 4436 case HIST_FIELD_FN_DIV_NOT_POWER2: 4437 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event); 4438 case HIST_FIELD_FN_DIV_MULT_SHIFT: 4439 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event); 4440 case HIST_FIELD_FN_EXECNAME: 4441 return hist_field_execname(hist_field, elt, buffer, rbe, event); 4442 case HIST_FIELD_FN_STACK: 4443 return hist_field_stack(hist_field, elt, buffer, rbe, event); 4444 default: 4445 return 0; 4446 } 4447 } 4448 4449 /* Convert a var that points to common_pid.execname to a string */ 4450 static void update_var_execname(struct hist_field *hist_field) 4451 { 4452 hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR | 4453 HIST_FIELD_FL_EXECNAME; 4454 hist_field->size = MAX_FILTER_STR_VAL; 4455 hist_field->is_signed = 0; 4456 4457 kfree_const(hist_field->type); 4458 hist_field->type = "char[]"; 4459 4460 hist_field->fn_num = HIST_FIELD_FN_EXECNAME; 4461 } 4462 4463 static int create_var_field(struct hist_trigger_data *hist_data, 4464 unsigned int val_idx, 4465 struct trace_event_file *file, 4466 char *var_name, char *expr_str) 4467 { 4468 struct trace_array *tr = hist_data->event_file->tr; 4469 unsigned long flags = 0; 4470 int ret; 4471 4472 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4473 return -EINVAL; 4474 4475 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4476 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4477 return -EINVAL; 4478 } 4479 4480 flags |= HIST_FIELD_FL_VAR; 4481 hist_data->n_vars++; 4482 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4483 return -EINVAL; 4484 4485 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4486 4487 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME) 4488 update_var_execname(hist_data->fields[val_idx]); 4489 4490 if (!ret && hist_data->fields[val_idx]->flags & 4491 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4492 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; 4493 4494 return ret; 4495 } 4496 4497 static int create_val_fields(struct hist_trigger_data *hist_data, 4498 struct trace_event_file *file) 4499 { 4500 unsigned int i, j = 1, n_hitcount = 0; 4501 char *fields_str, *field_str; 4502 int ret; 4503 4504 ret = create_hitcount_val(hist_data); 4505 if (ret) 4506 goto out; 4507 4508 fields_str = hist_data->attrs->vals_str; 4509 if (!fields_str) 4510 goto out; 4511 4512 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4513 j < TRACING_MAP_VALS_MAX; i++) { 4514 field_str = strsep(&fields_str, ","); 4515 if (!field_str) 4516 break; 4517 4518 if (strcmp(field_str, "hitcount") == 0) { 4519 if (!n_hitcount++) 4520 continue; 4521 } 4522 4523 ret = create_val_field(hist_data, j++, file, field_str); 4524 if (ret) 4525 goto out; 4526 } 4527 4528 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4529 ret = -EINVAL; 4530 out: 4531 /* There is only raw hitcount but nohitcount suppresses it. */ 4532 if (j == 1 && hist_data->attrs->no_hitcount) { 4533 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0); 4534 ret = -ENOENT; 4535 } 4536 4537 return ret; 4538 } 4539 4540 static int create_key_field(struct hist_trigger_data *hist_data, 4541 unsigned int key_idx, 4542 unsigned int key_offset, 4543 struct trace_event_file *file, 4544 char *field_str) 4545 { 4546 struct trace_array *tr = hist_data->event_file->tr; 4547 struct hist_field *hist_field = NULL; 4548 unsigned long flags = 0; 4549 unsigned int key_size; 4550 int ret = 0, n_subexprs = 0; 4551 4552 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4553 return -EINVAL; 4554 4555 flags |= HIST_FIELD_FL_KEY; 4556 4557 if (strcmp(field_str, "stacktrace") == 0) { 4558 flags |= HIST_FIELD_FL_STACKTRACE; 4559 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4560 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4561 } else { 4562 hist_field = parse_expr(hist_data, file, field_str, flags, 4563 NULL, &n_subexprs); 4564 if (IS_ERR(hist_field)) { 4565 ret = PTR_ERR(hist_field); 4566 goto out; 4567 } 4568 4569 if (field_has_hist_vars(hist_field, 0)) { 4570 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4571 destroy_hist_field(hist_field, 0); 4572 ret = -EINVAL; 4573 goto out; 4574 } 4575 4576 key_size = hist_field->size; 4577 } 4578 4579 hist_data->fields[key_idx] = hist_field; 4580 4581 key_size = ALIGN(key_size, sizeof(u64)); 4582 hist_data->fields[key_idx]->size = key_size; 4583 hist_data->fields[key_idx]->offset = key_offset; 4584 4585 hist_data->key_size += key_size; 4586 4587 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4588 ret = -EINVAL; 4589 goto out; 4590 } 4591 4592 hist_data->n_keys++; 4593 hist_data->n_fields++; 4594 4595 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4596 return -EINVAL; 4597 4598 ret = key_size; 4599 out: 4600 return ret; 4601 } 4602 4603 static int create_key_fields(struct hist_trigger_data *hist_data, 4604 struct trace_event_file *file) 4605 { 4606 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4607 char *fields_str, *field_str; 4608 int ret = -EINVAL; 4609 4610 fields_str = hist_data->attrs->keys_str; 4611 if (!fields_str) 4612 goto out; 4613 4614 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4615 field_str = strsep(&fields_str, ","); 4616 if (!field_str) 4617 break; 4618 ret = create_key_field(hist_data, i, key_offset, 4619 file, field_str); 4620 if (ret < 0) 4621 goto out; 4622 key_offset += ret; 4623 } 4624 if (fields_str) { 4625 ret = -EINVAL; 4626 goto out; 4627 } 4628 ret = 0; 4629 out: 4630 return ret; 4631 } 4632 4633 static int create_var_fields(struct hist_trigger_data *hist_data, 4634 struct trace_event_file *file) 4635 { 4636 unsigned int i, j = hist_data->n_vals; 4637 int ret = 0; 4638 4639 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4640 4641 for (i = 0; i < n_vars; i++) { 4642 char *var_name = hist_data->attrs->var_defs.name[i]; 4643 char *expr = hist_data->attrs->var_defs.expr[i]; 4644 4645 ret = create_var_field(hist_data, j++, file, var_name, expr); 4646 if (ret) 4647 goto out; 4648 } 4649 out: 4650 return ret; 4651 } 4652 4653 static void free_var_defs(struct hist_trigger_data *hist_data) 4654 { 4655 unsigned int i; 4656 4657 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4658 kfree(hist_data->attrs->var_defs.name[i]); 4659 kfree(hist_data->attrs->var_defs.expr[i]); 4660 } 4661 4662 hist_data->attrs->var_defs.n_vars = 0; 4663 } 4664 4665 static int parse_var_defs(struct hist_trigger_data *hist_data) 4666 { 4667 struct trace_array *tr = hist_data->event_file->tr; 4668 char *s, *str, *var_name, *field_str; 4669 unsigned int i, j, n_vars = 0; 4670 int ret = 0; 4671 4672 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4673 str = hist_data->attrs->assignment_str[i]; 4674 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4675 field_str = strsep(&str, ","); 4676 if (!field_str) 4677 break; 4678 4679 var_name = strsep(&field_str, "="); 4680 if (!var_name || !field_str) { 4681 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4682 errpos(var_name)); 4683 ret = -EINVAL; 4684 goto free; 4685 } 4686 4687 if (n_vars == TRACING_MAP_VARS_MAX) { 4688 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4689 ret = -EINVAL; 4690 goto free; 4691 } 4692 4693 s = kstrdup(var_name, GFP_KERNEL); 4694 if (!s) { 4695 ret = -ENOMEM; 4696 goto free; 4697 } 4698 hist_data->attrs->var_defs.name[n_vars] = s; 4699 4700 s = kstrdup(field_str, GFP_KERNEL); 4701 if (!s) { 4702 kfree(hist_data->attrs->var_defs.name[n_vars]); 4703 hist_data->attrs->var_defs.name[n_vars] = NULL; 4704 ret = -ENOMEM; 4705 goto free; 4706 } 4707 hist_data->attrs->var_defs.expr[n_vars++] = s; 4708 4709 hist_data->attrs->var_defs.n_vars = n_vars; 4710 } 4711 } 4712 4713 return ret; 4714 free: 4715 free_var_defs(hist_data); 4716 4717 return ret; 4718 } 4719 4720 static int create_hist_fields(struct hist_trigger_data *hist_data, 4721 struct trace_event_file *file) 4722 { 4723 int ret; 4724 4725 ret = parse_var_defs(hist_data); 4726 if (ret) 4727 return ret; 4728 4729 ret = create_val_fields(hist_data, file); 4730 if (ret) 4731 goto out; 4732 4733 ret = create_var_fields(hist_data, file); 4734 if (ret) 4735 goto out; 4736 4737 ret = create_key_fields(hist_data, file); 4738 4739 out: 4740 free_var_defs(hist_data); 4741 4742 return ret; 4743 } 4744 4745 static int is_descending(struct trace_array *tr, const char *str) 4746 { 4747 if (!str) 4748 return 0; 4749 4750 if (strcmp(str, "descending") == 0) 4751 return 1; 4752 4753 if (strcmp(str, "ascending") == 0) 4754 return 0; 4755 4756 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 4757 4758 return -EINVAL; 4759 } 4760 4761 static int create_sort_keys(struct hist_trigger_data *hist_data) 4762 { 4763 struct trace_array *tr = hist_data->event_file->tr; 4764 char *fields_str = hist_data->attrs->sort_key_str; 4765 struct tracing_map_sort_key *sort_key; 4766 int descending, ret = 0; 4767 unsigned int i, j, k; 4768 4769 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4770 4771 if (!fields_str) 4772 goto out; 4773 4774 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4775 struct hist_field *hist_field; 4776 char *field_str, *field_name; 4777 const char *test_name; 4778 4779 sort_key = &hist_data->sort_keys[i]; 4780 4781 field_str = strsep(&fields_str, ","); 4782 if (!field_str) 4783 break; 4784 4785 if (!*field_str) { 4786 ret = -EINVAL; 4787 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4788 break; 4789 } 4790 4791 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4792 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 4793 ret = -EINVAL; 4794 break; 4795 } 4796 4797 field_name = strsep(&field_str, "."); 4798 if (!field_name || !*field_name) { 4799 ret = -EINVAL; 4800 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4801 break; 4802 } 4803 4804 if (strcmp(field_name, "hitcount") == 0) { 4805 descending = is_descending(tr, field_str); 4806 if (descending < 0) { 4807 ret = descending; 4808 break; 4809 } 4810 sort_key->descending = descending; 4811 continue; 4812 } 4813 4814 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4815 unsigned int idx; 4816 4817 hist_field = hist_data->fields[j]; 4818 if (hist_field->flags & HIST_FIELD_FL_VAR) 4819 continue; 4820 4821 idx = k++; 4822 4823 test_name = hist_field_name(hist_field, 0); 4824 4825 if (strcmp(field_name, test_name) == 0) { 4826 sort_key->field_idx = idx; 4827 descending = is_descending(tr, field_str); 4828 if (descending < 0) { 4829 ret = descending; 4830 goto out; 4831 } 4832 sort_key->descending = descending; 4833 break; 4834 } 4835 } 4836 if (j == hist_data->n_fields) { 4837 ret = -EINVAL; 4838 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4839 break; 4840 } 4841 } 4842 4843 hist_data->n_sort_keys = i; 4844 out: 4845 return ret; 4846 } 4847 4848 static void destroy_actions(struct hist_trigger_data *hist_data) 4849 { 4850 unsigned int i; 4851 4852 for (i = 0; i < hist_data->n_actions; i++) { 4853 struct action_data *data = hist_data->actions[i]; 4854 4855 if (data->handler == HANDLER_ONMATCH) 4856 onmatch_destroy(data); 4857 else if (data->handler == HANDLER_ONMAX || 4858 data->handler == HANDLER_ONCHANGE) 4859 track_data_destroy(hist_data, data); 4860 else 4861 kfree(data); 4862 } 4863 } 4864 4865 static int parse_actions(struct hist_trigger_data *hist_data) 4866 { 4867 struct trace_array *tr = hist_data->event_file->tr; 4868 struct action_data *data; 4869 unsigned int i; 4870 int ret = 0; 4871 char *str; 4872 int len; 4873 4874 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4875 enum handler_id hid = 0; 4876 char *action_str; 4877 4878 str = hist_data->attrs->action_str[i]; 4879 4880 if ((len = str_has_prefix(str, "onmatch("))) 4881 hid = HANDLER_ONMATCH; 4882 else if ((len = str_has_prefix(str, "onmax("))) 4883 hid = HANDLER_ONMAX; 4884 else if ((len = str_has_prefix(str, "onchange("))) 4885 hid = HANDLER_ONCHANGE; 4886 4887 action_str = str + len; 4888 4889 switch (hid) { 4890 case HANDLER_ONMATCH: 4891 data = onmatch_parse(tr, action_str); 4892 break; 4893 case HANDLER_ONMAX: 4894 case HANDLER_ONCHANGE: 4895 data = track_data_parse(hist_data, action_str, hid); 4896 break; 4897 default: 4898 data = ERR_PTR(-EINVAL); 4899 break; 4900 } 4901 4902 if (IS_ERR(data)) { 4903 ret = PTR_ERR(data); 4904 break; 4905 } 4906 4907 hist_data->actions[hist_data->n_actions++] = data; 4908 } 4909 4910 return ret; 4911 } 4912 4913 static int create_actions(struct hist_trigger_data *hist_data) 4914 { 4915 struct action_data *data; 4916 unsigned int i; 4917 int ret = 0; 4918 4919 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4920 data = hist_data->actions[i]; 4921 4922 if (data->handler == HANDLER_ONMATCH) { 4923 ret = onmatch_create(hist_data, data); 4924 if (ret) 4925 break; 4926 } else if (data->handler == HANDLER_ONMAX || 4927 data->handler == HANDLER_ONCHANGE) { 4928 ret = track_data_create(hist_data, data); 4929 if (ret) 4930 break; 4931 } else { 4932 ret = -EINVAL; 4933 break; 4934 } 4935 } 4936 4937 return ret; 4938 } 4939 4940 static void print_actions(struct seq_file *m, 4941 struct hist_trigger_data *hist_data, 4942 struct tracing_map_elt *elt) 4943 { 4944 unsigned int i; 4945 4946 for (i = 0; i < hist_data->n_actions; i++) { 4947 struct action_data *data = hist_data->actions[i]; 4948 4949 if (data->action == ACTION_SNAPSHOT) 4950 continue; 4951 4952 if (data->handler == HANDLER_ONMAX || 4953 data->handler == HANDLER_ONCHANGE) 4954 track_data_print(m, hist_data, elt, data); 4955 } 4956 } 4957 4958 static void print_action_spec(struct seq_file *m, 4959 struct hist_trigger_data *hist_data, 4960 struct action_data *data) 4961 { 4962 unsigned int i; 4963 4964 if (data->action == ACTION_SAVE) { 4965 for (i = 0; i < hist_data->n_save_vars; i++) { 4966 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4967 if (i < hist_data->n_save_vars - 1) 4968 seq_puts(m, ","); 4969 } 4970 } else if (data->action == ACTION_TRACE) { 4971 if (data->use_trace_keyword) 4972 seq_printf(m, "%s", data->synth_event_name); 4973 for (i = 0; i < data->n_params; i++) { 4974 if (i || data->use_trace_keyword) 4975 seq_puts(m, ","); 4976 seq_printf(m, "%s", data->params[i]); 4977 } 4978 } 4979 } 4980 4981 static void print_track_data_spec(struct seq_file *m, 4982 struct hist_trigger_data *hist_data, 4983 struct action_data *data) 4984 { 4985 if (data->handler == HANDLER_ONMAX) 4986 seq_puts(m, ":onmax("); 4987 else if (data->handler == HANDLER_ONCHANGE) 4988 seq_puts(m, ":onchange("); 4989 seq_printf(m, "%s", data->track_data.var_str); 4990 seq_printf(m, ").%s(", data->action_name); 4991 4992 print_action_spec(m, hist_data, data); 4993 4994 seq_puts(m, ")"); 4995 } 4996 4997 static void print_onmatch_spec(struct seq_file *m, 4998 struct hist_trigger_data *hist_data, 4999 struct action_data *data) 5000 { 5001 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 5002 data->match_data.event); 5003 5004 seq_printf(m, "%s(", data->action_name); 5005 5006 print_action_spec(m, hist_data, data); 5007 5008 seq_puts(m, ")"); 5009 } 5010 5011 static bool actions_match(struct hist_trigger_data *hist_data, 5012 struct hist_trigger_data *hist_data_test) 5013 { 5014 unsigned int i, j; 5015 5016 if (hist_data->n_actions != hist_data_test->n_actions) 5017 return false; 5018 5019 for (i = 0; i < hist_data->n_actions; i++) { 5020 struct action_data *data = hist_data->actions[i]; 5021 struct action_data *data_test = hist_data_test->actions[i]; 5022 char *action_name, *action_name_test; 5023 5024 if (data->handler != data_test->handler) 5025 return false; 5026 if (data->action != data_test->action) 5027 return false; 5028 5029 if (data->n_params != data_test->n_params) 5030 return false; 5031 5032 for (j = 0; j < data->n_params; j++) { 5033 if (strcmp(data->params[j], data_test->params[j]) != 0) 5034 return false; 5035 } 5036 5037 if (data->use_trace_keyword) 5038 action_name = data->synth_event_name; 5039 else 5040 action_name = data->action_name; 5041 5042 if (data_test->use_trace_keyword) 5043 action_name_test = data_test->synth_event_name; 5044 else 5045 action_name_test = data_test->action_name; 5046 5047 if (strcmp(action_name, action_name_test) != 0) 5048 return false; 5049 5050 if (data->handler == HANDLER_ONMATCH) { 5051 if (strcmp(data->match_data.event_system, 5052 data_test->match_data.event_system) != 0) 5053 return false; 5054 if (strcmp(data->match_data.event, 5055 data_test->match_data.event) != 0) 5056 return false; 5057 } else if (data->handler == HANDLER_ONMAX || 5058 data->handler == HANDLER_ONCHANGE) { 5059 if (strcmp(data->track_data.var_str, 5060 data_test->track_data.var_str) != 0) 5061 return false; 5062 } 5063 } 5064 5065 return true; 5066 } 5067 5068 5069 static void print_actions_spec(struct seq_file *m, 5070 struct hist_trigger_data *hist_data) 5071 { 5072 unsigned int i; 5073 5074 for (i = 0; i < hist_data->n_actions; i++) { 5075 struct action_data *data = hist_data->actions[i]; 5076 5077 if (data->handler == HANDLER_ONMATCH) 5078 print_onmatch_spec(m, hist_data, data); 5079 else if (data->handler == HANDLER_ONMAX || 5080 data->handler == HANDLER_ONCHANGE) 5081 print_track_data_spec(m, hist_data, data); 5082 } 5083 } 5084 5085 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5086 { 5087 unsigned int i; 5088 5089 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5090 kfree(hist_data->field_var_hists[i]->cmd); 5091 kfree(hist_data->field_var_hists[i]); 5092 } 5093 } 5094 5095 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5096 { 5097 if (!hist_data) 5098 return; 5099 5100 destroy_hist_trigger_attrs(hist_data->attrs); 5101 destroy_hist_fields(hist_data); 5102 tracing_map_destroy(hist_data->map); 5103 5104 destroy_actions(hist_data); 5105 destroy_field_vars(hist_data); 5106 destroy_field_var_hists(hist_data); 5107 5108 kfree(hist_data); 5109 } 5110 5111 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5112 { 5113 struct tracing_map *map = hist_data->map; 5114 struct ftrace_event_field *field; 5115 struct hist_field *hist_field; 5116 int i, idx = 0; 5117 5118 for_each_hist_field(i, hist_data) { 5119 hist_field = hist_data->fields[i]; 5120 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5121 tracing_map_cmp_fn_t cmp_fn; 5122 5123 field = hist_field->field; 5124 5125 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5126 cmp_fn = tracing_map_cmp_none; 5127 else if (!field || hist_field->flags & HIST_FIELD_FL_CPU) 5128 cmp_fn = tracing_map_cmp_num(hist_field->size, 5129 hist_field->is_signed); 5130 else if (is_string_field(field)) 5131 cmp_fn = tracing_map_cmp_string; 5132 else 5133 cmp_fn = tracing_map_cmp_num(field->size, 5134 field->is_signed); 5135 idx = tracing_map_add_key_field(map, 5136 hist_field->offset, 5137 cmp_fn); 5138 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5139 idx = tracing_map_add_sum_field(map); 5140 5141 if (idx < 0) 5142 return idx; 5143 5144 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5145 idx = tracing_map_add_var(map); 5146 if (idx < 0) 5147 return idx; 5148 hist_field->var.idx = idx; 5149 hist_field->var.hist_data = hist_data; 5150 } 5151 } 5152 5153 return 0; 5154 } 5155 5156 static struct hist_trigger_data * 5157 create_hist_data(unsigned int map_bits, 5158 struct hist_trigger_attrs *attrs, 5159 struct trace_event_file *file, 5160 bool remove) 5161 { 5162 const struct tracing_map_ops *map_ops = NULL; 5163 struct hist_trigger_data *hist_data; 5164 int ret = 0; 5165 5166 hist_data = kzalloc_obj(*hist_data); 5167 if (!hist_data) 5168 return ERR_PTR(-ENOMEM); 5169 5170 hist_data->attrs = attrs; 5171 hist_data->remove = remove; 5172 hist_data->event_file = file; 5173 5174 ret = parse_actions(hist_data); 5175 if (ret) 5176 goto free; 5177 5178 ret = create_hist_fields(hist_data, file); 5179 if (ret) 5180 goto free; 5181 5182 ret = create_sort_keys(hist_data); 5183 if (ret) 5184 goto free; 5185 5186 map_ops = &hist_trigger_elt_data_ops; 5187 5188 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5189 map_ops, hist_data); 5190 if (IS_ERR(hist_data->map)) { 5191 ret = PTR_ERR(hist_data->map); 5192 hist_data->map = NULL; 5193 goto free; 5194 } 5195 5196 ret = create_tracing_map_fields(hist_data); 5197 if (ret) 5198 goto free; 5199 out: 5200 return hist_data; 5201 free: 5202 hist_data->attrs = NULL; 5203 5204 destroy_hist_data(hist_data); 5205 5206 hist_data = ERR_PTR(ret); 5207 5208 goto out; 5209 } 5210 5211 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5212 struct tracing_map_elt *elt, 5213 struct trace_buffer *buffer, void *rec, 5214 struct ring_buffer_event *rbe, 5215 u64 *var_ref_vals) 5216 { 5217 struct hist_elt_data *elt_data; 5218 struct hist_field *hist_field; 5219 unsigned int i, var_idx; 5220 u64 hist_val; 5221 5222 elt_data = elt->private_data; 5223 elt_data->var_ref_vals = var_ref_vals; 5224 5225 for_each_hist_val_field(i, hist_data) { 5226 hist_field = hist_data->fields[i]; 5227 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5228 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5229 var_idx = hist_field->var.idx; 5230 5231 if (hist_field->flags & 5232 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) { 5233 unsigned int str_start, var_str_idx, idx; 5234 char *str, *val_str; 5235 unsigned int size; 5236 5237 str_start = hist_data->n_field_var_str + 5238 hist_data->n_save_var_str; 5239 var_str_idx = hist_field->var_str_idx; 5240 idx = str_start + var_str_idx; 5241 5242 str = elt_data->field_var_str[idx]; 5243 val_str = (char *)(uintptr_t)hist_val; 5244 5245 if (hist_field->flags & HIST_FIELD_FL_STRING) { 5246 size = min(hist_field->size, STR_VAR_LEN_MAX); 5247 strscpy(str, val_str, size); 5248 } else { 5249 char *stack_start = str + sizeof(unsigned long); 5250 int e; 5251 5252 e = stack_trace_save((void *)stack_start, 5253 HIST_STACKTRACE_DEPTH, 5254 HIST_STACKTRACE_SKIP); 5255 if (e < HIST_STACKTRACE_DEPTH - 1) 5256 ((unsigned long *)stack_start)[e] = 0; 5257 *((unsigned long *)str) = e; 5258 } 5259 hist_val = (u64)(uintptr_t)str; 5260 } 5261 tracing_map_set_var(elt, var_idx, hist_val); 5262 continue; 5263 } 5264 tracing_map_update_sum(elt, i, hist_val); 5265 } 5266 5267 for_each_hist_key_field(i, hist_data) { 5268 hist_field = hist_data->fields[i]; 5269 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5270 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5271 var_idx = hist_field->var.idx; 5272 tracing_map_set_var(elt, var_idx, hist_val); 5273 } 5274 } 5275 5276 update_field_vars(hist_data, elt, buffer, rbe, rec); 5277 } 5278 5279 static inline void add_to_key(char *compound_key, void *key, 5280 struct hist_field *key_field, void *rec) 5281 { 5282 size_t size = key_field->size; 5283 5284 if (key_field->flags & HIST_FIELD_FL_STRING) { 5285 5286 if (key_field->flags & HIST_FIELD_FL_COMM) { 5287 size = strlen((char *)key); 5288 } else { 5289 struct ftrace_event_field *field; 5290 5291 field = key_field->field; 5292 if (field->filter_type == FILTER_DYN_STRING || 5293 field->filter_type == FILTER_RDYN_STRING) 5294 size = *(u32 *)(rec + field->offset) >> 16; 5295 else if (field->filter_type == FILTER_STATIC_STRING) 5296 size = field->size; 5297 } 5298 5299 /* ensure NULL-termination */ 5300 if (size > key_field->size - 1) 5301 size = key_field->size - 1; 5302 } 5303 memcpy(compound_key + key_field->offset, key, size); 5304 } 5305 5306 static void 5307 hist_trigger_actions(struct hist_trigger_data *hist_data, 5308 struct tracing_map_elt *elt, 5309 struct trace_buffer *buffer, void *rec, 5310 struct ring_buffer_event *rbe, void *key, 5311 u64 *var_ref_vals) 5312 { 5313 struct action_data *data; 5314 unsigned int i; 5315 5316 for (i = 0; i < hist_data->n_actions; i++) { 5317 data = hist_data->actions[i]; 5318 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals); 5319 } 5320 } 5321 5322 /* 5323 * The hist_pad structure is used to save information to create 5324 * a histogram from the histogram trigger. It's too big to store 5325 * on the stack, so when the histogram trigger is initialized 5326 * a percpu array of 4 hist_pad structures is allocated. 5327 * This will cover every context from normal, softirq, irq and NMI 5328 * in the very unlikely event that a trigger happens at each of 5329 * these contexts and interrupts a currently active trigger. 5330 */ 5331 struct hist_pad { 5332 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5333 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5334 char compound_key[HIST_KEY_SIZE_MAX]; 5335 }; 5336 5337 static struct hist_pad __percpu *hist_pads; 5338 static DEFINE_PER_CPU(int, hist_pad_cnt); 5339 static refcount_t hist_pad_ref; 5340 5341 /* One hist_pad for every context (normal, softirq, irq, NMI) */ 5342 #define MAX_HIST_CNT 4 5343 5344 static int alloc_hist_pad(void) 5345 { 5346 lockdep_assert_held(&event_mutex); 5347 5348 if (refcount_read(&hist_pad_ref)) { 5349 refcount_inc(&hist_pad_ref); 5350 return 0; 5351 } 5352 5353 hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT, 5354 __alignof__(struct hist_pad)); 5355 if (!hist_pads) 5356 return -ENOMEM; 5357 5358 refcount_set(&hist_pad_ref, 1); 5359 return 0; 5360 } 5361 5362 static void free_hist_pad(void) 5363 { 5364 lockdep_assert_held(&event_mutex); 5365 5366 if (!refcount_dec_and_test(&hist_pad_ref)) 5367 return; 5368 5369 free_percpu(hist_pads); 5370 hist_pads = NULL; 5371 } 5372 5373 static struct hist_pad *get_hist_pad(void) 5374 { 5375 struct hist_pad *hist_pad; 5376 int cnt; 5377 5378 if (WARN_ON_ONCE(!hist_pads)) 5379 return NULL; 5380 5381 preempt_disable(); 5382 5383 hist_pad = per_cpu_ptr(hist_pads, smp_processor_id()); 5384 5385 if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) { 5386 preempt_enable(); 5387 return NULL; 5388 } 5389 5390 cnt = this_cpu_inc_return(hist_pad_cnt) - 1; 5391 5392 return &hist_pad[cnt]; 5393 } 5394 5395 static void put_hist_pad(void) 5396 { 5397 this_cpu_dec(hist_pad_cnt); 5398 preempt_enable(); 5399 } 5400 5401 static void event_hist_trigger(struct event_trigger_data *data, 5402 struct trace_buffer *buffer, void *rec, 5403 struct ring_buffer_event *rbe) 5404 { 5405 struct hist_trigger_data *hist_data = data->private_data; 5406 bool use_compound_key = (hist_data->n_keys > 1); 5407 struct tracing_map_elt *elt = NULL; 5408 struct hist_field *key_field; 5409 struct hist_pad *hist_pad; 5410 u64 field_contents; 5411 void *key = NULL; 5412 unsigned int i; 5413 5414 if (unlikely(!rbe)) 5415 return; 5416 5417 hist_pad = get_hist_pad(); 5418 if (!hist_pad) 5419 return; 5420 5421 memset(hist_pad->compound_key, 0, hist_data->key_size); 5422 5423 for_each_hist_key_field(i, hist_data) { 5424 key_field = hist_data->fields[i]; 5425 5426 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5427 unsigned long *entries = hist_pad->entries; 5428 5429 memset(entries, 0, HIST_STACKTRACE_SIZE); 5430 if (key_field->field) { 5431 unsigned long *stack, n_entries; 5432 5433 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5434 stack = (unsigned long *)(long)field_contents; 5435 n_entries = *stack; 5436 memcpy(entries, ++stack, n_entries * sizeof(unsigned long)); 5437 } else { 5438 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5439 HIST_STACKTRACE_SKIP); 5440 } 5441 key = entries; 5442 } else { 5443 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5444 if (key_field->flags & HIST_FIELD_FL_STRING) { 5445 key = (void *)(unsigned long)field_contents; 5446 use_compound_key = true; 5447 } else 5448 key = (void *)&field_contents; 5449 } 5450 5451 if (use_compound_key) 5452 add_to_key(hist_pad->compound_key, key, key_field, rec); 5453 } 5454 5455 if (use_compound_key) 5456 key = hist_pad->compound_key; 5457 5458 if (hist_data->n_var_refs && 5459 !resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false)) 5460 goto out; 5461 5462 elt = tracing_map_insert(hist_data->map, key); 5463 if (!elt) 5464 goto out; 5465 5466 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals); 5467 5468 if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) { 5469 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, 5470 key, hist_pad->var_ref_vals); 5471 } 5472 5473 hist_poll_wakeup(); 5474 5475 out: 5476 put_hist_pad(); 5477 } 5478 5479 static void hist_trigger_stacktrace_print(struct seq_file *m, 5480 unsigned long *stacktrace_entries, 5481 unsigned int max_entries) 5482 { 5483 unsigned int spaces = 8; 5484 unsigned int i; 5485 5486 for (i = 0; i < max_entries; i++) { 5487 if (!stacktrace_entries[i]) 5488 return; 5489 5490 seq_printf(m, "%*c", 1 + spaces, ' '); 5491 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]); 5492 } 5493 } 5494 5495 static void hist_trigger_print_key(struct seq_file *m, 5496 struct hist_trigger_data *hist_data, 5497 void *key, 5498 struct tracing_map_elt *elt) 5499 { 5500 struct hist_field *key_field; 5501 bool multiline = false; 5502 const char *field_name; 5503 unsigned int i; 5504 u64 uval; 5505 5506 seq_puts(m, "{ "); 5507 5508 for_each_hist_key_field(i, hist_data) { 5509 key_field = hist_data->fields[i]; 5510 5511 if (i > hist_data->n_vals) 5512 seq_puts(m, ", "); 5513 5514 field_name = hist_field_name(key_field, 0); 5515 5516 if (key_field->flags & HIST_FIELD_FL_HEX) { 5517 uval = *(u64 *)(key + key_field->offset); 5518 seq_printf(m, "%s: %llx", field_name, uval); 5519 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5520 uval = *(u64 *)(key + key_field->offset); 5521 seq_printf(m, "%s: [%llx] %-45ps", field_name, 5522 uval, (void *)(uintptr_t)uval); 5523 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5524 uval = *(u64 *)(key + key_field->offset); 5525 seq_printf(m, "%s: [%llx] %-55pS", field_name, 5526 uval, (void *)(uintptr_t)uval); 5527 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5528 struct hist_elt_data *elt_data = elt->private_data; 5529 char *comm; 5530 5531 if (WARN_ON_ONCE(!elt_data)) 5532 return; 5533 5534 comm = elt_data->comm; 5535 5536 uval = *(u64 *)(key + key_field->offset); 5537 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5538 comm, uval); 5539 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5540 const char *syscall_name; 5541 5542 uval = *(u64 *)(key + key_field->offset); 5543 syscall_name = get_syscall_name(uval); 5544 if (!syscall_name) 5545 syscall_name = "unknown_syscall"; 5546 5547 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5548 syscall_name, uval); 5549 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5550 if (key_field->field) 5551 seq_printf(m, "%s.stacktrace", key_field->field->name); 5552 else 5553 seq_puts(m, "common_stacktrace:\n"); 5554 hist_trigger_stacktrace_print(m, 5555 key + key_field->offset, 5556 HIST_STACKTRACE_DEPTH); 5557 multiline = true; 5558 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5559 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5560 *(u64 *)(key + key_field->offset)); 5561 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) { 5562 unsigned long buckets = key_field->buckets; 5563 uval = *(u64 *)(key + key_field->offset); 5564 seq_printf(m, "%s: ~ %llu-%llu", field_name, 5565 uval, uval + buckets -1); 5566 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5567 seq_printf(m, "%s: %-50s", field_name, 5568 (char *)(key + key_field->offset)); 5569 } else { 5570 uval = *(u64 *)(key + key_field->offset); 5571 seq_printf(m, "%s: %10llu", field_name, uval); 5572 } 5573 } 5574 5575 if (!multiline) 5576 seq_puts(m, " "); 5577 5578 seq_puts(m, "}"); 5579 } 5580 5581 /* Get the 100 times of the percentage of @val in @total */ 5582 static inline unsigned int __get_percentage(u64 val, u64 total) 5583 { 5584 if (!total) 5585 goto div0; 5586 5587 if (val < (U64_MAX / 10000)) 5588 return (unsigned int)div64_ul(val * 10000, total); 5589 5590 total = div64_u64(total, 10000); 5591 if (!total) 5592 goto div0; 5593 5594 return (unsigned int)div64_ul(val, total); 5595 div0: 5596 return val ? UINT_MAX : 0; 5597 } 5598 5599 #define BAR_CHAR '#' 5600 5601 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max) 5602 { 5603 unsigned int len = __get_percentage(val, max); 5604 int i; 5605 5606 if (len == UINT_MAX) { 5607 snprintf(buf, size, "[ERROR]"); 5608 return buf; 5609 } 5610 5611 len = len * size / 10000; 5612 for (i = 0; i < len && i < size; i++) 5613 buf[i] = BAR_CHAR; 5614 while (i < size) 5615 buf[i++] = ' '; 5616 buf[size] = '\0'; 5617 5618 return buf; 5619 } 5620 5621 struct hist_val_stat { 5622 u64 max; 5623 u64 total; 5624 }; 5625 5626 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx, 5627 const char *field_name, unsigned long flags, 5628 struct hist_val_stat *stats, 5629 struct tracing_map_elt *elt) 5630 { 5631 u64 val = tracing_map_read_sum(elt, idx); 5632 unsigned int pc; 5633 char bar[21]; 5634 5635 if (flags & HIST_FIELD_FL_PERCENT) { 5636 pc = __get_percentage(val, stats[idx].total); 5637 if (pc == UINT_MAX) 5638 seq_printf(m, " %s (%%):[ERROR]", field_name); 5639 else 5640 seq_printf(m, " %s (%%): %3u.%02u", field_name, 5641 pc / 100, pc % 100); 5642 } else if (flags & HIST_FIELD_FL_GRAPH) { 5643 seq_printf(m, " %s: %20s", field_name, 5644 __fill_bar_str(bar, 20, val, stats[idx].max)); 5645 } else if (flags & HIST_FIELD_FL_HEX) { 5646 seq_printf(m, " %s: %10llx", field_name, val); 5647 } else { 5648 seq_printf(m, " %s: %10llu", field_name, val); 5649 } 5650 } 5651 5652 static void hist_trigger_entry_print(struct seq_file *m, 5653 struct hist_trigger_data *hist_data, 5654 struct hist_val_stat *stats, 5655 void *key, 5656 struct tracing_map_elt *elt) 5657 { 5658 const char *field_name; 5659 unsigned int i = HITCOUNT_IDX; 5660 unsigned long flags; 5661 5662 hist_trigger_print_key(m, hist_data, key, elt); 5663 5664 /* At first, show the raw hitcount if !nohitcount */ 5665 if (!hist_data->attrs->no_hitcount) 5666 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt); 5667 5668 for (i = 1; i < hist_data->n_vals; i++) { 5669 field_name = hist_field_name(hist_data->fields[i], 0); 5670 flags = hist_data->fields[i]->flags; 5671 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR) 5672 continue; 5673 5674 seq_puts(m, " "); 5675 hist_trigger_print_val(m, i, field_name, flags, stats, elt); 5676 } 5677 5678 print_actions(m, hist_data, elt); 5679 5680 seq_puts(m, "\n"); 5681 } 5682 5683 static int print_entries(struct seq_file *m, 5684 struct hist_trigger_data *hist_data) 5685 { 5686 struct tracing_map_sort_entry **sort_entries = NULL; 5687 struct tracing_map *map = hist_data->map; 5688 int i, j, n_entries, ret; 5689 struct hist_val_stat *stats = NULL; 5690 u64 val; 5691 5692 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5693 hist_data->n_sort_keys, 5694 &sort_entries); 5695 if (n_entries < 0) 5696 return n_entries; 5697 5698 ret = n_entries; 5699 5700 /* Calculate the max and the total for each field if needed. */ 5701 for (j = 0; j < hist_data->n_vals; j++) { 5702 if (!(hist_data->fields[j]->flags & 5703 (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH))) 5704 continue; 5705 if (!stats) { 5706 stats = kzalloc_objs(*stats, hist_data->n_vals); 5707 if (!stats) { 5708 ret = -ENOMEM; 5709 goto out; 5710 } 5711 } 5712 for (i = 0; i < n_entries; i++) { 5713 val = tracing_map_read_sum(sort_entries[i]->elt, j); 5714 stats[j].total += val; 5715 if (stats[j].max < val) 5716 stats[j].max = val; 5717 } 5718 } 5719 5720 for (i = 0; i < n_entries; i++) 5721 hist_trigger_entry_print(m, hist_data, stats, 5722 sort_entries[i]->key, 5723 sort_entries[i]->elt); 5724 5725 kfree(stats); 5726 out: 5727 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5728 5729 return ret; 5730 } 5731 5732 static void hist_trigger_show(struct seq_file *m, 5733 struct event_trigger_data *data, int n) 5734 { 5735 struct hist_trigger_data *hist_data; 5736 int n_entries; 5737 5738 if (n > 0) 5739 seq_puts(m, "\n\n"); 5740 5741 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5742 data->cmd_ops->print(m, data); 5743 seq_puts(m, "#\n\n"); 5744 5745 hist_data = data->private_data; 5746 n_entries = print_entries(m, hist_data); 5747 if (n_entries < 0) 5748 n_entries = 0; 5749 5750 track_data_snapshot_print(m, hist_data); 5751 5752 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5753 (u64)atomic64_read(&hist_data->map->hits), 5754 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5755 } 5756 5757 struct hist_file_data { 5758 struct file *file; 5759 u64 last_read; 5760 u64 last_act; 5761 }; 5762 5763 static u64 get_hist_hit_count(struct trace_event_file *event_file) 5764 { 5765 struct hist_trigger_data *hist_data; 5766 struct event_trigger_data *data; 5767 u64 ret = 0; 5768 5769 list_for_each_entry(data, &event_file->triggers, list) { 5770 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5771 hist_data = data->private_data; 5772 ret += atomic64_read(&hist_data->map->hits); 5773 } 5774 } 5775 return ret; 5776 } 5777 5778 static int hist_show(struct seq_file *m, void *v) 5779 { 5780 struct hist_file_data *hist_file = m->private; 5781 struct event_trigger_data *data; 5782 struct trace_event_file *event_file; 5783 int n = 0; 5784 5785 guard(mutex)(&event_mutex); 5786 5787 event_file = event_file_file(hist_file->file); 5788 if (unlikely(!event_file)) 5789 return -ENODEV; 5790 5791 list_for_each_entry(data, &event_file->triggers, list) { 5792 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5793 hist_trigger_show(m, data, n++); 5794 } 5795 hist_file->last_read = get_hist_hit_count(event_file); 5796 /* 5797 * Update last_act too so that poll()/POLLPRI can wait for the next 5798 * event after any syscall on hist file. 5799 */ 5800 hist_file->last_act = hist_file->last_read; 5801 5802 return 0; 5803 } 5804 5805 static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wait) 5806 { 5807 struct trace_event_file *event_file; 5808 struct seq_file *m = file->private_data; 5809 struct hist_file_data *hist_file = m->private; 5810 __poll_t ret = 0; 5811 u64 cnt; 5812 5813 guard(mutex)(&event_mutex); 5814 5815 event_file = event_file_file(file); 5816 if (!event_file) 5817 return EPOLLERR; 5818 5819 hist_poll_wait(file, wait); 5820 5821 cnt = get_hist_hit_count(event_file); 5822 if (hist_file->last_read != cnt) 5823 ret |= EPOLLIN | EPOLLRDNORM; 5824 if (hist_file->last_act != cnt) { 5825 hist_file->last_act = cnt; 5826 ret |= EPOLLPRI; 5827 } 5828 5829 return ret; 5830 } 5831 5832 static int event_hist_release(struct inode *inode, struct file *file) 5833 { 5834 struct seq_file *m = file->private_data; 5835 struct hist_file_data *hist_file = m->private; 5836 5837 kfree(hist_file); 5838 return tracing_single_release_file_tr(inode, file); 5839 } 5840 5841 static int event_hist_open(struct inode *inode, struct file *file) 5842 { 5843 struct trace_event_file *event_file; 5844 struct hist_file_data *hist_file; 5845 int ret; 5846 5847 ret = tracing_open_file_tr(inode, file); 5848 if (ret) 5849 return ret; 5850 5851 guard(mutex)(&event_mutex); 5852 5853 event_file = event_file_file(file); 5854 if (!event_file) { 5855 ret = -ENODEV; 5856 goto err; 5857 } 5858 5859 hist_file = kzalloc_obj(*hist_file); 5860 if (!hist_file) { 5861 ret = -ENOMEM; 5862 goto err; 5863 } 5864 5865 hist_file->file = file; 5866 hist_file->last_act = get_hist_hit_count(event_file); 5867 5868 ret = single_open(file, hist_show, hist_file); 5869 if (ret) { 5870 kfree(hist_file); 5871 goto err; 5872 } 5873 5874 return 0; 5875 err: 5876 tracing_release_file_tr(inode, file); 5877 return ret; 5878 } 5879 5880 const struct file_operations event_hist_fops = { 5881 .open = event_hist_open, 5882 .read = seq_read, 5883 .llseek = seq_lseek, 5884 .release = event_hist_release, 5885 .poll = event_hist_poll, 5886 }; 5887 5888 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 5889 5890 #undef C 5891 #define C(a, b) b 5892 5893 static const char * const field_funcs[] = { FIELD_FUNCS }; 5894 5895 static void hist_field_debug_show_flags(struct seq_file *m, 5896 unsigned long flags) 5897 { 5898 seq_puts(m, " flags:\n"); 5899 5900 if (flags & HIST_FIELD_FL_KEY) 5901 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 5902 else if (flags & HIST_FIELD_FL_HITCOUNT) 5903 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 5904 else if (flags & HIST_FIELD_FL_VAR) 5905 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 5906 else if (flags & HIST_FIELD_FL_VAR_REF) 5907 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 5908 else 5909 seq_puts(m, " VAL: normal u64 value\n"); 5910 5911 if (flags & HIST_FIELD_FL_ALIAS) 5912 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 5913 else if (flags & HIST_FIELD_FL_CONST) 5914 seq_puts(m, " HIST_FIELD_FL_CONST\n"); 5915 } 5916 5917 static int hist_field_debug_show(struct seq_file *m, 5918 struct hist_field *field, unsigned long flags) 5919 { 5920 if ((field->flags & flags) != flags) { 5921 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 5922 return -EINVAL; 5923 } 5924 5925 hist_field_debug_show_flags(m, field->flags); 5926 if (field->field) 5927 seq_printf(m, " ftrace_event_field name: %s\n", 5928 field->field->name); 5929 5930 if (field->flags & HIST_FIELD_FL_VAR) { 5931 seq_printf(m, " var.name: %s\n", field->var.name); 5932 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5933 field->var.idx); 5934 } 5935 5936 if (field->flags & HIST_FIELD_FL_CONST) 5937 seq_printf(m, " constant: %llu\n", field->constant); 5938 5939 if (field->flags & HIST_FIELD_FL_ALIAS) 5940 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5941 field->var_ref_idx); 5942 5943 if (field->flags & HIST_FIELD_FL_VAR_REF) { 5944 seq_printf(m, " name: %s\n", field->name); 5945 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5946 field->var.idx); 5947 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 5948 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5949 field->var_ref_idx); 5950 if (field->system) 5951 seq_printf(m, " system: %s\n", field->system); 5952 if (field->event_name) 5953 seq_printf(m, " event_name: %s\n", field->event_name); 5954 } 5955 5956 seq_printf(m, " type: %s\n", field->type); 5957 seq_printf(m, " size: %u\n", field->size); 5958 seq_printf(m, " is_signed: %u\n", field->is_signed); 5959 seq_printf(m, " function: hist_field_%s()\n", field_funcs[field->fn_num]); 5960 5961 return 0; 5962 } 5963 5964 static int field_var_debug_show(struct seq_file *m, 5965 struct field_var *field_var, unsigned int i, 5966 bool save_vars) 5967 { 5968 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 5969 struct hist_field *field; 5970 int ret = 0; 5971 5972 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 5973 5974 field = field_var->var; 5975 5976 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 5977 5978 hist_field_debug_show_flags(m, field->flags); 5979 seq_printf(m, " var.name: %s\n", field->var.name); 5980 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5981 field->var.idx); 5982 5983 field = field_var->val; 5984 5985 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 5986 if (field->field) 5987 seq_printf(m, " ftrace_event_field name: %s\n", 5988 field->field->name); 5989 else { 5990 ret = -EINVAL; 5991 goto out; 5992 } 5993 5994 seq_printf(m, " type: %s\n", field->type); 5995 seq_printf(m, " size: %u\n", field->size); 5996 seq_printf(m, " is_signed: %u\n", field->is_signed); 5997 out: 5998 return ret; 5999 } 6000 6001 static int hist_action_debug_show(struct seq_file *m, 6002 struct action_data *data, int i) 6003 { 6004 int ret = 0; 6005 6006 if (data->handler == HANDLER_ONMAX || 6007 data->handler == HANDLER_ONCHANGE) { 6008 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 6009 ret = hist_field_debug_show(m, data->track_data.var_ref, 6010 HIST_FIELD_FL_VAR_REF); 6011 if (ret) 6012 goto out; 6013 6014 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 6015 ret = hist_field_debug_show(m, data->track_data.track_var, 6016 HIST_FIELD_FL_VAR); 6017 if (ret) 6018 goto out; 6019 } 6020 6021 if (data->handler == HANDLER_ONMATCH) { 6022 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 6023 i, data->match_data.event_system); 6024 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 6025 i, data->match_data.event); 6026 } 6027 out: 6028 return ret; 6029 } 6030 6031 static int hist_actions_debug_show(struct seq_file *m, 6032 struct hist_trigger_data *hist_data) 6033 { 6034 int i, ret = 0; 6035 6036 if (hist_data->n_actions) 6037 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 6038 6039 for (i = 0; i < hist_data->n_actions; i++) { 6040 struct action_data *action = hist_data->actions[i]; 6041 6042 ret = hist_action_debug_show(m, action, i); 6043 if (ret) 6044 goto out; 6045 } 6046 6047 if (hist_data->n_save_vars) 6048 seq_puts(m, "\n save action variables (save() params):\n"); 6049 6050 for (i = 0; i < hist_data->n_save_vars; i++) { 6051 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 6052 if (ret) 6053 goto out; 6054 } 6055 out: 6056 return ret; 6057 } 6058 6059 static void hist_trigger_debug_show(struct seq_file *m, 6060 struct event_trigger_data *data, int n) 6061 { 6062 struct hist_trigger_data *hist_data; 6063 int i, ret; 6064 6065 if (n > 0) 6066 seq_puts(m, "\n\n"); 6067 6068 seq_puts(m, "# event histogram\n#\n# trigger info: "); 6069 data->cmd_ops->print(m, data); 6070 seq_puts(m, "#\n\n"); 6071 6072 hist_data = data->private_data; 6073 6074 seq_printf(m, "hist_data: %p\n\n", hist_data); 6075 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 6076 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 6077 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 6078 6079 seq_puts(m, "\n val fields:\n\n"); 6080 6081 seq_puts(m, " hist_data->fields[0]:\n"); 6082 ret = hist_field_debug_show(m, hist_data->fields[0], 6083 HIST_FIELD_FL_HITCOUNT); 6084 if (ret) 6085 return; 6086 6087 for (i = 1; i < hist_data->n_vals; i++) { 6088 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6089 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 6090 if (ret) 6091 return; 6092 } 6093 6094 seq_puts(m, "\n key fields:\n"); 6095 6096 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 6097 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6098 ret = hist_field_debug_show(m, hist_data->fields[i], 6099 HIST_FIELD_FL_KEY); 6100 if (ret) 6101 return; 6102 } 6103 6104 if (hist_data->n_var_refs) 6105 seq_puts(m, "\n variable reference fields:\n"); 6106 6107 for (i = 0; i < hist_data->n_var_refs; i++) { 6108 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 6109 ret = hist_field_debug_show(m, hist_data->var_refs[i], 6110 HIST_FIELD_FL_VAR_REF); 6111 if (ret) 6112 return; 6113 } 6114 6115 if (hist_data->n_field_vars) 6116 seq_puts(m, "\n field variables:\n"); 6117 6118 for (i = 0; i < hist_data->n_field_vars; i++) { 6119 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 6120 if (ret) 6121 return; 6122 } 6123 6124 ret = hist_actions_debug_show(m, hist_data); 6125 if (ret) 6126 return; 6127 } 6128 6129 static int hist_debug_show(struct seq_file *m, void *v) 6130 { 6131 struct event_trigger_data *data; 6132 struct trace_event_file *event_file; 6133 int n = 0; 6134 6135 guard(mutex)(&event_mutex); 6136 6137 event_file = event_file_file(m->private); 6138 if (unlikely(!event_file)) 6139 return -ENODEV; 6140 6141 list_for_each_entry(data, &event_file->triggers, list) { 6142 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 6143 hist_trigger_debug_show(m, data, n++); 6144 } 6145 return 0; 6146 } 6147 6148 static int event_hist_debug_open(struct inode *inode, struct file *file) 6149 { 6150 int ret; 6151 6152 ret = tracing_open_file_tr(inode, file); 6153 if (ret) 6154 return ret; 6155 6156 ret = single_open(file, hist_debug_show, file); 6157 if (ret) 6158 tracing_release_file_tr(inode, file); 6159 return ret; 6160 } 6161 6162 const struct file_operations event_hist_debug_fops = { 6163 .open = event_hist_debug_open, 6164 .read = seq_read, 6165 .llseek = seq_lseek, 6166 .release = tracing_single_release_file_tr, 6167 }; 6168 #endif 6169 6170 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 6171 { 6172 const char *field_name = hist_field_name(hist_field, 0); 6173 6174 if (hist_field->var.name) 6175 seq_printf(m, "%s=", hist_field->var.name); 6176 6177 if (hist_field->flags & HIST_FIELD_FL_CPU) 6178 seq_puts(m, "common_cpu"); 6179 if (hist_field->flags & HIST_FIELD_FL_COMM) 6180 seq_puts(m, "common_comm"); 6181 else if (hist_field->flags & HIST_FIELD_FL_CONST) 6182 seq_printf(m, "%llu", hist_field->constant); 6183 else if (field_name) { 6184 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 6185 hist_field->flags & HIST_FIELD_FL_ALIAS) 6186 if (!hist_field->system) 6187 seq_putc(m, '$'); 6188 seq_printf(m, "%s", field_name); 6189 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 6190 seq_puts(m, "common_timestamp"); 6191 6192 if (hist_field->flags) { 6193 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 6194 !(hist_field->flags & HIST_FIELD_FL_EXPR) && 6195 !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) { 6196 const char *flags = get_hist_field_flags(hist_field); 6197 6198 if (flags) 6199 seq_printf(m, ".%s", flags); 6200 } 6201 } 6202 if (hist_field->buckets) 6203 seq_printf(m, "=%ld", hist_field->buckets); 6204 } 6205 6206 static int event_hist_trigger_print(struct seq_file *m, 6207 struct event_trigger_data *data) 6208 { 6209 struct hist_trigger_data *hist_data = data->private_data; 6210 struct hist_field *field; 6211 bool have_var = false; 6212 bool show_val = false; 6213 unsigned int i; 6214 6215 seq_puts(m, HIST_PREFIX); 6216 6217 if (data->name) 6218 seq_printf(m, "%s:", data->name); 6219 6220 seq_puts(m, "keys="); 6221 6222 for_each_hist_key_field(i, hist_data) { 6223 field = hist_data->fields[i]; 6224 6225 if (i > hist_data->n_vals) 6226 seq_puts(m, ","); 6227 6228 if (field->flags & HIST_FIELD_FL_STACKTRACE) { 6229 if (field->field) 6230 seq_printf(m, "%s.stacktrace", field->field->name); 6231 else 6232 seq_puts(m, "common_stacktrace"); 6233 } else 6234 hist_field_print(m, field); 6235 } 6236 6237 seq_puts(m, ":vals="); 6238 6239 for_each_hist_val_field(i, hist_data) { 6240 field = hist_data->fields[i]; 6241 if (field->flags & HIST_FIELD_FL_VAR) { 6242 have_var = true; 6243 continue; 6244 } 6245 6246 if (i == HITCOUNT_IDX) { 6247 if (hist_data->attrs->no_hitcount) 6248 continue; 6249 seq_puts(m, "hitcount"); 6250 } else { 6251 if (show_val) 6252 seq_puts(m, ","); 6253 hist_field_print(m, field); 6254 } 6255 show_val = true; 6256 } 6257 6258 if (have_var) { 6259 unsigned int n = 0; 6260 6261 seq_puts(m, ":"); 6262 6263 for_each_hist_val_field(i, hist_data) { 6264 field = hist_data->fields[i]; 6265 6266 if (field->flags & HIST_FIELD_FL_VAR) { 6267 if (n++) 6268 seq_puts(m, ","); 6269 hist_field_print(m, field); 6270 } 6271 } 6272 } 6273 6274 seq_puts(m, ":sort="); 6275 6276 for (i = 0; i < hist_data->n_sort_keys; i++) { 6277 struct tracing_map_sort_key *sort_key; 6278 unsigned int idx, first_key_idx; 6279 6280 /* skip VAR vals */ 6281 first_key_idx = hist_data->n_vals - hist_data->n_vars; 6282 6283 sort_key = &hist_data->sort_keys[i]; 6284 idx = sort_key->field_idx; 6285 6286 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 6287 return -EINVAL; 6288 6289 if (i > 0) 6290 seq_puts(m, ","); 6291 6292 if (idx == HITCOUNT_IDX) 6293 seq_puts(m, "hitcount"); 6294 else { 6295 if (idx >= first_key_idx) 6296 idx += hist_data->n_vars; 6297 hist_field_print(m, hist_data->fields[idx]); 6298 } 6299 6300 if (sort_key->descending) 6301 seq_puts(m, ".descending"); 6302 } 6303 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 6304 if (hist_data->enable_timestamps) 6305 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 6306 if (hist_data->attrs->no_hitcount) 6307 seq_puts(m, ":nohitcount"); 6308 6309 print_actions_spec(m, hist_data); 6310 6311 if (data->filter_str) 6312 seq_printf(m, " if %s", data->filter_str); 6313 6314 if (data->paused) 6315 seq_puts(m, " [paused]"); 6316 else 6317 seq_puts(m, " [active]"); 6318 6319 seq_putc(m, '\n'); 6320 6321 return 0; 6322 } 6323 6324 static int event_hist_trigger_init(struct event_trigger_data *data) 6325 { 6326 struct hist_trigger_data *hist_data = data->private_data; 6327 6328 if (alloc_hist_pad() < 0) 6329 return -ENOMEM; 6330 6331 if (!data->ref && hist_data->attrs->name) 6332 save_named_trigger(hist_data->attrs->name, data); 6333 6334 data->ref++; 6335 6336 return 0; 6337 } 6338 6339 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 6340 { 6341 struct trace_event_file *file; 6342 unsigned int i; 6343 char *cmd; 6344 int ret; 6345 6346 for (i = 0; i < hist_data->n_field_var_hists; i++) { 6347 file = hist_data->field_var_hists[i]->hist_data->event_file; 6348 cmd = hist_data->field_var_hists[i]->cmd; 6349 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 6350 "!hist", "hist", cmd); 6351 WARN_ON_ONCE(ret < 0); 6352 } 6353 } 6354 6355 static void event_hist_trigger_free(struct event_trigger_data *data) 6356 { 6357 struct hist_trigger_data *hist_data = data->private_data; 6358 6359 if (WARN_ON_ONCE(data->ref <= 0)) 6360 return; 6361 6362 data->ref--; 6363 if (!data->ref) { 6364 if (data->name) 6365 del_named_trigger(data); 6366 6367 trigger_data_free(data); 6368 6369 tracepoint_synchronize_unregister(); 6370 remove_hist_vars(hist_data); 6371 6372 unregister_field_var_hists(hist_data); 6373 6374 destroy_hist_data(hist_data); 6375 } 6376 free_hist_pad(); 6377 } 6378 6379 static int event_hist_trigger_named_init(struct event_trigger_data *data) 6380 { 6381 int ret; 6382 6383 ret = event_hist_trigger_init(data->named_data); 6384 if (ret < 0) { 6385 kfree(data->cmd_ops); 6386 data->cmd_ops = &trigger_hist_cmd; 6387 return ret; 6388 } 6389 6390 data->ref++; 6391 6392 save_named_trigger(data->named_data->name, data); 6393 6394 return 0; 6395 } 6396 6397 static void event_hist_trigger_named_free(struct event_trigger_data *data) 6398 { 6399 if (WARN_ON_ONCE(data->ref <= 0)) 6400 return; 6401 6402 event_hist_trigger_free(data->named_data); 6403 6404 data->ref--; 6405 if (!data->ref) { 6406 struct event_command *cmd_ops = data->cmd_ops; 6407 6408 del_named_trigger(data); 6409 trigger_data_free(data); 6410 tracepoint_synchronize_unregister(); 6411 kfree(cmd_ops); 6412 } 6413 } 6414 6415 static void hist_clear(struct event_trigger_data *data) 6416 { 6417 struct hist_trigger_data *hist_data = data->private_data; 6418 6419 if (data->name) 6420 pause_named_trigger(data); 6421 6422 tracepoint_synchronize_unregister(); 6423 6424 tracing_map_clear(hist_data->map); 6425 6426 if (data->name) 6427 unpause_named_trigger(data); 6428 } 6429 6430 static bool compatible_field(struct ftrace_event_field *field, 6431 struct ftrace_event_field *test_field) 6432 { 6433 if (field == test_field) 6434 return true; 6435 if (field == NULL || test_field == NULL) 6436 return false; 6437 if (strcmp(field->name, test_field->name) != 0) 6438 return false; 6439 if (strcmp(field->type, test_field->type) != 0) 6440 return false; 6441 if (field->size != test_field->size) 6442 return false; 6443 if (field->is_signed != test_field->is_signed) 6444 return false; 6445 6446 return true; 6447 } 6448 6449 static bool hist_trigger_match(struct event_trigger_data *data, 6450 struct event_trigger_data *data_test, 6451 struct event_trigger_data *named_data, 6452 bool ignore_filter) 6453 { 6454 struct tracing_map_sort_key *sort_key, *sort_key_test; 6455 struct hist_trigger_data *hist_data, *hist_data_test; 6456 struct hist_field *key_field, *key_field_test; 6457 unsigned int i; 6458 6459 if (named_data && (named_data != data_test) && 6460 (named_data != data_test->named_data)) 6461 return false; 6462 6463 if (!named_data && is_named_trigger(data_test)) 6464 return false; 6465 6466 hist_data = data->private_data; 6467 hist_data_test = data_test->private_data; 6468 6469 if (hist_data->n_vals != hist_data_test->n_vals || 6470 hist_data->n_fields != hist_data_test->n_fields || 6471 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 6472 return false; 6473 6474 if (!ignore_filter) { 6475 if ((data->filter_str && !data_test->filter_str) || 6476 (!data->filter_str && data_test->filter_str)) 6477 return false; 6478 } 6479 6480 for_each_hist_field(i, hist_data) { 6481 key_field = hist_data->fields[i]; 6482 key_field_test = hist_data_test->fields[i]; 6483 6484 if (key_field->flags != key_field_test->flags) 6485 return false; 6486 if (!compatible_field(key_field->field, key_field_test->field)) 6487 return false; 6488 if (key_field->offset != key_field_test->offset) 6489 return false; 6490 if (key_field->size != key_field_test->size) 6491 return false; 6492 if (key_field->is_signed != key_field_test->is_signed) 6493 return false; 6494 if (!!key_field->var.name != !!key_field_test->var.name) 6495 return false; 6496 if (key_field->var.name && 6497 strcmp(key_field->var.name, key_field_test->var.name) != 0) 6498 return false; 6499 } 6500 6501 for (i = 0; i < hist_data->n_sort_keys; i++) { 6502 sort_key = &hist_data->sort_keys[i]; 6503 sort_key_test = &hist_data_test->sort_keys[i]; 6504 6505 if (sort_key->field_idx != sort_key_test->field_idx || 6506 sort_key->descending != sort_key_test->descending) 6507 return false; 6508 } 6509 6510 if (!ignore_filter && data->filter_str && 6511 (strcmp(data->filter_str, data_test->filter_str) != 0)) 6512 return false; 6513 6514 if (!actions_match(hist_data, hist_data_test)) 6515 return false; 6516 6517 return true; 6518 } 6519 6520 static bool existing_hist_update_only(char *glob, 6521 struct event_trigger_data *data, 6522 struct trace_event_file *file) 6523 { 6524 struct hist_trigger_data *hist_data = data->private_data; 6525 struct event_trigger_data *test, *named_data = NULL; 6526 bool updated = false; 6527 6528 if (!hist_data->attrs->pause && !hist_data->attrs->cont && 6529 !hist_data->attrs->clear) 6530 goto out; 6531 6532 if (hist_data->attrs->name) { 6533 named_data = find_named_trigger(hist_data->attrs->name); 6534 if (named_data) { 6535 if (!hist_trigger_match(data, named_data, named_data, 6536 true)) 6537 goto out; 6538 } 6539 } 6540 6541 if (hist_data->attrs->name && !named_data) 6542 goto out; 6543 6544 list_for_each_entry(test, &file->triggers, list) { 6545 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6546 if (!hist_trigger_match(data, test, named_data, false)) 6547 continue; 6548 if (hist_data->attrs->pause) 6549 test->paused = true; 6550 else if (hist_data->attrs->cont) 6551 test->paused = false; 6552 else if (hist_data->attrs->clear) 6553 hist_clear(test); 6554 updated = true; 6555 goto out; 6556 } 6557 } 6558 out: 6559 return updated; 6560 } 6561 6562 /* 6563 * Set or disable using the per CPU trace_buffer_event when possible. 6564 */ 6565 static int tracing_set_filter_buffering(struct trace_array *tr, bool set) 6566 { 6567 guard(mutex)(&trace_types_lock); 6568 6569 if (set && tr->no_filter_buffering_ref++) 6570 return 0; 6571 6572 if (!set) { 6573 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) 6574 return -EINVAL; 6575 6576 --tr->no_filter_buffering_ref; 6577 } 6578 6579 return 0; 6580 } 6581 6582 static int hist_register_trigger(char *glob, 6583 struct event_trigger_data *data, 6584 struct trace_event_file *file) 6585 { 6586 struct hist_trigger_data *hist_data = data->private_data; 6587 struct event_trigger_data *test, *named_data = NULL; 6588 struct trace_array *tr = file->tr; 6589 int ret = 0; 6590 6591 if (hist_data->attrs->name) { 6592 named_data = find_named_trigger(hist_data->attrs->name); 6593 if (named_data) { 6594 if (!hist_trigger_match(data, named_data, named_data, 6595 true)) { 6596 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 6597 ret = -EINVAL; 6598 goto out; 6599 } 6600 } 6601 } 6602 6603 if (hist_data->attrs->name && !named_data) 6604 goto new; 6605 6606 lockdep_assert_held(&event_mutex); 6607 6608 list_for_each_entry(test, &file->triggers, list) { 6609 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6610 if (hist_trigger_match(data, test, named_data, false)) { 6611 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 6612 ret = -EEXIST; 6613 goto out; 6614 } 6615 } 6616 } 6617 new: 6618 if (hist_data->attrs->cont || hist_data->attrs->clear) { 6619 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 6620 ret = -ENOENT; 6621 goto out; 6622 } 6623 6624 if (hist_data->attrs->pause) 6625 data->paused = true; 6626 6627 if (named_data) { 6628 struct event_command *cmd_ops; 6629 6630 data->private_data = named_data->private_data; 6631 set_named_trigger_data(data, named_data); 6632 /* Copy the command ops and update some of the functions */ 6633 cmd_ops = kmalloc_obj(*cmd_ops); 6634 if (!cmd_ops) { 6635 ret = -ENOMEM; 6636 goto out; 6637 } 6638 *cmd_ops = *data->cmd_ops; 6639 cmd_ops->init = event_hist_trigger_named_init; 6640 cmd_ops->free = event_hist_trigger_named_free; 6641 data->cmd_ops = cmd_ops; 6642 } 6643 6644 if (hist_data->enable_timestamps) { 6645 char *clock = hist_data->attrs->clock; 6646 6647 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 6648 if (ret) { 6649 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 6650 goto out; 6651 } 6652 6653 tracing_set_filter_buffering(file->tr, true); 6654 } 6655 6656 if (data->cmd_ops->init) { 6657 ret = data->cmd_ops->init(data); 6658 if (ret < 0) { 6659 if (hist_data->enable_timestamps) 6660 tracing_set_filter_buffering(file->tr, false); 6661 goto out; 6662 } 6663 } 6664 6665 if (named_data) { 6666 remove_hist_vars(hist_data); 6667 destroy_hist_data(hist_data); 6668 } 6669 out: 6670 return ret; 6671 } 6672 6673 static int hist_trigger_enable(struct event_trigger_data *data, 6674 struct trace_event_file *file) 6675 { 6676 int ret = 0; 6677 6678 list_add_tail_rcu(&data->list, &file->triggers); 6679 6680 update_cond_flag(file); 6681 6682 /* 6683 * On failure the caller undoes the registration, and 6684 * hist_unregister_trigger() can only find the trigger here. 6685 */ 6686 if (trace_event_trigger_enable_disable(file, 1) < 0) 6687 ret--; 6688 6689 return ret; 6690 } 6691 6692 static bool have_hist_trigger_match(struct event_trigger_data *data, 6693 struct trace_event_file *file) 6694 { 6695 struct hist_trigger_data *hist_data = data->private_data; 6696 struct event_trigger_data *test, *named_data = NULL; 6697 bool match = false; 6698 6699 lockdep_assert_held(&event_mutex); 6700 6701 if (hist_data->attrs->name) 6702 named_data = find_named_trigger(hist_data->attrs->name); 6703 6704 list_for_each_entry(test, &file->triggers, list) { 6705 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6706 if (hist_trigger_match(data, test, named_data, false)) { 6707 match = true; 6708 break; 6709 } 6710 } 6711 } 6712 6713 return match; 6714 } 6715 6716 static bool hist_trigger_check_refs(struct event_trigger_data *data, 6717 struct trace_event_file *file) 6718 { 6719 struct hist_trigger_data *hist_data = data->private_data; 6720 struct event_trigger_data *test, *named_data = NULL; 6721 6722 lockdep_assert_held(&event_mutex); 6723 6724 if (hist_data->attrs->name) 6725 named_data = find_named_trigger(hist_data->attrs->name); 6726 6727 list_for_each_entry(test, &file->triggers, list) { 6728 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6729 if (!hist_trigger_match(data, test, named_data, false)) 6730 continue; 6731 hist_data = test->private_data; 6732 if (check_var_refs(hist_data)) 6733 return true; 6734 break; 6735 } 6736 } 6737 6738 return false; 6739 } 6740 6741 static void hist_unregister_trigger(char *glob, 6742 struct event_trigger_data *data, 6743 struct trace_event_file *file) 6744 { 6745 struct event_trigger_data *test = NULL, *iter, *named_data = NULL; 6746 struct hist_trigger_data *hist_data = data->private_data; 6747 6748 lockdep_assert_held(&event_mutex); 6749 6750 if (hist_data->attrs->name) 6751 named_data = find_named_trigger(hist_data->attrs->name); 6752 6753 list_for_each_entry(iter, &file->triggers, list) { 6754 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6755 if (!hist_trigger_match(data, iter, named_data, false)) 6756 continue; 6757 test = iter; 6758 list_del_rcu(&test->list); 6759 trace_event_trigger_enable_disable(file, 0); 6760 update_cond_flag(file); 6761 break; 6762 } 6763 } 6764 6765 if (hist_data->enable_timestamps) { 6766 if (!hist_data->remove || test) 6767 tracing_set_filter_buffering(file->tr, false); 6768 } 6769 6770 if (test && test->cmd_ops->free) 6771 test->cmd_ops->free(test); 6772 } 6773 6774 static bool hist_file_check_refs(struct trace_event_file *file) 6775 { 6776 struct hist_trigger_data *hist_data; 6777 struct event_trigger_data *test; 6778 6779 lockdep_assert_held(&event_mutex); 6780 6781 list_for_each_entry(test, &file->triggers, list) { 6782 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6783 hist_data = test->private_data; 6784 if (check_var_refs(hist_data)) 6785 return true; 6786 } 6787 } 6788 6789 return false; 6790 } 6791 6792 static void hist_unreg_all(struct trace_event_file *file) 6793 { 6794 struct event_trigger_data *test, *n; 6795 struct hist_trigger_data *hist_data; 6796 struct synth_event *se; 6797 const char *se_name; 6798 6799 lockdep_assert_held(&event_mutex); 6800 6801 if (hist_file_check_refs(file)) 6802 return; 6803 6804 list_for_each_entry_safe(test, n, &file->triggers, list) { 6805 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6806 hist_data = test->private_data; 6807 list_del_rcu(&test->list); 6808 trace_event_trigger_enable_disable(file, 0); 6809 6810 se_name = trace_event_name(file->event_call); 6811 se = find_synth_event(se_name); 6812 if (se) 6813 se->ref--; 6814 6815 update_cond_flag(file); 6816 if (hist_data->enable_timestamps) 6817 tracing_set_filter_buffering(file->tr, false); 6818 if (test->cmd_ops->free) 6819 test->cmd_ops->free(test); 6820 } 6821 } 6822 } 6823 6824 static int event_hist_trigger_parse(struct event_command *cmd_ops, 6825 struct trace_event_file *file, 6826 char *glob, char *cmd, 6827 char *param_and_filter) 6828 { 6829 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6830 struct event_trigger_data *trigger_data; 6831 struct hist_trigger_attrs *attrs; 6832 struct hist_trigger_data *hist_data; 6833 char *param, *filter, *p, *start; 6834 struct synth_event *se; 6835 const char *se_name; 6836 bool remove; 6837 int ret = 0; 6838 6839 lockdep_assert_held(&event_mutex); 6840 6841 if (WARN_ON(!glob)) 6842 return -EINVAL; 6843 6844 if (glob[0]) { 6845 hist_err_clear(); 6846 last_cmd_set(file, param_and_filter); 6847 } 6848 6849 remove = event_trigger_check_remove(glob); 6850 6851 if (event_trigger_empty_param(param_and_filter)) 6852 return -EINVAL; 6853 6854 /* 6855 * separate the trigger from the filter (k:v [if filter]) 6856 * allowing for whitespace in the trigger 6857 */ 6858 p = param = param_and_filter; 6859 do { 6860 p = strstr(p, "if"); 6861 if (!p) 6862 break; 6863 if (p == param_and_filter) 6864 return -EINVAL; 6865 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6866 p++; 6867 continue; 6868 } 6869 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1) 6870 return -EINVAL; 6871 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6872 p++; 6873 continue; 6874 } 6875 break; 6876 } while (1); 6877 6878 if (!p) 6879 filter = NULL; 6880 else { 6881 *(p - 1) = '\0'; 6882 filter = strstrip(p); 6883 param = strstrip(param); 6884 } 6885 6886 /* 6887 * To simplify arithmetic expression parsing, replace occurrences of 6888 * '.sym-offset' modifier with '.symXoffset' 6889 */ 6890 start = strstr(param, ".sym-offset"); 6891 while (start) { 6892 *(start + 4) = 'X'; 6893 start = strstr(start + 11, ".sym-offset"); 6894 } 6895 6896 attrs = parse_hist_trigger_attrs(file->tr, param); 6897 if (IS_ERR(attrs)) 6898 return PTR_ERR(attrs); 6899 6900 if (attrs->map_bits) 6901 hist_trigger_bits = attrs->map_bits; 6902 6903 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6904 if (IS_ERR(hist_data)) { 6905 destroy_hist_trigger_attrs(attrs); 6906 return PTR_ERR(hist_data); 6907 } 6908 6909 trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data); 6910 if (!trigger_data) { 6911 ret = -ENOMEM; 6912 goto out_free; 6913 } 6914 6915 ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data); 6916 if (ret < 0) 6917 goto out_free; 6918 6919 if (remove) { 6920 if (!have_hist_trigger_match(trigger_data, file)) 6921 goto out_free; 6922 6923 if (hist_trigger_check_refs(trigger_data, file)) { 6924 ret = -EBUSY; 6925 goto out_free; 6926 } 6927 6928 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6929 se_name = trace_event_name(file->event_call); 6930 se = find_synth_event(se_name); 6931 if (se) 6932 se->ref--; 6933 ret = 0; 6934 goto out_free; 6935 } 6936 6937 if (existing_hist_update_only(glob, trigger_data, file)) 6938 goto out_free; 6939 6940 if (!get_named_trigger_data(trigger_data)) { 6941 6942 ret = create_actions(hist_data); 6943 if (ret) 6944 goto out_free; 6945 6946 if (has_hist_vars(hist_data) || hist_data->n_var_refs) { 6947 ret = save_hist_vars(hist_data); 6948 if (ret) 6949 goto out_free; 6950 } 6951 6952 ret = tracing_map_init(hist_data->map); 6953 if (ret) 6954 goto out_free; 6955 } 6956 6957 ret = event_trigger_register(cmd_ops, file, glob, trigger_data); 6958 if (ret < 0) 6959 goto out_free; 6960 6961 ret = hist_trigger_enable(trigger_data, file); 6962 if (ret) 6963 goto out_unreg; 6964 6965 se_name = trace_event_name(file->event_call); 6966 se = find_synth_event(se_name); 6967 if (se) 6968 se->ref++; 6969 out: 6970 if (ret == 0 && glob[0]) 6971 hist_err_clear(); 6972 6973 return ret; 6974 out_unreg: 6975 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6976 /* The unregister frees trigger_data, skip out_free */ 6977 goto out; 6978 out_free: 6979 remove_hist_vars(hist_data); 6980 6981 trigger_data_free(trigger_data); 6982 6983 destroy_hist_data(hist_data); 6984 goto out; 6985 } 6986 6987 static struct event_command trigger_hist_cmd = { 6988 .name = "hist", 6989 .trigger_type = ETT_EVENT_HIST, 6990 .flags = EVENT_CMD_FL_NEEDS_REC, 6991 .parse = event_hist_trigger_parse, 6992 .reg = hist_register_trigger, 6993 .unreg = hist_unregister_trigger, 6994 .unreg_all = hist_unreg_all, 6995 .set_filter = set_trigger_filter, 6996 .trigger = event_hist_trigger, 6997 .print = event_hist_trigger_print, 6998 .init = event_hist_trigger_init, 6999 .free = event_hist_trigger_free, 7000 }; 7001 7002 __init int register_trigger_hist_cmd(void) 7003 { 7004 int ret; 7005 7006 ret = register_event_command(&trigger_hist_cmd); 7007 WARN_ON(ret < 0); 7008 7009 return ret; 7010 } 7011 7012 static void 7013 hist_enable_trigger(struct event_trigger_data *data, 7014 struct trace_buffer *buffer, void *rec, 7015 struct ring_buffer_event *event) 7016 { 7017 struct enable_trigger_data *enable_data = data->private_data; 7018 struct event_trigger_data *test; 7019 7020 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 7021 lockdep_is_held(&event_mutex)) { 7022 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 7023 if (enable_data->enable) 7024 test->paused = false; 7025 else 7026 test->paused = true; 7027 } 7028 } 7029 } 7030 7031 static void hist_enable_unreg_all(struct trace_event_file *file) 7032 { 7033 struct event_trigger_data *test, *n; 7034 7035 list_for_each_entry_safe(test, n, &file->triggers, list) { 7036 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 7037 list_del_rcu(&test->list); 7038 update_cond_flag(file); 7039 trace_event_trigger_enable_disable(file, 0); 7040 if (test->cmd_ops->free) 7041 test->cmd_ops->free(test); 7042 } 7043 } 7044 } 7045 7046 static struct event_command trigger_hist_enable_cmd = { 7047 .name = ENABLE_HIST_STR, 7048 .trigger_type = ETT_HIST_ENABLE, 7049 .parse = event_enable_trigger_parse, 7050 .reg = event_enable_register_trigger, 7051 .unreg = event_enable_unregister_trigger, 7052 .unreg_all = hist_enable_unreg_all, 7053 .set_filter = set_trigger_filter, 7054 .trigger = hist_enable_trigger, 7055 .count_func = event_trigger_count, 7056 .print = event_enable_trigger_print, 7057 .init = event_trigger_init, 7058 .free = event_enable_trigger_free, 7059 }; 7060 7061 static struct event_command trigger_hist_disable_cmd = { 7062 .name = DISABLE_HIST_STR, 7063 .trigger_type = ETT_HIST_ENABLE, 7064 .parse = event_enable_trigger_parse, 7065 .reg = event_enable_register_trigger, 7066 .unreg = event_enable_unregister_trigger, 7067 .unreg_all = hist_enable_unreg_all, 7068 .set_filter = set_trigger_filter, 7069 .trigger = hist_enable_trigger, 7070 .count_func = event_trigger_count, 7071 .print = event_enable_trigger_print, 7072 .init = event_trigger_init, 7073 .free = event_enable_trigger_free, 7074 }; 7075 7076 static __init void unregister_trigger_hist_enable_disable_cmds(void) 7077 { 7078 unregister_event_command(&trigger_hist_enable_cmd); 7079 unregister_event_command(&trigger_hist_disable_cmd); 7080 } 7081 7082 __init int register_trigger_hist_enable_disable_cmds(void) 7083 { 7084 int ret; 7085 7086 ret = register_event_command(&trigger_hist_enable_cmd); 7087 if (WARN_ON(ret < 0)) 7088 return ret; 7089 ret = register_event_command(&trigger_hist_disable_cmd); 7090 if (WARN_ON(ret < 0)) 7091 unregister_trigger_hist_enable_disable_cmds(); 7092 7093 return ret; 7094 } 7095