1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/seq_buf.h> 12 #include <linux/mutex.h> 13 #include <linux/slab.h> 14 #include <linux/stacktrace.h> 15 #include <linux/rculist.h> 16 #include <linux/tracefs.h> 17 18 /* for gfp flag names */ 19 #include <linux/trace_events.h> 20 #include <trace/events/mmflags.h> 21 22 #include "tracing_map.h" 23 #include "trace_synth.h" 24 25 #define ERRORS \ 26 C(NONE, "No error"), \ 27 C(DUPLICATE_VAR, "Variable already defined"), \ 28 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 29 C(TOO_MANY_VARS, "Too many variables defined"), \ 30 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 31 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 32 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 33 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 34 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 35 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 36 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 37 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 38 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 39 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 40 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 41 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 42 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 43 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 44 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 45 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 46 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 47 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 48 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 49 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 50 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 51 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 52 C(TOO_MANY_PARAMS, "Too many action params"), \ 53 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 54 C(INVALID_PARAM, "Invalid action param"), \ 55 C(ACTION_NOT_FOUND, "No action found"), \ 56 C(NO_SAVE_PARAMS, "No params found for save()"), \ 57 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 58 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 59 C(NO_CLOSING_PAREN, "No closing paren found"), \ 60 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 61 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 62 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 63 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 64 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 65 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 66 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 67 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 68 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 69 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \ 70 C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \ 71 C(EXPECT_NUMBER, "Expecting numeric literal"), \ 72 C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \ 73 C(DIVISION_BY_ZERO, "Division by zero"), \ 74 C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"), 75 76 #undef C 77 #define C(a, b) HIST_ERR_##a 78 79 enum { ERRORS }; 80 81 #undef C 82 #define C(a, b) b 83 84 static const char *err_text[] = { ERRORS }; 85 86 struct hist_field; 87 88 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 89 struct tracing_map_elt *elt, 90 struct trace_buffer *buffer, 91 struct ring_buffer_event *rbe, 92 void *event); 93 94 #define HIST_FIELD_OPERANDS_MAX 2 95 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 96 #define HIST_ACTIONS_MAX 8 97 #define HIST_DIV_SHIFT 20 /* For optimizing division by constants */ 98 99 enum field_op_id { 100 FIELD_OP_NONE, 101 FIELD_OP_PLUS, 102 FIELD_OP_MINUS, 103 FIELD_OP_UNARY_MINUS, 104 FIELD_OP_DIV, 105 FIELD_OP_MULT, 106 }; 107 108 #define FIELD_FUNCS \ 109 C(NOP, "nop"), \ 110 C(VAR_REF, "var_ref"), \ 111 C(COUNTER, "counter"), \ 112 C(CONST, "const"), \ 113 C(LOG2, "log2"), \ 114 C(BUCKET, "bucket"), \ 115 C(TIMESTAMP, "timestamp"), \ 116 C(CPU, "cpu"), \ 117 C(COMM, "comm"), \ 118 C(STRING, "string"), \ 119 C(DYNSTRING, "dynstring"), \ 120 C(RELDYNSTRING, "reldynstring"), \ 121 C(PSTRING, "pstring"), \ 122 C(S64, "s64"), \ 123 C(U64, "u64"), \ 124 C(S32, "s32"), \ 125 C(U32, "u32"), \ 126 C(S16, "s16"), \ 127 C(U16, "u16"), \ 128 C(S8, "s8"), \ 129 C(U8, "u8"), \ 130 C(UMINUS, "uminus"), \ 131 C(MINUS, "minus"), \ 132 C(PLUS, "plus"), \ 133 C(DIV, "div"), \ 134 C(MULT, "mult"), \ 135 C(DIV_POWER2, "div_power2"), \ 136 C(DIV_NOT_POWER2, "div_not_power2"), \ 137 C(DIV_MULT_SHIFT, "div_mult_shift"), \ 138 C(EXECNAME, "execname"), \ 139 C(STACK, "stack"), 140 141 #undef C 142 #define C(a, b) HIST_FIELD_FN_##a 143 144 enum hist_field_fn { 145 FIELD_FUNCS 146 }; 147 148 /* 149 * A hist_var (histogram variable) contains variable information for 150 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 151 * flag set. A hist_var has a variable name e.g. ts0, and is 152 * associated with a given histogram trigger, as specified by 153 * hist_data. The hist_var idx is the unique index assigned to the 154 * variable by the hist trigger's tracing_map. The idx is what is 155 * used to set a variable's value and, by a variable reference, to 156 * retrieve it. 157 */ 158 struct hist_var { 159 char *name; 160 struct hist_trigger_data *hist_data; 161 unsigned int idx; 162 }; 163 164 struct hist_field { 165 struct ftrace_event_field *field; 166 unsigned long flags; 167 unsigned long buckets; 168 const char *type; 169 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 170 struct hist_trigger_data *hist_data; 171 enum hist_field_fn fn_num; 172 unsigned int size; 173 unsigned int offset; 174 unsigned int is_signed; 175 176 /* 177 * Variable fields contain variable-specific info in var. 178 */ 179 struct hist_var var; 180 enum field_op_id operator; 181 char *system; 182 char *event_name; 183 184 /* 185 * The name field is used for EXPR and VAR_REF fields. VAR 186 * fields contain the variable name in var.name. 187 */ 188 char *name; 189 190 /* 191 * When a histogram trigger is hit, if it has any references 192 * to variables, the values of those variables are collected 193 * into a var_ref_vals array by resolve_var_refs(). The 194 * current value of each variable is read from the tracing_map 195 * using the hist field's hist_var.idx and entered into the 196 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 197 */ 198 unsigned int var_ref_idx; 199 bool read_once; 200 201 unsigned int var_str_idx; 202 203 /* Numeric literals are represented as u64 */ 204 u64 constant; 205 /* Used to optimize division by constants */ 206 u64 div_multiplier; 207 }; 208 209 static u64 hist_fn_call(struct hist_field *hist_field, 210 struct tracing_map_elt *elt, 211 struct trace_buffer *buffer, 212 struct ring_buffer_event *rbe, 213 void *event); 214 215 static u64 hist_field_const(struct hist_field *field, 216 struct tracing_map_elt *elt, 217 struct trace_buffer *buffer, 218 struct ring_buffer_event *rbe, 219 void *event) 220 { 221 return field->constant; 222 } 223 224 static u64 hist_field_counter(struct hist_field *field, 225 struct tracing_map_elt *elt, 226 struct trace_buffer *buffer, 227 struct ring_buffer_event *rbe, 228 void *event) 229 { 230 return 1; 231 } 232 233 static u64 hist_field_string(struct hist_field *hist_field, 234 struct tracing_map_elt *elt, 235 struct trace_buffer *buffer, 236 struct ring_buffer_event *rbe, 237 void *event) 238 { 239 char *addr = (char *)(event + hist_field->field->offset); 240 241 return (u64)(unsigned long)addr; 242 } 243 244 static u64 hist_field_dynstring(struct hist_field *hist_field, 245 struct tracing_map_elt *elt, 246 struct trace_buffer *buffer, 247 struct ring_buffer_event *rbe, 248 void *event) 249 { 250 u32 str_item = *(u32 *)(event + hist_field->field->offset); 251 int str_loc = str_item & 0xffff; 252 char *addr = (char *)(event + str_loc); 253 254 return (u64)(unsigned long)addr; 255 } 256 257 static u64 hist_field_reldynstring(struct hist_field *hist_field, 258 struct tracing_map_elt *elt, 259 struct trace_buffer *buffer, 260 struct ring_buffer_event *rbe, 261 void *event) 262 { 263 u32 *item = event + hist_field->field->offset; 264 u32 str_item = *item; 265 int str_loc = str_item & 0xffff; 266 char *addr = (char *)&item[1] + str_loc; 267 268 return (u64)(unsigned long)addr; 269 } 270 271 static u64 hist_field_pstring(struct hist_field *hist_field, 272 struct tracing_map_elt *elt, 273 struct trace_buffer *buffer, 274 struct ring_buffer_event *rbe, 275 void *event) 276 { 277 char **addr = (char **)(event + hist_field->field->offset); 278 279 return (u64)(unsigned long)*addr; 280 } 281 282 static bool hist_field_is_signed(struct hist_field *hist_field) 283 { 284 if (hist_field->field) 285 return hist_field->field->is_signed; 286 287 return hist_field->is_signed; 288 } 289 290 static u64 hist_field_log2(struct hist_field *hist_field, 291 struct tracing_map_elt *elt, 292 struct trace_buffer *buffer, 293 struct ring_buffer_event *rbe, 294 void *event) 295 { 296 struct hist_field *operand = hist_field->operands[0]; 297 298 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 299 300 return order_base_2(val); 301 } 302 303 static u64 hist_field_bucket(struct hist_field *hist_field, 304 struct tracing_map_elt *elt, 305 struct trace_buffer *buffer, 306 struct ring_buffer_event *rbe, 307 void *event) 308 { 309 struct hist_field *operand = hist_field->operands[0]; 310 unsigned long buckets = hist_field->buckets; 311 312 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 313 314 if (WARN_ON_ONCE(!buckets)) 315 return val; 316 317 if (val >= LONG_MAX) 318 val = div64_ul(val, buckets); 319 else 320 val = (u64)((unsigned long)val / buckets); 321 return val * buckets; 322 } 323 324 static u64 hist_field_plus(struct hist_field *hist_field, 325 struct tracing_map_elt *elt, 326 struct trace_buffer *buffer, 327 struct ring_buffer_event *rbe, 328 void *event) 329 { 330 struct hist_field *operand1 = hist_field->operands[0]; 331 struct hist_field *operand2 = hist_field->operands[1]; 332 333 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 334 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 335 336 return val1 + val2; 337 } 338 339 static u64 hist_field_minus(struct hist_field *hist_field, 340 struct tracing_map_elt *elt, 341 struct trace_buffer *buffer, 342 struct ring_buffer_event *rbe, 343 void *event) 344 { 345 struct hist_field *operand1 = hist_field->operands[0]; 346 struct hist_field *operand2 = hist_field->operands[1]; 347 348 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 349 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 350 351 return val1 - val2; 352 } 353 354 static u64 hist_field_div(struct hist_field *hist_field, 355 struct tracing_map_elt *elt, 356 struct trace_buffer *buffer, 357 struct ring_buffer_event *rbe, 358 void *event) 359 { 360 struct hist_field *operand1 = hist_field->operands[0]; 361 struct hist_field *operand2 = hist_field->operands[1]; 362 363 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 364 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 365 366 /* Return -1 for the undefined case */ 367 if (!val2) 368 return -1; 369 370 /* Use shift if the divisor is a power of 2 */ 371 if (!(val2 & (val2 - 1))) 372 return val1 >> __ffs64(val2); 373 374 return div64_u64(val1, val2); 375 } 376 377 static u64 div_by_power_of_two(struct hist_field *hist_field, 378 struct tracing_map_elt *elt, 379 struct trace_buffer *buffer, 380 struct ring_buffer_event *rbe, 381 void *event) 382 { 383 struct hist_field *operand1 = hist_field->operands[0]; 384 struct hist_field *operand2 = hist_field->operands[1]; 385 386 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 387 388 return val1 >> __ffs64(operand2->constant); 389 } 390 391 static u64 div_by_not_power_of_two(struct hist_field *hist_field, 392 struct tracing_map_elt *elt, 393 struct trace_buffer *buffer, 394 struct ring_buffer_event *rbe, 395 void *event) 396 { 397 struct hist_field *operand1 = hist_field->operands[0]; 398 struct hist_field *operand2 = hist_field->operands[1]; 399 400 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 401 402 return div64_u64(val1, operand2->constant); 403 } 404 405 static u64 div_by_mult_and_shift(struct hist_field *hist_field, 406 struct tracing_map_elt *elt, 407 struct trace_buffer *buffer, 408 struct ring_buffer_event *rbe, 409 void *event) 410 { 411 struct hist_field *operand1 = hist_field->operands[0]; 412 struct hist_field *operand2 = hist_field->operands[1]; 413 414 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 415 416 /* 417 * If the divisor is a constant, do a multiplication and shift instead. 418 * 419 * Choose Z = some power of 2. If Y <= Z, then: 420 * X / Y = (X * (Z / Y)) / Z 421 * 422 * (Z / Y) is a constant (mult) which is calculated at parse time, so: 423 * X / Y = (X * mult) / Z 424 * 425 * The division by Z can be replaced by a shift since Z is a power of 2: 426 * X / Y = (X * mult) >> HIST_DIV_SHIFT 427 * 428 * As long, as X < Z the results will not be off by more than 1. 429 */ 430 if (val1 < (1 << HIST_DIV_SHIFT)) { 431 u64 mult = operand2->div_multiplier; 432 433 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT; 434 } 435 436 return div64_u64(val1, operand2->constant); 437 } 438 439 static u64 hist_field_mult(struct hist_field *hist_field, 440 struct tracing_map_elt *elt, 441 struct trace_buffer *buffer, 442 struct ring_buffer_event *rbe, 443 void *event) 444 { 445 struct hist_field *operand1 = hist_field->operands[0]; 446 struct hist_field *operand2 = hist_field->operands[1]; 447 448 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 449 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 450 451 return val1 * val2; 452 } 453 454 static u64 hist_field_unary_minus(struct hist_field *hist_field, 455 struct tracing_map_elt *elt, 456 struct trace_buffer *buffer, 457 struct ring_buffer_event *rbe, 458 void *event) 459 { 460 struct hist_field *operand = hist_field->operands[0]; 461 462 s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event); 463 u64 val = (u64)-sval; 464 465 return val; 466 } 467 468 #define DEFINE_HIST_FIELD_FN(type) \ 469 static u64 hist_field_##type(struct hist_field *hist_field, \ 470 struct tracing_map_elt *elt, \ 471 struct trace_buffer *buffer, \ 472 struct ring_buffer_event *rbe, \ 473 void *event) \ 474 { \ 475 type *addr = (type *)(event + hist_field->field->offset); \ 476 \ 477 return (u64)*addr; \ 478 } 479 480 DEFINE_HIST_FIELD_FN(s64); 481 DEFINE_HIST_FIELD_FN(u64); 482 DEFINE_HIST_FIELD_FN(s32); 483 DEFINE_HIST_FIELD_FN(u32); 484 DEFINE_HIST_FIELD_FN(s16); 485 DEFINE_HIST_FIELD_FN(u16); 486 DEFINE_HIST_FIELD_FN(s8); 487 DEFINE_HIST_FIELD_FN(u8); 488 489 #define for_each_hist_field(i, hist_data) \ 490 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 491 492 #define for_each_hist_val_field(i, hist_data) \ 493 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 494 495 #define for_each_hist_key_field(i, hist_data) \ 496 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 497 498 #define HITCOUNT_IDX 0 499 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 500 501 enum hist_field_flags { 502 HIST_FIELD_FL_HITCOUNT = 1 << 0, 503 HIST_FIELD_FL_KEY = 1 << 1, 504 HIST_FIELD_FL_STRING = 1 << 2, 505 HIST_FIELD_FL_HEX = 1 << 3, 506 HIST_FIELD_FL_SYM = 1 << 4, 507 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 508 HIST_FIELD_FL_EXECNAME = 1 << 6, 509 HIST_FIELD_FL_SYSCALL = 1 << 7, 510 HIST_FIELD_FL_STACKTRACE = 1 << 8, 511 HIST_FIELD_FL_LOG2 = 1 << 9, 512 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 513 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 514 HIST_FIELD_FL_VAR = 1 << 12, 515 HIST_FIELD_FL_EXPR = 1 << 13, 516 HIST_FIELD_FL_VAR_REF = 1 << 14, 517 HIST_FIELD_FL_CPU = 1 << 15, 518 HIST_FIELD_FL_ALIAS = 1 << 16, 519 HIST_FIELD_FL_BUCKET = 1 << 17, 520 HIST_FIELD_FL_CONST = 1 << 18, 521 HIST_FIELD_FL_PERCENT = 1 << 19, 522 HIST_FIELD_FL_GRAPH = 1 << 20, 523 HIST_FIELD_FL_COMM = 1 << 21, 524 }; 525 526 struct var_defs { 527 unsigned int n_vars; 528 char *name[TRACING_MAP_VARS_MAX]; 529 char *expr[TRACING_MAP_VARS_MAX]; 530 }; 531 532 struct hist_trigger_attrs { 533 char *keys_str; 534 char *vals_str; 535 char *sort_key_str; 536 char *name; 537 char *clock; 538 bool pause; 539 bool cont; 540 bool clear; 541 bool ts_in_usecs; 542 bool no_hitcount; 543 unsigned int map_bits; 544 545 char *assignment_str[TRACING_MAP_VARS_MAX]; 546 unsigned int n_assignments; 547 548 char *action_str[HIST_ACTIONS_MAX]; 549 unsigned int n_actions; 550 551 struct var_defs var_defs; 552 }; 553 554 struct field_var { 555 struct hist_field *var; 556 struct hist_field *val; 557 }; 558 559 struct field_var_hist { 560 struct trace_event_file *file; 561 char *cmd; 562 }; 563 564 struct hist_trigger_data { 565 struct hist_field *fields[HIST_FIELDS_MAX]; 566 unsigned int n_vals; 567 unsigned int n_keys; 568 unsigned int n_fields; 569 unsigned int n_vars; 570 unsigned int n_var_str; 571 unsigned int key_size; 572 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 573 unsigned int n_sort_keys; 574 struct trace_event_file *event_file; 575 struct hist_trigger_attrs *attrs; 576 struct tracing_map *map; 577 bool enable_timestamps; 578 bool remove; 579 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 580 unsigned int n_var_refs; 581 582 struct action_data *actions[HIST_ACTIONS_MAX]; 583 unsigned int n_actions; 584 585 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 586 unsigned int n_field_vars; 587 unsigned int n_field_var_str; 588 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 589 unsigned int n_field_var_hists; 590 591 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 592 unsigned int n_save_vars; 593 unsigned int n_save_var_str; 594 }; 595 596 struct action_data; 597 598 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 599 struct tracing_map_elt *elt, 600 struct trace_buffer *buffer, void *rec, 601 struct ring_buffer_event *rbe, void *key, 602 struct action_data *data, u64 *var_ref_vals); 603 604 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 605 606 enum handler_id { 607 HANDLER_ONMATCH = 1, 608 HANDLER_ONMAX, 609 HANDLER_ONCHANGE, 610 }; 611 612 enum action_id { 613 ACTION_SAVE = 1, 614 ACTION_TRACE, 615 ACTION_SNAPSHOT, 616 }; 617 618 struct action_data { 619 enum handler_id handler; 620 enum action_id action; 621 char *action_name; 622 action_fn_t fn; 623 624 unsigned int n_params; 625 char *params[SYNTH_FIELDS_MAX]; 626 627 /* 628 * When a histogram trigger is hit, the values of any 629 * references to variables, including variables being passed 630 * as parameters to synthetic events, are collected into a 631 * var_ref_vals array. This var_ref_idx array is an array of 632 * indices into the var_ref_vals array, one for each synthetic 633 * event param, and is passed to the synthetic event 634 * invocation. 635 */ 636 unsigned int var_ref_idx[SYNTH_FIELDS_MAX]; 637 struct synth_event *synth_event; 638 bool use_trace_keyword; 639 char *synth_event_name; 640 641 union { 642 struct { 643 char *event; 644 char *event_system; 645 } match_data; 646 647 struct { 648 /* 649 * var_str contains the $-unstripped variable 650 * name referenced by var_ref, and used when 651 * printing the action. Because var_ref 652 * creation is deferred to create_actions(), 653 * we need a per-action way to save it until 654 * then, thus var_str. 655 */ 656 char *var_str; 657 658 /* 659 * var_ref refers to the variable being 660 * tracked e.g onmax($var). 661 */ 662 struct hist_field *var_ref; 663 664 /* 665 * track_var contains the 'invisible' tracking 666 * variable created to keep the current 667 * e.g. max value. 668 */ 669 struct hist_field *track_var; 670 671 check_track_val_fn_t check_val; 672 action_fn_t save_data; 673 } track_data; 674 }; 675 }; 676 677 struct track_data { 678 u64 track_val; 679 bool updated; 680 681 unsigned int key_len; 682 void *key; 683 struct tracing_map_elt elt; 684 685 struct action_data *action_data; 686 struct hist_trigger_data *hist_data; 687 }; 688 689 struct hist_elt_data { 690 char *comm; 691 u64 *var_ref_vals; 692 int n_field_var_str; 693 char *field_var_str[] __counted_by(n_field_var_str); 694 }; 695 696 struct snapshot_context { 697 struct tracing_map_elt *elt; 698 void *key; 699 }; 700 701 /* 702 * Returns the specific division function to use if the divisor 703 * is constant. This avoids extra branches when the trigger is hit. 704 */ 705 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor) 706 { 707 u64 div = divisor->constant; 708 709 if (!(div & (div - 1))) 710 return HIST_FIELD_FN_DIV_POWER2; 711 712 /* If the divisor is too large, do a regular division */ 713 if (div > (1 << HIST_DIV_SHIFT)) 714 return HIST_FIELD_FN_DIV_NOT_POWER2; 715 716 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div); 717 return HIST_FIELD_FN_DIV_MULT_SHIFT; 718 } 719 720 static void track_data_free(struct track_data *track_data) 721 { 722 struct hist_elt_data *elt_data; 723 724 if (!track_data) 725 return; 726 727 kfree(track_data->key); 728 729 elt_data = track_data->elt.private_data; 730 if (elt_data) { 731 kfree(elt_data->comm); 732 kfree(elt_data); 733 } 734 735 kfree(track_data); 736 } 737 738 static struct track_data *track_data_alloc(unsigned int key_len, 739 struct action_data *action_data, 740 struct hist_trigger_data *hist_data) 741 { 742 struct track_data *data = kzalloc_obj(*data); 743 struct hist_elt_data *elt_data; 744 745 if (!data) 746 return ERR_PTR(-ENOMEM); 747 748 data->key = kzalloc(key_len, GFP_KERNEL); 749 if (!data->key) { 750 track_data_free(data); 751 return ERR_PTR(-ENOMEM); 752 } 753 754 data->key_len = key_len; 755 data->action_data = action_data; 756 data->hist_data = hist_data; 757 758 elt_data = kzalloc_obj(*elt_data); 759 if (!elt_data) { 760 track_data_free(data); 761 return ERR_PTR(-ENOMEM); 762 } 763 764 data->elt.private_data = elt_data; 765 766 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 767 if (!elt_data->comm) { 768 track_data_free(data); 769 return ERR_PTR(-ENOMEM); 770 } 771 772 return data; 773 } 774 775 #define HIST_PREFIX "hist:" 776 777 static char *last_cmd; 778 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 779 780 static int errpos(char *str) 781 { 782 if (!str || !last_cmd) 783 return 0; 784 785 return err_pos(last_cmd, str); 786 } 787 788 static void last_cmd_set(struct trace_event_file *file, char *str) 789 { 790 const char *system = NULL, *name = NULL; 791 struct trace_event_call *call; 792 793 if (!str) 794 return; 795 796 kfree(last_cmd); 797 798 last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str); 799 if (!last_cmd) 800 return; 801 802 if (file) { 803 call = file->event_call; 804 system = call->class->system; 805 if (system) { 806 name = trace_event_name(call); 807 if (!name) 808 system = NULL; 809 } 810 } 811 812 if (system) 813 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name); 814 } 815 816 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos) 817 { 818 if (!last_cmd) 819 return; 820 821 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 822 err_type, err_pos); 823 } 824 825 static void hist_err_clear(void) 826 { 827 if (last_cmd) 828 last_cmd[0] = '\0'; 829 last_cmd_loc[0] = '\0'; 830 } 831 832 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 833 unsigned int *var_ref_idx); 834 835 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 836 unsigned int *var_ref_idx) 837 { 838 struct tracepoint *tp = event->tp; 839 840 if (unlikely(static_key_enabled(&tp->key))) { 841 struct tracepoint_func *probe_func_ptr; 842 synth_probe_func_t probe_func; 843 void *__data; 844 845 if (!(cpu_online(raw_smp_processor_id()))) 846 return; 847 848 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 849 if (probe_func_ptr) { 850 do { 851 probe_func = probe_func_ptr->func; 852 __data = probe_func_ptr->data; 853 probe_func(__data, var_ref_vals, var_ref_idx); 854 } while ((++probe_func_ptr)->func); 855 } 856 } 857 } 858 859 static void action_trace(struct hist_trigger_data *hist_data, 860 struct tracing_map_elt *elt, 861 struct trace_buffer *buffer, void *rec, 862 struct ring_buffer_event *rbe, void *key, 863 struct action_data *data, u64 *var_ref_vals) 864 { 865 struct synth_event *event = data->synth_event; 866 867 trace_synth(event, var_ref_vals, data->var_ref_idx); 868 } 869 870 struct hist_var_data { 871 struct list_head list; 872 struct hist_trigger_data *hist_data; 873 }; 874 875 static u64 hist_field_timestamp(struct hist_field *hist_field, 876 struct tracing_map_elt *elt, 877 struct trace_buffer *buffer, 878 struct ring_buffer_event *rbe, 879 void *event) 880 { 881 struct hist_trigger_data *hist_data = hist_field->hist_data; 882 struct trace_array *tr = hist_data->event_file->tr; 883 884 u64 ts = tracing_event_time_stamp(buffer, rbe); 885 886 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 887 ts = ns2usecs(ts); 888 889 return ts; 890 } 891 892 static u64 hist_field_cpu(struct hist_field *hist_field, 893 struct tracing_map_elt *elt, 894 struct trace_buffer *buffer, 895 struct ring_buffer_event *rbe, 896 void *event) 897 { 898 int cpu = smp_processor_id(); 899 900 return cpu; 901 } 902 903 static u64 hist_field_comm(struct hist_field *hist_field, 904 struct tracing_map_elt *elt, 905 struct trace_buffer *buffer, 906 struct ring_buffer_event *rbe, 907 void *event) 908 { 909 return (u64)(unsigned long)current->comm; 910 } 911 912 /** 913 * check_field_for_var_ref - Check if a VAR_REF field references a variable 914 * @hist_field: The VAR_REF field to check 915 * @var_data: The hist trigger that owns the variable 916 * @var_idx: The trigger variable identifier 917 * 918 * Check the given VAR_REF field to see whether or not it references 919 * the given variable associated with the given trigger. 920 * 921 * Return: The VAR_REF field if it does reference the variable, NULL if not 922 */ 923 static struct hist_field * 924 check_field_for_var_ref(struct hist_field *hist_field, 925 struct hist_trigger_data *var_data, 926 unsigned int var_idx) 927 { 928 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 929 930 if (hist_field && hist_field->var.idx == var_idx && 931 hist_field->var.hist_data == var_data) 932 return hist_field; 933 934 return NULL; 935 } 936 937 /** 938 * find_var_ref - Check if a trigger has a reference to a trigger variable 939 * @hist_data: The hist trigger that might have a reference to the variable 940 * @var_data: The hist trigger that owns the variable 941 * @var_idx: The trigger variable identifier 942 * 943 * Check the list of var_refs[] on the first hist trigger to see 944 * whether any of them are references to the variable on the second 945 * trigger. 946 * 947 * Return: The VAR_REF field referencing the variable if so, NULL if not 948 */ 949 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 950 struct hist_trigger_data *var_data, 951 unsigned int var_idx) 952 { 953 struct hist_field *hist_field; 954 unsigned int i; 955 956 for (i = 0; i < hist_data->n_var_refs; i++) { 957 hist_field = hist_data->var_refs[i]; 958 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 959 return hist_field; 960 } 961 962 return NULL; 963 } 964 965 /** 966 * find_any_var_ref - Check if there is a reference to a given trigger variable 967 * @hist_data: The hist trigger 968 * @var_idx: The trigger variable identifier 969 * 970 * Check to see whether the given variable is currently referenced by 971 * any other trigger. 972 * 973 * The trigger the variable is defined on is explicitly excluded - the 974 * assumption being that a self-reference doesn't prevent a trigger 975 * from being removed. 976 * 977 * Return: The VAR_REF field referencing the variable if so, NULL if not 978 */ 979 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 980 unsigned int var_idx) 981 { 982 struct trace_array *tr = hist_data->event_file->tr; 983 struct hist_field *found = NULL; 984 struct hist_var_data *var_data; 985 986 list_for_each_entry(var_data, &tr->hist_vars, list) { 987 if (var_data->hist_data == hist_data) 988 continue; 989 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 990 if (found) 991 break; 992 } 993 994 return found; 995 } 996 997 /** 998 * check_var_refs - Check if there is a reference to any of trigger's variables 999 * @hist_data: The hist trigger 1000 * 1001 * A trigger can define one or more variables. If any one of them is 1002 * currently referenced by any other trigger, this function will 1003 * determine that. 1004 * 1005 * Typically used to determine whether or not a trigger can be removed 1006 * - if there are any references to a trigger's variables, it cannot. 1007 * 1008 * Return: True if there is a reference to any of trigger's variables 1009 */ 1010 static bool check_var_refs(struct hist_trigger_data *hist_data) 1011 { 1012 struct hist_field *field; 1013 bool found = false; 1014 int i; 1015 1016 for_each_hist_field(i, hist_data) { 1017 field = hist_data->fields[i]; 1018 if (field && field->flags & HIST_FIELD_FL_VAR) { 1019 if (find_any_var_ref(hist_data, field->var.idx)) { 1020 found = true; 1021 break; 1022 } 1023 } 1024 } 1025 1026 return found; 1027 } 1028 1029 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1030 { 1031 struct trace_array *tr = hist_data->event_file->tr; 1032 struct hist_var_data *var_data, *found = NULL; 1033 1034 list_for_each_entry(var_data, &tr->hist_vars, list) { 1035 if (var_data->hist_data == hist_data) { 1036 found = var_data; 1037 break; 1038 } 1039 } 1040 1041 return found; 1042 } 1043 1044 static bool field_has_hist_vars(struct hist_field *hist_field, 1045 unsigned int level) 1046 { 1047 int i; 1048 1049 if (level > 3) 1050 return false; 1051 1052 if (!hist_field) 1053 return false; 1054 1055 if (hist_field->flags & HIST_FIELD_FL_VAR || 1056 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1057 return true; 1058 1059 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1060 struct hist_field *operand; 1061 1062 operand = hist_field->operands[i]; 1063 if (field_has_hist_vars(operand, level + 1)) 1064 return true; 1065 } 1066 1067 return false; 1068 } 1069 1070 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1071 { 1072 struct hist_field *hist_field; 1073 int i; 1074 1075 for_each_hist_field(i, hist_data) { 1076 hist_field = hist_data->fields[i]; 1077 if (field_has_hist_vars(hist_field, 0)) 1078 return true; 1079 } 1080 1081 return false; 1082 } 1083 1084 static int save_hist_vars(struct hist_trigger_data *hist_data) 1085 { 1086 struct trace_array *tr = hist_data->event_file->tr; 1087 struct hist_var_data *var_data; 1088 1089 var_data = find_hist_vars(hist_data); 1090 if (var_data) 1091 return 0; 1092 1093 if (tracing_check_open_get_tr(tr)) 1094 return -ENODEV; 1095 1096 var_data = kzalloc_obj(*var_data); 1097 if (!var_data) { 1098 trace_array_put(tr); 1099 return -ENOMEM; 1100 } 1101 1102 var_data->hist_data = hist_data; 1103 list_add(&var_data->list, &tr->hist_vars); 1104 1105 return 0; 1106 } 1107 1108 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1109 { 1110 struct trace_array *tr = hist_data->event_file->tr; 1111 struct hist_var_data *var_data; 1112 1113 var_data = find_hist_vars(hist_data); 1114 if (!var_data) 1115 return; 1116 1117 if (WARN_ON(check_var_refs(hist_data))) 1118 return; 1119 1120 list_del(&var_data->list); 1121 1122 kfree(var_data); 1123 1124 trace_array_put(tr); 1125 } 1126 1127 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1128 const char *var_name) 1129 { 1130 struct hist_field *hist_field, *found = NULL; 1131 int i; 1132 1133 for_each_hist_field(i, hist_data) { 1134 hist_field = hist_data->fields[i]; 1135 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1136 strcmp(hist_field->var.name, var_name) == 0) { 1137 found = hist_field; 1138 break; 1139 } 1140 } 1141 1142 return found; 1143 } 1144 1145 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1146 struct trace_event_file *file, 1147 const char *var_name) 1148 { 1149 struct hist_trigger_data *test_data; 1150 struct event_trigger_data *test; 1151 struct hist_field *hist_field; 1152 1153 lockdep_assert_held(&event_mutex); 1154 1155 hist_field = find_var_field(hist_data, var_name); 1156 if (hist_field) 1157 return hist_field; 1158 1159 list_for_each_entry(test, &file->triggers, list) { 1160 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1161 test_data = test->private_data; 1162 hist_field = find_var_field(test_data, var_name); 1163 if (hist_field) 1164 return hist_field; 1165 } 1166 } 1167 1168 return NULL; 1169 } 1170 1171 static struct trace_event_file *find_var_file(struct trace_array *tr, 1172 char *system, 1173 char *event_name, 1174 char *var_name) 1175 { 1176 struct hist_trigger_data *var_hist_data; 1177 struct hist_var_data *var_data; 1178 struct trace_event_file *file, *found = NULL; 1179 1180 if (system) 1181 return find_event_file(tr, system, event_name); 1182 1183 list_for_each_entry(var_data, &tr->hist_vars, list) { 1184 var_hist_data = var_data->hist_data; 1185 file = var_hist_data->event_file; 1186 if (file == found) 1187 continue; 1188 1189 if (find_var_field(var_hist_data, var_name)) { 1190 if (found) { 1191 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1192 return NULL; 1193 } 1194 1195 found = file; 1196 } 1197 } 1198 1199 return found; 1200 } 1201 1202 static struct hist_field *find_file_var(struct trace_event_file *file, 1203 const char *var_name) 1204 { 1205 struct hist_trigger_data *test_data; 1206 struct event_trigger_data *test; 1207 struct hist_field *hist_field; 1208 1209 lockdep_assert_held(&event_mutex); 1210 1211 list_for_each_entry(test, &file->triggers, list) { 1212 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1213 test_data = test->private_data; 1214 hist_field = find_var_field(test_data, var_name); 1215 if (hist_field) 1216 return hist_field; 1217 } 1218 } 1219 1220 return NULL; 1221 } 1222 1223 static struct hist_field * 1224 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1225 { 1226 struct trace_array *tr = hist_data->event_file->tr; 1227 struct hist_field *hist_field, *found = NULL; 1228 struct trace_event_file *file; 1229 unsigned int i; 1230 1231 for (i = 0; i < hist_data->n_actions; i++) { 1232 struct action_data *data = hist_data->actions[i]; 1233 1234 if (data->handler == HANDLER_ONMATCH) { 1235 char *system = data->match_data.event_system; 1236 char *event_name = data->match_data.event; 1237 1238 file = find_var_file(tr, system, event_name, var_name); 1239 if (!file) 1240 continue; 1241 hist_field = find_file_var(file, var_name); 1242 if (hist_field) { 1243 if (found) { 1244 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1245 errpos(var_name)); 1246 return ERR_PTR(-EINVAL); 1247 } 1248 1249 found = hist_field; 1250 } 1251 } 1252 } 1253 return found; 1254 } 1255 1256 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1257 char *system, 1258 char *event_name, 1259 char *var_name) 1260 { 1261 struct trace_array *tr = hist_data->event_file->tr; 1262 struct hist_field *hist_field = NULL; 1263 struct trace_event_file *file; 1264 1265 if (!system || !event_name) { 1266 hist_field = find_match_var(hist_data, var_name); 1267 if (IS_ERR(hist_field)) 1268 return NULL; 1269 if (hist_field) 1270 return hist_field; 1271 } 1272 1273 file = find_var_file(tr, system, event_name, var_name); 1274 if (!file) 1275 return NULL; 1276 1277 hist_field = find_file_var(file, var_name); 1278 1279 return hist_field; 1280 } 1281 1282 static u64 hist_field_var_ref(struct hist_field *hist_field, 1283 struct tracing_map_elt *elt, 1284 struct trace_buffer *buffer, 1285 struct ring_buffer_event *rbe, 1286 void *event) 1287 { 1288 struct hist_elt_data *elt_data; 1289 u64 var_val = 0; 1290 1291 if (WARN_ON_ONCE(!elt)) 1292 return var_val; 1293 1294 elt_data = elt->private_data; 1295 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1296 1297 return var_val; 1298 } 1299 1300 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1301 u64 *var_ref_vals, bool self) 1302 { 1303 struct hist_trigger_data *var_data; 1304 struct tracing_map_elt *var_elt; 1305 struct hist_field *hist_field; 1306 unsigned int i, var_idx; 1307 bool resolved = true; 1308 u64 var_val = 0; 1309 1310 for (i = 0; i < hist_data->n_var_refs; i++) { 1311 hist_field = hist_data->var_refs[i]; 1312 var_idx = hist_field->var.idx; 1313 var_data = hist_field->var.hist_data; 1314 1315 if (var_data == NULL) { 1316 resolved = false; 1317 break; 1318 } 1319 1320 if ((self && var_data != hist_data) || 1321 (!self && var_data == hist_data)) 1322 continue; 1323 1324 var_elt = tracing_map_lookup(var_data->map, key); 1325 if (!var_elt) { 1326 resolved = false; 1327 break; 1328 } 1329 1330 if (!tracing_map_var_set(var_elt, var_idx)) { 1331 resolved = false; 1332 break; 1333 } 1334 1335 if (self || !hist_field->read_once) 1336 var_val = tracing_map_read_var(var_elt, var_idx); 1337 else 1338 var_val = tracing_map_read_var_once(var_elt, var_idx); 1339 1340 var_ref_vals[i] = var_val; 1341 } 1342 1343 return resolved; 1344 } 1345 1346 static const char *hist_field_name(struct hist_field *field, 1347 unsigned int level) 1348 { 1349 const char *field_name = ""; 1350 1351 if (WARN_ON_ONCE(!field)) 1352 return field_name; 1353 1354 if (level > 1) 1355 return field_name; 1356 1357 if (field->field) 1358 field_name = field->field->name; 1359 else if (field->flags & HIST_FIELD_FL_LOG2 || 1360 field->flags & HIST_FIELD_FL_ALIAS || 1361 field->flags & HIST_FIELD_FL_BUCKET) 1362 field_name = hist_field_name(field->operands[0], ++level); 1363 else if (field->flags & HIST_FIELD_FL_CPU) 1364 field_name = "common_cpu"; 1365 else if (field->flags & HIST_FIELD_FL_COMM) 1366 field_name = "common_comm"; 1367 else if (field->flags & HIST_FIELD_FL_EXPR || 1368 field->flags & HIST_FIELD_FL_VAR_REF) { 1369 if (field->system) { 1370 static char full_name[MAX_FILTER_STR_VAL]; 1371 static char *fmt; 1372 int len; 1373 1374 fmt = field->flags & HIST_FIELD_FL_VAR_REF ? "%s.%s.$%s" : "%s.%s.%s"; 1375 1376 len = snprintf(full_name, sizeof(full_name), fmt, 1377 field->system, field->event_name, 1378 field->name); 1379 if (len < sizeof(full_name)) 1380 field_name = full_name; 1381 } else 1382 field_name = field->name; 1383 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1384 field_name = "common_timestamp"; 1385 else if (field->flags & HIST_FIELD_FL_STACKTRACE) { 1386 field_name = "common_stacktrace"; 1387 } else if (field->flags & HIST_FIELD_FL_HITCOUNT) 1388 field_name = "hitcount"; 1389 1390 if (field_name == NULL) 1391 field_name = ""; 1392 1393 return field_name; 1394 } 1395 1396 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed) 1397 { 1398 switch (field_size) { 1399 case 8: 1400 if (field_is_signed) 1401 return HIST_FIELD_FN_S64; 1402 else 1403 return HIST_FIELD_FN_U64; 1404 case 4: 1405 if (field_is_signed) 1406 return HIST_FIELD_FN_S32; 1407 else 1408 return HIST_FIELD_FN_U32; 1409 case 2: 1410 if (field_is_signed) 1411 return HIST_FIELD_FN_S16; 1412 else 1413 return HIST_FIELD_FN_U16; 1414 case 1: 1415 if (field_is_signed) 1416 return HIST_FIELD_FN_S8; 1417 else 1418 return HIST_FIELD_FN_U8; 1419 } 1420 1421 return HIST_FIELD_FN_NOP; 1422 } 1423 1424 static int parse_map_size(char *str) 1425 { 1426 unsigned long size, map_bits; 1427 int ret; 1428 1429 ret = kstrtoul(str, 0, &size); 1430 if (ret) 1431 goto out; 1432 1433 map_bits = ilog2(roundup_pow_of_two(size)); 1434 if (map_bits < TRACING_MAP_BITS_MIN || 1435 map_bits > TRACING_MAP_BITS_MAX) 1436 ret = -EINVAL; 1437 else 1438 ret = map_bits; 1439 out: 1440 return ret; 1441 } 1442 1443 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1444 { 1445 unsigned int i; 1446 1447 if (!attrs) 1448 return; 1449 1450 for (i = 0; i < attrs->n_assignments; i++) 1451 kfree(attrs->assignment_str[i]); 1452 1453 for (i = 0; i < attrs->n_actions; i++) 1454 kfree(attrs->action_str[i]); 1455 1456 kfree(attrs->name); 1457 kfree(attrs->sort_key_str); 1458 kfree(attrs->keys_str); 1459 kfree(attrs->vals_str); 1460 kfree(attrs->clock); 1461 kfree(attrs); 1462 } 1463 1464 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1465 { 1466 int ret = -EINVAL; 1467 1468 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1469 return ret; 1470 1471 if ((str_has_prefix(str, "onmatch(")) || 1472 (str_has_prefix(str, "onmax(")) || 1473 (str_has_prefix(str, "onchange("))) { 1474 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1475 if (!attrs->action_str[attrs->n_actions]) { 1476 ret = -ENOMEM; 1477 return ret; 1478 } 1479 attrs->n_actions++; 1480 ret = 0; 1481 } 1482 return ret; 1483 } 1484 1485 static int parse_assignment(struct trace_array *tr, 1486 char *str, struct hist_trigger_attrs *attrs) 1487 { 1488 int len, ret = 0; 1489 1490 if ((len = str_has_prefix(str, "key=")) || 1491 (len = str_has_prefix(str, "keys="))) { 1492 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1493 if (!attrs->keys_str) { 1494 ret = -ENOMEM; 1495 goto out; 1496 } 1497 } else if ((len = str_has_prefix(str, "val=")) || 1498 (len = str_has_prefix(str, "vals=")) || 1499 (len = str_has_prefix(str, "values="))) { 1500 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1501 if (!attrs->vals_str) { 1502 ret = -ENOMEM; 1503 goto out; 1504 } 1505 } else if ((len = str_has_prefix(str, "sort="))) { 1506 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1507 if (!attrs->sort_key_str) { 1508 ret = -ENOMEM; 1509 goto out; 1510 } 1511 } else if (str_has_prefix(str, "name=")) { 1512 attrs->name = kstrdup(str, GFP_KERNEL); 1513 if (!attrs->name) { 1514 ret = -ENOMEM; 1515 goto out; 1516 } 1517 } else if ((len = str_has_prefix(str, "clock="))) { 1518 str += len; 1519 1520 str = strstrip(str); 1521 attrs->clock = kstrdup(str, GFP_KERNEL); 1522 if (!attrs->clock) { 1523 ret = -ENOMEM; 1524 goto out; 1525 } 1526 } else if ((len = str_has_prefix(str, "size="))) { 1527 int map_bits = parse_map_size(str + len); 1528 1529 if (map_bits < 0) { 1530 ret = map_bits; 1531 goto out; 1532 } 1533 attrs->map_bits = map_bits; 1534 } else { 1535 char *assignment; 1536 1537 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1538 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1539 ret = -EINVAL; 1540 goto out; 1541 } 1542 1543 assignment = kstrdup(str, GFP_KERNEL); 1544 if (!assignment) { 1545 ret = -ENOMEM; 1546 goto out; 1547 } 1548 1549 attrs->assignment_str[attrs->n_assignments++] = assignment; 1550 } 1551 out: 1552 return ret; 1553 } 1554 1555 static struct hist_trigger_attrs * 1556 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1557 { 1558 struct hist_trigger_attrs *attrs; 1559 int ret = 0; 1560 1561 attrs = kzalloc_obj(*attrs); 1562 if (!attrs) 1563 return ERR_PTR(-ENOMEM); 1564 1565 while (trigger_str) { 1566 char *str = strsep(&trigger_str, ":"); 1567 char *rhs; 1568 1569 rhs = strchr(str, '='); 1570 if (rhs) { 1571 if (!strlen(++rhs)) { 1572 ret = -EINVAL; 1573 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1574 goto free; 1575 } 1576 ret = parse_assignment(tr, str, attrs); 1577 if (ret) 1578 goto free; 1579 } else if (strcmp(str, "nohitcount") == 0 || 1580 strcmp(str, "NOHC") == 0) 1581 attrs->no_hitcount = true; 1582 else if (strcmp(str, "pause") == 0) 1583 attrs->pause = true; 1584 else if ((strcmp(str, "cont") == 0) || 1585 (strcmp(str, "continue") == 0)) 1586 attrs->cont = true; 1587 else if (strcmp(str, "clear") == 0) 1588 attrs->clear = true; 1589 else { 1590 ret = parse_action(str, attrs); 1591 if (ret) 1592 goto free; 1593 } 1594 } 1595 1596 if (!attrs->keys_str) { 1597 ret = -EINVAL; 1598 goto free; 1599 } 1600 1601 if (!attrs->clock) { 1602 attrs->clock = kstrdup("global", GFP_KERNEL); 1603 if (!attrs->clock) { 1604 ret = -ENOMEM; 1605 goto free; 1606 } 1607 } 1608 1609 return attrs; 1610 free: 1611 destroy_hist_trigger_attrs(attrs); 1612 1613 return ERR_PTR(ret); 1614 } 1615 1616 static inline void save_comm(char *comm, struct task_struct *task) 1617 { 1618 if (!task->pid) { 1619 strcpy(comm, "<idle>"); 1620 return; 1621 } 1622 1623 if (WARN_ON_ONCE(task->pid < 0)) { 1624 strcpy(comm, "<XXX>"); 1625 return; 1626 } 1627 1628 strscpy(comm, task->comm, TASK_COMM_LEN); 1629 } 1630 1631 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1632 { 1633 unsigned int i; 1634 1635 for (i = 0; i < elt_data->n_field_var_str; i++) 1636 kfree(elt_data->field_var_str[i]); 1637 1638 kfree(elt_data->comm); 1639 kfree(elt_data); 1640 } 1641 1642 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1643 { 1644 struct hist_elt_data *elt_data = elt->private_data; 1645 1646 hist_elt_data_free(elt_data); 1647 } 1648 1649 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1650 { 1651 struct hist_trigger_data *hist_data = elt->map->private_data; 1652 unsigned int size = TASK_COMM_LEN; 1653 struct hist_elt_data *elt_data; 1654 struct hist_field *hist_field; 1655 unsigned int i, n_str; 1656 1657 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); 1658 1659 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + 1660 hist_data->n_var_str; 1661 if (n_str > SYNTH_FIELDS_MAX) 1662 return -EINVAL; 1663 1664 elt_data = kzalloc_flex(*elt_data, field_var_str, n_str); 1665 if (!elt_data) 1666 return -ENOMEM; 1667 1668 elt_data->n_field_var_str = n_str; 1669 1670 for_each_hist_field(i, hist_data) { 1671 hist_field = hist_data->fields[i]; 1672 1673 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) { 1674 elt_data->comm = kzalloc(size, GFP_KERNEL); 1675 if (!elt_data->comm) { 1676 kfree(elt_data); 1677 return -ENOMEM; 1678 } 1679 break; 1680 } 1681 } 1682 1683 size = STR_VAR_LEN_MAX; 1684 1685 for (i = 0; i < n_str; i++) { 1686 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1687 if (!elt_data->field_var_str[i]) { 1688 hist_elt_data_free(elt_data); 1689 return -ENOMEM; 1690 } 1691 } 1692 1693 elt->private_data = elt_data; 1694 1695 return 0; 1696 } 1697 1698 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1699 { 1700 struct hist_elt_data *elt_data = elt->private_data; 1701 1702 if (elt_data->comm) 1703 save_comm(elt_data->comm, current); 1704 } 1705 1706 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1707 .elt_alloc = hist_trigger_elt_data_alloc, 1708 .elt_free = hist_trigger_elt_data_free, 1709 .elt_init = hist_trigger_elt_data_init, 1710 }; 1711 1712 static const char *get_hist_field_flags(struct hist_field *hist_field) 1713 { 1714 const char *flags_str = NULL; 1715 1716 if (hist_field->flags & HIST_FIELD_FL_HEX) 1717 flags_str = "hex"; 1718 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1719 flags_str = "sym"; 1720 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1721 flags_str = "sym-offset"; 1722 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1723 flags_str = "execname"; 1724 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1725 flags_str = "syscall"; 1726 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1727 flags_str = "log2"; 1728 else if (hist_field->flags & HIST_FIELD_FL_BUCKET) 1729 flags_str = "buckets"; 1730 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1731 flags_str = "usecs"; 1732 else if (hist_field->flags & HIST_FIELD_FL_PERCENT) 1733 flags_str = "percent"; 1734 else if (hist_field->flags & HIST_FIELD_FL_GRAPH) 1735 flags_str = "graph"; 1736 else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE && hist_field->field) 1737 flags_str = "stacktrace"; 1738 1739 return flags_str; 1740 } 1741 1742 static bool expr_field_str(struct hist_field *field, struct seq_buf *s) 1743 { 1744 const char *field_name; 1745 1746 if (field->flags & HIST_FIELD_FL_VAR_REF) { 1747 if (!field->system) 1748 seq_buf_putc(s, '$'); 1749 } else if (field->flags & HIST_FIELD_FL_CONST) 1750 seq_buf_printf(s, "%llu", field->constant); 1751 1752 field_name = hist_field_name(field, 0); 1753 if (!field_name) 1754 return false; 1755 1756 seq_buf_puts(s, field_name); 1757 1758 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1759 const char *flags_str = get_hist_field_flags(field); 1760 1761 if (flags_str) 1762 seq_buf_printf(s, ".%s", flags_str); 1763 } 1764 1765 if (field->buckets) 1766 seq_buf_printf(s, "=%lu", field->buckets); 1767 1768 return !seq_buf_has_overflowed(s); 1769 } 1770 1771 static char *expr_str(struct hist_field *field, unsigned int level) 1772 { 1773 char *expr __free(kfree) = NULL; 1774 struct seq_buf s; 1775 1776 if (level > 1) 1777 return ERR_PTR(-EINVAL); 1778 1779 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1780 if (!expr) 1781 return ERR_PTR(-ENOMEM); 1782 1783 seq_buf_init(&s, expr, MAX_FILTER_STR_VAL); 1784 1785 if (!field->operands[0]) { 1786 if (!expr_field_str(field, &s)) 1787 return ERR_PTR(-E2BIG); 1788 1789 return_ptr(expr); 1790 } 1791 1792 if (field->operator == FIELD_OP_UNARY_MINUS) { 1793 char *subexpr; 1794 1795 subexpr = expr_str(field->operands[0], ++level); 1796 if (IS_ERR(subexpr)) 1797 return subexpr; 1798 1799 seq_buf_printf(&s, "-(%s)", subexpr); 1800 kfree(subexpr); 1801 1802 if (seq_buf_has_overflowed(&s)) 1803 return ERR_PTR(-E2BIG); 1804 1805 return_ptr(expr); 1806 } 1807 1808 if (!expr_field_str(field->operands[0], &s)) 1809 return ERR_PTR(-E2BIG); 1810 1811 switch (field->operator) { 1812 case FIELD_OP_MINUS: 1813 seq_buf_putc(&s, '-'); 1814 break; 1815 case FIELD_OP_PLUS: 1816 seq_buf_putc(&s, '+'); 1817 break; 1818 case FIELD_OP_DIV: 1819 seq_buf_putc(&s, '/'); 1820 break; 1821 case FIELD_OP_MULT: 1822 seq_buf_putc(&s, '*'); 1823 break; 1824 default: 1825 return ERR_PTR(-EINVAL); 1826 } 1827 1828 if (seq_buf_has_overflowed(&s) || 1829 !expr_field_str(field->operands[1], &s)) 1830 return ERR_PTR(-E2BIG); 1831 1832 return_ptr(expr); 1833 } 1834 1835 /* 1836 * If field_op != FIELD_OP_NONE, *sep points to the root operator 1837 * of the expression tree to be evaluated. 1838 */ 1839 static int contains_operator(char *str, char **sep) 1840 { 1841 enum field_op_id field_op = FIELD_OP_NONE; 1842 char *minus_op, *plus_op, *div_op, *mult_op; 1843 1844 1845 /* 1846 * Report the last occurrence of the operators first, so that the 1847 * expression is evaluated left to right. This is important since 1848 * subtraction and division are not associative. 1849 * 1850 * e.g 1851 * 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2 1852 * 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2 1853 */ 1854 1855 /* 1856 * First, find lower precedence addition and subtraction 1857 * since the expression will be evaluated recursively. 1858 */ 1859 minus_op = strrchr(str, '-'); 1860 if (minus_op) { 1861 /* 1862 * Unary minus is not supported in sub-expressions. If 1863 * present, it is always the next root operator. 1864 */ 1865 if (minus_op == str) { 1866 field_op = FIELD_OP_UNARY_MINUS; 1867 goto out; 1868 } 1869 1870 field_op = FIELD_OP_MINUS; 1871 } 1872 1873 plus_op = strrchr(str, '+'); 1874 if (plus_op || minus_op) { 1875 /* 1876 * For operators of the same precedence use to rightmost as the 1877 * root, so that the expression is evaluated left to right. 1878 */ 1879 if (plus_op > minus_op) 1880 field_op = FIELD_OP_PLUS; 1881 goto out; 1882 } 1883 1884 /* 1885 * Multiplication and division have higher precedence than addition and 1886 * subtraction. 1887 */ 1888 div_op = strrchr(str, '/'); 1889 if (div_op) 1890 field_op = FIELD_OP_DIV; 1891 1892 mult_op = strrchr(str, '*'); 1893 /* 1894 * For operators of the same precedence use to rightmost as the 1895 * root, so that the expression is evaluated left to right. 1896 */ 1897 if (mult_op > div_op) 1898 field_op = FIELD_OP_MULT; 1899 1900 out: 1901 if (sep) { 1902 switch (field_op) { 1903 case FIELD_OP_UNARY_MINUS: 1904 case FIELD_OP_MINUS: 1905 *sep = minus_op; 1906 break; 1907 case FIELD_OP_PLUS: 1908 *sep = plus_op; 1909 break; 1910 case FIELD_OP_DIV: 1911 *sep = div_op; 1912 break; 1913 case FIELD_OP_MULT: 1914 *sep = mult_op; 1915 break; 1916 case FIELD_OP_NONE: 1917 default: 1918 *sep = NULL; 1919 break; 1920 } 1921 } 1922 1923 return field_op; 1924 } 1925 1926 static void __destroy_hist_field(struct hist_field *hist_field) 1927 { 1928 kfree(hist_field->var.name); 1929 kfree(hist_field->name); 1930 1931 /* Can likely be a const */ 1932 kfree_const(hist_field->type); 1933 1934 kfree(hist_field->system); 1935 kfree(hist_field->event_name); 1936 1937 kfree(hist_field); 1938 } 1939 1940 static void destroy_hist_field(struct hist_field *hist_field, 1941 unsigned int level) 1942 { 1943 unsigned int i; 1944 1945 if (level > 3) 1946 return; 1947 1948 if (!hist_field) 1949 return; 1950 1951 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1952 return; /* var refs will be destroyed separately */ 1953 1954 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1955 destroy_hist_field(hist_field->operands[i], level + 1); 1956 1957 __destroy_hist_field(hist_field); 1958 } 1959 1960 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1961 struct ftrace_event_field *field, 1962 unsigned long flags, 1963 char *var_name) 1964 { 1965 struct hist_field *hist_field; 1966 1967 if (field && is_function_field(field)) 1968 return NULL; 1969 1970 hist_field = kzalloc_obj(struct hist_field); 1971 if (!hist_field) 1972 return NULL; 1973 1974 hist_field->hist_data = hist_data; 1975 1976 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1977 goto out; /* caller will populate */ 1978 1979 if (flags & HIST_FIELD_FL_VAR_REF) { 1980 hist_field->fn_num = HIST_FIELD_FN_VAR_REF; 1981 goto out; 1982 } 1983 1984 if (flags & HIST_FIELD_FL_HITCOUNT) { 1985 hist_field->fn_num = HIST_FIELD_FN_COUNTER; 1986 hist_field->size = sizeof(u64); 1987 hist_field->type = "u64"; 1988 goto out; 1989 } 1990 1991 if (flags & HIST_FIELD_FL_CONST) { 1992 hist_field->fn_num = HIST_FIELD_FN_CONST; 1993 hist_field->size = sizeof(u64); 1994 hist_field->type = "u64"; 1995 goto out; 1996 } 1997 1998 if (flags & HIST_FIELD_FL_STACKTRACE) { 1999 if (field) 2000 hist_field->fn_num = HIST_FIELD_FN_STACK; 2001 else 2002 hist_field->fn_num = HIST_FIELD_FN_NOP; 2003 hist_field->size = HIST_STACKTRACE_SIZE; 2004 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL); 2005 if (!hist_field->type) 2006 goto free; 2007 goto out; 2008 } 2009 2010 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) { 2011 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET); 2012 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 : 2013 HIST_FIELD_FN_BUCKET; 2014 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2015 if (!hist_field->operands[0]) 2016 goto free; 2017 hist_field->size = hist_field->operands[0]->size; 2018 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL); 2019 if (!hist_field->type) 2020 goto free; 2021 goto out; 2022 } 2023 2024 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2025 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP; 2026 hist_field->size = sizeof(u64); 2027 hist_field->type = "u64"; 2028 goto out; 2029 } 2030 2031 if (flags & HIST_FIELD_FL_CPU) { 2032 hist_field->fn_num = HIST_FIELD_FN_CPU; 2033 hist_field->size = sizeof(int); 2034 hist_field->type = "unsigned int"; 2035 goto out; 2036 } 2037 2038 if (flags & HIST_FIELD_FL_COMM) { 2039 hist_field->fn_num = HIST_FIELD_FN_COMM; 2040 hist_field->size = MAX_FILTER_STR_VAL; 2041 hist_field->type = "char[]"; 2042 goto out; 2043 } 2044 2045 if (WARN_ON_ONCE(!field)) 2046 goto out; 2047 2048 /* Pointers to strings are just pointers and dangerous to dereference */ 2049 if (is_string_field(field) && 2050 (field->filter_type != FILTER_PTR_STRING)) { 2051 flags |= HIST_FIELD_FL_STRING; 2052 2053 hist_field->size = MAX_FILTER_STR_VAL; 2054 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2055 if (!hist_field->type) 2056 goto free; 2057 2058 if (field->filter_type == FILTER_STATIC_STRING) { 2059 hist_field->fn_num = HIST_FIELD_FN_STRING; 2060 hist_field->size = field->size; 2061 } else if (field->filter_type == FILTER_DYN_STRING) { 2062 hist_field->fn_num = HIST_FIELD_FN_DYNSTRING; 2063 } else if (field->filter_type == FILTER_RDYN_STRING) 2064 hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; 2065 else 2066 hist_field->fn_num = HIST_FIELD_FN_PSTRING; 2067 } else if (field->filter_type == FILTER_STACKTRACE) { 2068 flags |= HIST_FIELD_FL_STACKTRACE; 2069 2070 hist_field->size = MAX_FILTER_STR_VAL; 2071 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2072 if (!hist_field->type) 2073 goto free; 2074 2075 hist_field->fn_num = HIST_FIELD_FN_STACK; 2076 } else { 2077 hist_field->size = field->size; 2078 hist_field->is_signed = field->is_signed; 2079 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2080 if (!hist_field->type) 2081 goto free; 2082 2083 hist_field->fn_num = select_value_fn(field->size, 2084 field->is_signed); 2085 if (hist_field->fn_num == HIST_FIELD_FN_NOP) { 2086 destroy_hist_field(hist_field, 0); 2087 return NULL; 2088 } 2089 } 2090 out: 2091 hist_field->field = field; 2092 hist_field->flags = flags; 2093 2094 if (var_name) { 2095 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2096 if (!hist_field->var.name) 2097 goto free; 2098 } 2099 2100 return hist_field; 2101 free: 2102 destroy_hist_field(hist_field, 0); 2103 return NULL; 2104 } 2105 2106 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2107 { 2108 unsigned int i; 2109 2110 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2111 if (hist_data->fields[i]) { 2112 destroy_hist_field(hist_data->fields[i], 0); 2113 hist_data->fields[i] = NULL; 2114 } 2115 } 2116 2117 for (i = 0; i < hist_data->n_var_refs; i++) { 2118 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2119 __destroy_hist_field(hist_data->var_refs[i]); 2120 hist_data->var_refs[i] = NULL; 2121 } 2122 } 2123 2124 static int init_var_ref(struct hist_field *ref_field, 2125 struct hist_field *var_field, 2126 char *system, char *event_name) 2127 { 2128 int err = 0; 2129 2130 ref_field->var.idx = var_field->var.idx; 2131 ref_field->var.hist_data = var_field->hist_data; 2132 ref_field->size = var_field->size; 2133 ref_field->is_signed = var_field->is_signed; 2134 ref_field->flags |= var_field->flags & 2135 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS | 2136 HIST_FIELD_FL_STACKTRACE); 2137 2138 if (system) { 2139 ref_field->system = kstrdup(system, GFP_KERNEL); 2140 if (!ref_field->system) 2141 return -ENOMEM; 2142 } 2143 2144 if (event_name) { 2145 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2146 if (!ref_field->event_name) { 2147 err = -ENOMEM; 2148 goto free; 2149 } 2150 } 2151 2152 if (var_field->var.name) { 2153 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2154 if (!ref_field->name) { 2155 err = -ENOMEM; 2156 goto free; 2157 } 2158 } else if (var_field->name) { 2159 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2160 if (!ref_field->name) { 2161 err = -ENOMEM; 2162 goto free; 2163 } 2164 } 2165 2166 ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL); 2167 if (!ref_field->type) { 2168 err = -ENOMEM; 2169 goto free; 2170 } 2171 out: 2172 return err; 2173 free: 2174 kfree(ref_field->system); 2175 ref_field->system = NULL; 2176 kfree(ref_field->event_name); 2177 ref_field->event_name = NULL; 2178 kfree(ref_field->name); 2179 ref_field->name = NULL; 2180 2181 goto out; 2182 } 2183 2184 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 2185 struct hist_field *var_field) 2186 { 2187 struct hist_field *ref_field; 2188 int i; 2189 2190 for (i = 0; i < hist_data->n_var_refs; i++) { 2191 ref_field = hist_data->var_refs[i]; 2192 if (ref_field->var.idx == var_field->var.idx && 2193 ref_field->var.hist_data == var_field->hist_data) 2194 return i; 2195 } 2196 2197 return -ENOENT; 2198 } 2199 2200 /** 2201 * create_var_ref - Create a variable reference and attach it to trigger 2202 * @hist_data: The trigger that will be referencing the variable 2203 * @var_field: The VAR field to create a reference to 2204 * @system: The optional system string 2205 * @event_name: The optional event_name string 2206 * 2207 * Given a variable hist_field, create a VAR_REF hist_field that 2208 * represents a reference to it. 2209 * 2210 * This function also adds the reference to the trigger that 2211 * now references the variable. 2212 * 2213 * Return: The VAR_REF field if successful, NULL if not 2214 */ 2215 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2216 struct hist_field *var_field, 2217 char *system, char *event_name) 2218 { 2219 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2220 struct hist_field *ref_field; 2221 int i; 2222 2223 /* Check if the variable already exists */ 2224 for (i = 0; i < hist_data->n_var_refs; i++) { 2225 ref_field = hist_data->var_refs[i]; 2226 if (ref_field->var.idx == var_field->var.idx && 2227 ref_field->var.hist_data == var_field->hist_data) 2228 return ref_field; 2229 } 2230 /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ 2231 if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) 2232 return NULL; 2233 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2234 if (ref_field) { 2235 if (init_var_ref(ref_field, var_field, system, event_name)) { 2236 __destroy_hist_field(ref_field); 2237 return NULL; 2238 } 2239 2240 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2241 ref_field->var_ref_idx = hist_data->n_var_refs++; 2242 } 2243 2244 return ref_field; 2245 } 2246 2247 static bool is_var_ref(char *var_name) 2248 { 2249 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2250 return false; 2251 2252 return true; 2253 } 2254 2255 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2256 char *var_name) 2257 { 2258 char *name, *field; 2259 unsigned int i; 2260 2261 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2262 name = hist_data->attrs->var_defs.name[i]; 2263 2264 if (strcmp(var_name, name) == 0) { 2265 field = hist_data->attrs->var_defs.expr[i]; 2266 if (contains_operator(field, NULL) || is_var_ref(field)) 2267 continue; 2268 return field; 2269 } 2270 } 2271 2272 return NULL; 2273 } 2274 2275 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2276 char *system, char *event_name, 2277 char *var_name) 2278 { 2279 struct trace_event_call *call; 2280 2281 if (system && event_name) { 2282 call = hist_data->event_file->event_call; 2283 2284 if (strcmp(system, call->class->system) != 0) 2285 return NULL; 2286 2287 if (strcmp(event_name, trace_event_name(call)) != 0) 2288 return NULL; 2289 } 2290 2291 if (!!system != !!event_name) 2292 return NULL; 2293 2294 if (!is_var_ref(var_name)) 2295 return NULL; 2296 2297 var_name++; 2298 2299 return field_name_from_var(hist_data, var_name); 2300 } 2301 2302 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2303 char *system, char *event_name, 2304 char *var_name) 2305 { 2306 struct hist_field *var_field = NULL, *ref_field = NULL; 2307 struct trace_array *tr = hist_data->event_file->tr; 2308 2309 if (!is_var_ref(var_name)) 2310 return NULL; 2311 2312 var_name++; 2313 2314 var_field = find_event_var(hist_data, system, event_name, var_name); 2315 if (var_field) 2316 ref_field = create_var_ref(hist_data, var_field, 2317 system, event_name); 2318 2319 if (!ref_field) 2320 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2321 2322 return ref_field; 2323 } 2324 2325 static struct ftrace_event_field * 2326 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2327 char *field_str, unsigned long *flags, unsigned long *buckets) 2328 { 2329 struct ftrace_event_field *field = NULL; 2330 char *field_name, *modifier, *str; 2331 struct trace_array *tr = file->tr; 2332 bool stack_modifier = false; 2333 2334 modifier = str = kstrdup(field_str, GFP_KERNEL); 2335 if (!modifier) 2336 return ERR_PTR(-ENOMEM); 2337 2338 field_name = strsep(&modifier, "."); 2339 if (modifier) { 2340 if (strcmp(modifier, "hex") == 0) 2341 *flags |= HIST_FIELD_FL_HEX; 2342 else if (strcmp(modifier, "sym") == 0) 2343 *flags |= HIST_FIELD_FL_SYM; 2344 /* 2345 * 'sym-offset' occurrences in the trigger string are modified 2346 * to 'symXoffset' to simplify arithmetic expression parsing. 2347 */ 2348 else if (strcmp(modifier, "symXoffset") == 0) 2349 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2350 else if ((strcmp(modifier, "execname") == 0) && 2351 (strcmp(field_name, "common_pid") == 0)) 2352 *flags |= HIST_FIELD_FL_EXECNAME; 2353 else if (strcmp(modifier, "syscall") == 0) 2354 *flags |= HIST_FIELD_FL_SYSCALL; 2355 else if (strcmp(modifier, "stacktrace") == 0) { 2356 *flags |= HIST_FIELD_FL_STACKTRACE; 2357 stack_modifier = true; 2358 } else if (strcmp(modifier, "log2") == 0) 2359 *flags |= HIST_FIELD_FL_LOG2; 2360 else if (strcmp(modifier, "usecs") == 0) 2361 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2362 else if (strncmp(modifier, "bucket", 6) == 0) { 2363 int ret; 2364 2365 modifier += 6; 2366 2367 if (*modifier == 's') 2368 modifier++; 2369 if (*modifier != '=') 2370 goto error; 2371 modifier++; 2372 ret = kstrtoul(modifier, 0, buckets); 2373 if (ret || !(*buckets)) 2374 goto error; 2375 *flags |= HIST_FIELD_FL_BUCKET; 2376 } else if (strncmp(modifier, "percent", 7) == 0) { 2377 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2378 goto error; 2379 *flags |= HIST_FIELD_FL_PERCENT; 2380 } else if (strncmp(modifier, "graph", 5) == 0) { 2381 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2382 goto error; 2383 *flags |= HIST_FIELD_FL_GRAPH; 2384 } else { 2385 error: 2386 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2387 field = ERR_PTR(-EINVAL); 2388 goto out; 2389 } 2390 } 2391 2392 if (strcmp(field_name, "common_timestamp") == 0) { 2393 *flags |= HIST_FIELD_FL_TIMESTAMP; 2394 hist_data->enable_timestamps = true; 2395 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2396 hist_data->attrs->ts_in_usecs = true; 2397 } else if (strcmp(field_name, "common_stacktrace") == 0) { 2398 *flags |= HIST_FIELD_FL_STACKTRACE; 2399 } else if (strcmp(field_name, "common_cpu") == 0) { 2400 *flags |= HIST_FIELD_FL_CPU; 2401 } else if (strcmp(field_name, "common_comm") == 0) { 2402 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2403 } else if (strcmp(field_name, "hitcount") == 0) 2404 *flags |= HIST_FIELD_FL_HITCOUNT; 2405 else { 2406 field = trace_find_event_field(file->event_call, field_name); 2407 if (!field || !field->size) { 2408 /* 2409 * For backward compatibility, if field_name 2410 * was "cpu" or "stacktrace", then we treat this 2411 * the same as common_cpu and common_stacktrace 2412 * respectively. This also works for "CPU", and 2413 * "STACKTRACE". 2414 */ 2415 if (field && field->filter_type == FILTER_CPU) { 2416 *flags |= HIST_FIELD_FL_CPU; 2417 } else if (field && field->filter_type == FILTER_STACKTRACE) { 2418 *flags |= HIST_FIELD_FL_STACKTRACE; 2419 field = NULL; 2420 } else if (field && field->filter_type == FILTER_COMM) { 2421 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2422 } else { 2423 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, 2424 errpos(field_name)); 2425 field = ERR_PTR(-EINVAL); 2426 goto out; 2427 } 2428 } 2429 } 2430 2431 if (stack_modifier && 2432 (!field || field->filter_type != FILTER_STACKTRACE)) { 2433 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 2434 field = ERR_PTR(-EINVAL); 2435 } 2436 out: 2437 kfree(str); 2438 2439 return field; 2440 } 2441 2442 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2443 struct hist_field *var_ref, 2444 char *var_name) 2445 { 2446 struct hist_field *alias = NULL; 2447 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2448 2449 alias = create_hist_field(hist_data, NULL, flags, var_name); 2450 if (!alias) 2451 return NULL; 2452 2453 alias->fn_num = var_ref->fn_num; 2454 alias->operands[0] = var_ref; 2455 2456 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2457 destroy_hist_field(alias, 0); 2458 return NULL; 2459 } 2460 2461 alias->var_ref_idx = var_ref->var_ref_idx; 2462 2463 return alias; 2464 } 2465 2466 static struct hist_field *parse_const(struct hist_trigger_data *hist_data, 2467 char *str, char *var_name, 2468 unsigned long *flags) 2469 { 2470 struct trace_array *tr = hist_data->event_file->tr; 2471 struct hist_field *field = NULL; 2472 u64 constant; 2473 2474 if (kstrtoull(str, 0, &constant)) { 2475 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str)); 2476 return NULL; 2477 } 2478 2479 *flags |= HIST_FIELD_FL_CONST; 2480 field = create_hist_field(hist_data, NULL, *flags, var_name); 2481 if (!field) 2482 return NULL; 2483 2484 field->constant = constant; 2485 2486 return field; 2487 } 2488 2489 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2490 struct trace_event_file *file, char *str, 2491 unsigned long *flags, char *var_name) 2492 { 2493 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2494 struct ftrace_event_field *field = NULL; 2495 struct hist_field *hist_field = NULL; 2496 unsigned long buckets = 0; 2497 int ret = 0; 2498 2499 if (isdigit(str[0])) { 2500 hist_field = parse_const(hist_data, str, var_name, flags); 2501 if (!hist_field) { 2502 ret = -EINVAL; 2503 goto out; 2504 } 2505 return hist_field; 2506 } 2507 2508 s = strchr(str, '.'); 2509 if (s) { 2510 s = strchr(++s, '.'); 2511 if (s) { 2512 ref_system = strsep(&str, "."); 2513 if (!str) { 2514 ret = -EINVAL; 2515 goto out; 2516 } 2517 ref_event = strsep(&str, "."); 2518 if (!str) { 2519 ret = -EINVAL; 2520 goto out; 2521 } 2522 ref_var = str; 2523 } 2524 } 2525 2526 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2527 if (!s) { 2528 hist_field = parse_var_ref(hist_data, ref_system, 2529 ref_event, ref_var); 2530 if (hist_field) { 2531 if (var_name) { 2532 hist_field = create_alias(hist_data, hist_field, var_name); 2533 if (!hist_field) { 2534 ret = -ENOMEM; 2535 goto out; 2536 } 2537 } 2538 return hist_field; 2539 } 2540 } else 2541 str = s; 2542 2543 field = parse_field(hist_data, file, str, flags, &buckets); 2544 if (IS_ERR(field)) { 2545 ret = PTR_ERR(field); 2546 goto out; 2547 } 2548 2549 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2550 if (!hist_field) { 2551 ret = -ENOMEM; 2552 goto out; 2553 } 2554 hist_field->buckets = buckets; 2555 2556 return hist_field; 2557 out: 2558 return ERR_PTR(ret); 2559 } 2560 2561 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2562 struct trace_event_file *file, 2563 char *str, unsigned long flags, 2564 char *var_name, unsigned int *n_subexprs); 2565 2566 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2567 struct trace_event_file *file, 2568 char *str, unsigned long flags, 2569 char *var_name, unsigned int *n_subexprs) 2570 { 2571 struct hist_field *operand1, *expr = NULL; 2572 unsigned long operand_flags; 2573 int ret = 0; 2574 char *s; 2575 2576 /* Unary minus operator, increment n_subexprs */ 2577 ++*n_subexprs; 2578 2579 /* we support only -(xxx) i.e. explicit parens required */ 2580 2581 if (*n_subexprs > 3) { 2582 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2583 ret = -EINVAL; 2584 goto free; 2585 } 2586 2587 str++; /* skip leading '-' */ 2588 2589 s = strchr(str, '('); 2590 if (s) 2591 str++; 2592 else { 2593 ret = -EINVAL; 2594 goto free; 2595 } 2596 2597 s = strrchr(str, ')'); 2598 if (s) { 2599 /* unary minus not supported in sub-expressions */ 2600 if (*(s+1) != '\0') { 2601 hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR, 2602 errpos(str)); 2603 ret = -EINVAL; 2604 goto free; 2605 } 2606 *s = '\0'; 2607 } 2608 else { 2609 ret = -EINVAL; /* no closing ')' */ 2610 goto free; 2611 } 2612 2613 flags |= HIST_FIELD_FL_EXPR; 2614 expr = create_hist_field(hist_data, NULL, flags, var_name); 2615 if (!expr) { 2616 ret = -ENOMEM; 2617 goto free; 2618 } 2619 2620 operand_flags = 0; 2621 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2622 if (IS_ERR(operand1)) { 2623 ret = PTR_ERR(operand1); 2624 goto free; 2625 } 2626 if (operand1->flags & HIST_FIELD_FL_STRING) { 2627 /* String type can not be the operand of unary operator. */ 2628 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2629 destroy_hist_field(operand1, 0); 2630 ret = -EINVAL; 2631 goto free; 2632 } 2633 2634 expr->flags |= operand1->flags & 2635 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2636 expr->fn_num = HIST_FIELD_FN_UMINUS; 2637 expr->operands[0] = operand1; 2638 expr->size = operand1->size; 2639 expr->is_signed = operand1->is_signed; 2640 expr->operator = FIELD_OP_UNARY_MINUS; 2641 expr->name = expr_str(expr, 0); 2642 if (IS_ERR(expr->name)) { 2643 ret = PTR_ERR(expr->name); 2644 expr->name = NULL; 2645 goto free; 2646 } 2647 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2648 if (!expr->type) { 2649 ret = -ENOMEM; 2650 goto free; 2651 } 2652 2653 return expr; 2654 free: 2655 destroy_hist_field(expr, 0); 2656 return ERR_PTR(ret); 2657 } 2658 2659 /* 2660 * If the operands are var refs, return pointers the 2661 * variable(s) referenced in var1 and var2, else NULL. 2662 */ 2663 static int check_expr_operands(struct trace_array *tr, 2664 struct hist_field *operand1, 2665 struct hist_field *operand2, 2666 struct hist_field **var1, 2667 struct hist_field **var2) 2668 { 2669 unsigned long operand1_flags = operand1->flags; 2670 unsigned long operand2_flags = operand2->flags; 2671 2672 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2673 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2674 struct hist_field *var; 2675 2676 var = find_var_field(operand1->var.hist_data, operand1->name); 2677 if (!var) 2678 return -EINVAL; 2679 operand1_flags = var->flags; 2680 *var1 = var; 2681 } 2682 2683 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2684 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2685 struct hist_field *var; 2686 2687 var = find_var_field(operand2->var.hist_data, operand2->name); 2688 if (!var) 2689 return -EINVAL; 2690 operand2_flags = var->flags; 2691 *var2 = var; 2692 } 2693 2694 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2695 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2696 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2697 return -EINVAL; 2698 } 2699 2700 return 0; 2701 } 2702 2703 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2704 struct trace_event_file *file, 2705 char *str, unsigned long flags, 2706 char *var_name, unsigned int *n_subexprs) 2707 { 2708 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2709 struct hist_field *var1 = NULL, *var2 = NULL; 2710 unsigned long operand_flags, operand2_flags; 2711 int field_op, ret = -EINVAL; 2712 char *sep, *operand1_str; 2713 enum hist_field_fn op_fn; 2714 bool combine_consts; 2715 2716 if (*n_subexprs > 3) { 2717 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2718 return ERR_PTR(-EINVAL); 2719 } 2720 2721 field_op = contains_operator(str, &sep); 2722 2723 if (field_op == FIELD_OP_NONE) 2724 return parse_atom(hist_data, file, str, &flags, var_name); 2725 2726 if (field_op == FIELD_OP_UNARY_MINUS) 2727 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs); 2728 2729 /* Binary operator found, increment n_subexprs */ 2730 ++*n_subexprs; 2731 2732 /* Split the expression string at the root operator */ 2733 if (!sep) 2734 return ERR_PTR(-EINVAL); 2735 2736 *sep = '\0'; 2737 operand1_str = str; 2738 str = sep+1; 2739 2740 /* Binary operator requires both operands */ 2741 if (*operand1_str == '\0' || *str == '\0') 2742 return ERR_PTR(-EINVAL); 2743 2744 operand_flags = 0; 2745 2746 /* LHS of string is an expression e.g. a+b in a+b+c */ 2747 operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs); 2748 if (IS_ERR(operand1)) 2749 return ERR_CAST(operand1); 2750 2751 if (operand1->flags & HIST_FIELD_FL_STRING) { 2752 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str)); 2753 ret = -EINVAL; 2754 goto free_op1; 2755 } 2756 2757 /* RHS of string is another expression e.g. c in a+b+c */ 2758 operand_flags = 0; 2759 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2760 if (IS_ERR(operand2)) { 2761 ret = PTR_ERR(operand2); 2762 goto free_op1; 2763 } 2764 if (operand2->flags & HIST_FIELD_FL_STRING) { 2765 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2766 ret = -EINVAL; 2767 goto free_operands; 2768 } 2769 2770 switch (field_op) { 2771 case FIELD_OP_MINUS: 2772 op_fn = HIST_FIELD_FN_MINUS; 2773 break; 2774 case FIELD_OP_PLUS: 2775 op_fn = HIST_FIELD_FN_PLUS; 2776 break; 2777 case FIELD_OP_DIV: 2778 op_fn = HIST_FIELD_FN_DIV; 2779 break; 2780 case FIELD_OP_MULT: 2781 op_fn = HIST_FIELD_FN_MULT; 2782 break; 2783 default: 2784 ret = -EINVAL; 2785 goto free_operands; 2786 } 2787 2788 ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2); 2789 if (ret) 2790 goto free_operands; 2791 2792 operand_flags = var1 ? var1->flags : operand1->flags; 2793 operand2_flags = var2 ? var2->flags : operand2->flags; 2794 2795 /* 2796 * If both operands are constant, the expression can be 2797 * collapsed to a single constant. 2798 */ 2799 combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST; 2800 2801 flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR; 2802 2803 flags |= operand1->flags & 2804 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2805 2806 expr = create_hist_field(hist_data, NULL, flags, var_name); 2807 if (!expr) { 2808 ret = -ENOMEM; 2809 goto free_operands; 2810 } 2811 2812 operand1->read_once = true; 2813 operand2->read_once = true; 2814 2815 /* The operands are now owned and free'd by 'expr' */ 2816 expr->operands[0] = operand1; 2817 expr->operands[1] = operand2; 2818 2819 if (field_op == FIELD_OP_DIV && 2820 operand2_flags & HIST_FIELD_FL_CONST) { 2821 u64 divisor = var2 ? var2->constant : operand2->constant; 2822 2823 if (!divisor) { 2824 hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str)); 2825 ret = -EDOM; 2826 goto free_expr; 2827 } 2828 2829 /* 2830 * Copy the divisor here so we don't have to look it up 2831 * later if this is a var ref 2832 */ 2833 operand2->constant = divisor; 2834 op_fn = hist_field_get_div_fn(operand2); 2835 } 2836 2837 expr->fn_num = op_fn; 2838 2839 if (combine_consts) { 2840 if (var1) 2841 expr->operands[0] = var1; 2842 if (var2) 2843 expr->operands[1] = var2; 2844 2845 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL); 2846 expr->fn_num = HIST_FIELD_FN_CONST; 2847 2848 expr->operands[0] = NULL; 2849 expr->operands[1] = NULL; 2850 2851 /* 2852 * var refs won't be destroyed immediately 2853 * See: destroy_hist_field() 2854 */ 2855 destroy_hist_field(operand2, 0); 2856 destroy_hist_field(operand1, 0); 2857 2858 expr->name = expr_str(expr, 0); 2859 if (IS_ERR(expr->name)) { 2860 ret = PTR_ERR(expr->name); 2861 expr->name = NULL; 2862 goto free_expr; 2863 } 2864 } else { 2865 /* The operand sizes should be the same, so just pick one */ 2866 expr->size = operand1->size; 2867 expr->is_signed = operand1->is_signed; 2868 2869 expr->operator = field_op; 2870 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2871 if (!expr->type) { 2872 ret = -ENOMEM; 2873 goto free_expr; 2874 } 2875 2876 expr->name = expr_str(expr, 0); 2877 if (IS_ERR(expr->name)) { 2878 ret = PTR_ERR(expr->name); 2879 expr->name = NULL; 2880 goto free_expr; 2881 } 2882 } 2883 2884 return expr; 2885 2886 free_operands: 2887 destroy_hist_field(operand2, 0); 2888 free_op1: 2889 destroy_hist_field(operand1, 0); 2890 return ERR_PTR(ret); 2891 2892 free_expr: 2893 destroy_hist_field(expr, 0); 2894 return ERR_PTR(ret); 2895 } 2896 2897 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2898 struct trace_event_file *file) 2899 { 2900 struct event_trigger_data *test; 2901 2902 lockdep_assert_held(&event_mutex); 2903 2904 list_for_each_entry(test, &file->triggers, list) { 2905 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2906 if (test->private_data == hist_data) 2907 return test->filter_str; 2908 } 2909 } 2910 2911 return NULL; 2912 } 2913 2914 static struct event_command trigger_hist_cmd; 2915 static int event_hist_trigger_parse(struct event_command *cmd_ops, 2916 struct trace_event_file *file, 2917 char *glob, char *cmd, 2918 char *param_and_filter); 2919 2920 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2921 struct hist_trigger_data *hist_data, 2922 unsigned int n_keys) 2923 { 2924 struct hist_field *target_hist_field, *hist_field; 2925 unsigned int n, i, j; 2926 2927 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2928 return false; 2929 2930 i = hist_data->n_vals; 2931 j = target_hist_data->n_vals; 2932 2933 for (n = 0; n < n_keys; n++) { 2934 hist_field = hist_data->fields[i + n]; 2935 target_hist_field = target_hist_data->fields[j + n]; 2936 2937 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2938 return false; 2939 if (hist_field->size != target_hist_field->size) 2940 return false; 2941 if (hist_field->is_signed != target_hist_field->is_signed) 2942 return false; 2943 } 2944 2945 return true; 2946 } 2947 2948 static struct hist_trigger_data * 2949 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2950 struct trace_event_file *file) 2951 { 2952 struct hist_trigger_data *hist_data; 2953 struct event_trigger_data *test; 2954 unsigned int n_keys; 2955 2956 lockdep_assert_held(&event_mutex); 2957 2958 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2959 2960 list_for_each_entry(test, &file->triggers, list) { 2961 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2962 hist_data = test->private_data; 2963 2964 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2965 return hist_data; 2966 } 2967 } 2968 2969 return NULL; 2970 } 2971 2972 static struct trace_event_file *event_file(struct trace_array *tr, 2973 char *system, char *event_name) 2974 { 2975 struct trace_event_file *file; 2976 2977 file = __find_event_file(tr, system, event_name); 2978 if (!file) 2979 return ERR_PTR(-EINVAL); 2980 2981 return file; 2982 } 2983 2984 static struct hist_field * 2985 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2986 char *system, char *event_name, char *field_name) 2987 { 2988 struct hist_field *event_var; 2989 char *synthetic_name; 2990 struct seq_buf s; 2991 2992 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2993 if (!synthetic_name) 2994 return ERR_PTR(-ENOMEM); 2995 2996 seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL); 2997 seq_buf_printf(&s, "synthetic_%s", field_name); 2998 2999 seq_buf_terminate(&s); 3000 3001 if (seq_buf_has_overflowed(&s)) { 3002 kfree(synthetic_name); 3003 return ERR_PTR(-E2BIG); 3004 } 3005 3006 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 3007 3008 kfree(synthetic_name); 3009 3010 return event_var; 3011 } 3012 3013 /** 3014 * create_field_var_hist - Automatically create a histogram and var for a field 3015 * @target_hist_data: The target hist trigger 3016 * @subsys_name: Optional subsystem name 3017 * @event_name: Optional event name 3018 * @field_name: The name of the field (and the resulting variable) 3019 * 3020 * Hist trigger actions fetch data from variables, not directly from 3021 * events. However, for convenience, users are allowed to directly 3022 * specify an event field in an action, which will be automatically 3023 * converted into a variable on their behalf. 3024 * 3025 * If a user specifies a field on an event that isn't the event the 3026 * histogram currently being defined (the target event histogram), the 3027 * only way that can be accomplished is if a new hist trigger is 3028 * created and the field variable defined on that. 3029 * 3030 * This function creates a new histogram compatible with the target 3031 * event (meaning a histogram with the same key as the target 3032 * histogram), and creates a variable for the specified field, but 3033 * with 'synthetic_' prepended to the variable name in order to avoid 3034 * collision with normal field variables. 3035 * 3036 * Return: The variable created for the field. 3037 */ 3038 static struct hist_field * 3039 create_field_var_hist(struct hist_trigger_data *target_hist_data, 3040 char *subsys_name, char *event_name, char *field_name) 3041 { 3042 struct trace_array *tr = target_hist_data->event_file->tr; 3043 struct hist_trigger_data *hist_data; 3044 unsigned int i, n, first = true; 3045 struct field_var_hist *var_hist; 3046 struct trace_event_file *file; 3047 struct hist_field *key_field; 3048 struct hist_field *event_var; 3049 char *saved_filter; 3050 struct seq_buf s; 3051 char *cmd; 3052 int ret; 3053 3054 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 3055 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3056 return ERR_PTR(-EINVAL); 3057 } 3058 3059 file = event_file(tr, subsys_name, event_name); 3060 3061 if (IS_ERR(file)) { 3062 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3063 ret = PTR_ERR(file); 3064 return ERR_PTR(ret); 3065 } 3066 3067 /* 3068 * Look for a histogram compatible with target. We'll use the 3069 * found histogram specification to create a new matching 3070 * histogram with our variable on it. target_hist_data is not 3071 * yet a registered histogram so we can't use that. 3072 */ 3073 hist_data = find_compatible_hist(target_hist_data, file); 3074 if (!hist_data) { 3075 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3076 return ERR_PTR(-EINVAL); 3077 } 3078 3079 /* See if a synthetic field variable has already been created */ 3080 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3081 event_name, field_name); 3082 if (!IS_ERR_OR_NULL(event_var)) 3083 return event_var; 3084 3085 var_hist = kzalloc_obj(*var_hist); 3086 if (!var_hist) 3087 return ERR_PTR(-ENOMEM); 3088 3089 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3090 if (!cmd) { 3091 kfree(var_hist); 3092 return ERR_PTR(-ENOMEM); 3093 } 3094 3095 seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL); 3096 3097 /* Use the same keys as the compatible histogram */ 3098 seq_buf_puts(&s, "keys="); 3099 3100 for_each_hist_key_field(i, hist_data) { 3101 key_field = hist_data->fields[i]; 3102 if (!first) 3103 seq_buf_putc(&s, ','); 3104 expr_field_str(key_field, &s); 3105 first = false; 3106 } 3107 3108 /* Create the synthetic field variable specification */ 3109 seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name); 3110 3111 /* Use the same filter as the compatible histogram */ 3112 saved_filter = find_trigger_filter(hist_data, file); 3113 if (saved_filter) 3114 seq_buf_printf(&s, " if %s", saved_filter); 3115 3116 seq_buf_terminate(&s); 3117 3118 if (seq_buf_has_overflowed(&s)) { 3119 kfree(cmd); 3120 kfree(var_hist); 3121 return ERR_PTR(-E2BIG); 3122 } 3123 3124 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3125 if (!var_hist->cmd) { 3126 kfree(cmd); 3127 kfree(var_hist); 3128 return ERR_PTR(-ENOMEM); 3129 } 3130 3131 /* Needed to remove the histogram when the target goes away */ 3132 var_hist->file = file; 3133 3134 /* Create the new histogram with our variable */ 3135 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 3136 "", "hist", cmd); 3137 if (ret) { 3138 kfree(cmd); 3139 kfree(var_hist->cmd); 3140 kfree(var_hist); 3141 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3142 return ERR_PTR(ret); 3143 } 3144 3145 kfree(cmd); 3146 3147 n = target_hist_data->n_field_var_hists; 3148 target_hist_data->field_var_hists[n] = var_hist; 3149 target_hist_data->n_field_var_hists++; 3150 3151 /* If we can't find the variable, something went wrong */ 3152 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3153 event_name, field_name); 3154 if (IS_ERR_OR_NULL(event_var)) { 3155 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3156 return ERR_PTR(-EINVAL); 3157 } 3158 3159 return event_var; 3160 } 3161 3162 static struct hist_field * 3163 find_target_event_var(struct hist_trigger_data *hist_data, 3164 char *subsys_name, char *event_name, char *var_name) 3165 { 3166 struct trace_event_file *file = hist_data->event_file; 3167 struct hist_field *hist_field = NULL; 3168 3169 if (subsys_name) { 3170 struct trace_event_call *call; 3171 3172 if (!event_name) 3173 return NULL; 3174 3175 call = file->event_call; 3176 3177 if (strcmp(subsys_name, call->class->system) != 0) 3178 return NULL; 3179 3180 if (strcmp(event_name, trace_event_name(call)) != 0) 3181 return NULL; 3182 } 3183 3184 hist_field = find_var_field(hist_data, var_name); 3185 3186 return hist_field; 3187 } 3188 3189 static inline void __update_field_vars(struct tracing_map_elt *elt, 3190 struct trace_buffer *buffer, 3191 struct ring_buffer_event *rbe, 3192 void *rec, 3193 struct field_var **field_vars, 3194 unsigned int n_field_vars, 3195 unsigned int field_var_str_start) 3196 { 3197 struct hist_elt_data *elt_data = elt->private_data; 3198 unsigned int i, j, var_idx; 3199 u64 var_val; 3200 3201 /* Make sure stacktrace can fit in the string variable length */ 3202 BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) > STR_VAR_LEN_MAX); 3203 3204 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3205 struct field_var *field_var = field_vars[i]; 3206 struct hist_field *var = field_var->var; 3207 struct hist_field *val = field_var->val; 3208 3209 var_val = hist_fn_call(val, elt, buffer, rbe, rec); 3210 var_idx = var->var.idx; 3211 3212 if (val->flags & (HIST_FIELD_FL_STRING | 3213 HIST_FIELD_FL_STACKTRACE)) { 3214 char *str = elt_data->field_var_str[j++]; 3215 char *val_str = (char *)(uintptr_t)var_val; 3216 unsigned int size; 3217 3218 if (val->flags & HIST_FIELD_FL_STRING) { 3219 size = min(val->size, STR_VAR_LEN_MAX); 3220 strscpy(str, val_str, size); 3221 } else { 3222 char *stack_start = str + sizeof(unsigned long); 3223 int e; 3224 3225 e = stack_trace_save((void *)stack_start, 3226 HIST_STACKTRACE_DEPTH, 3227 HIST_STACKTRACE_SKIP); 3228 if (e < HIST_STACKTRACE_DEPTH - 1) 3229 ((unsigned long *)stack_start)[e] = 0; 3230 *((unsigned long *)str) = e; 3231 } 3232 var_val = (u64)(uintptr_t)str; 3233 } 3234 tracing_map_set_var(elt, var_idx, var_val); 3235 } 3236 } 3237 3238 static void update_field_vars(struct hist_trigger_data *hist_data, 3239 struct tracing_map_elt *elt, 3240 struct trace_buffer *buffer, 3241 struct ring_buffer_event *rbe, 3242 void *rec) 3243 { 3244 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars, 3245 hist_data->n_field_vars, 0); 3246 } 3247 3248 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3249 struct tracing_map_elt *elt, 3250 struct trace_buffer *buffer, void *rec, 3251 struct ring_buffer_event *rbe, void *key, 3252 struct action_data *data, u64 *var_ref_vals) 3253 { 3254 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars, 3255 hist_data->n_save_vars, hist_data->n_field_var_str); 3256 } 3257 3258 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3259 struct trace_event_file *file, 3260 char *name, int size, const char *type) 3261 { 3262 struct hist_field *var; 3263 int idx; 3264 3265 if (find_var(hist_data, file, name) && !hist_data->remove) { 3266 var = ERR_PTR(-EINVAL); 3267 goto out; 3268 } 3269 3270 var = kzalloc_obj(struct hist_field); 3271 if (!var) { 3272 var = ERR_PTR(-ENOMEM); 3273 goto out; 3274 } 3275 3276 idx = tracing_map_add_var(hist_data->map); 3277 if (idx < 0) { 3278 kfree(var); 3279 var = ERR_PTR(-EINVAL); 3280 goto out; 3281 } 3282 3283 var->flags = HIST_FIELD_FL_VAR; 3284 var->var.idx = idx; 3285 var->var.hist_data = var->hist_data = hist_data; 3286 var->size = size; 3287 var->var.name = kstrdup(name, GFP_KERNEL); 3288 var->type = kstrdup_const(type, GFP_KERNEL); 3289 if (!var->var.name || !var->type) { 3290 kfree_const(var->type); 3291 kfree(var->var.name); 3292 kfree(var); 3293 var = ERR_PTR(-ENOMEM); 3294 } 3295 out: 3296 return var; 3297 } 3298 3299 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3300 struct trace_event_file *file, 3301 char *field_name) 3302 { 3303 struct hist_field *val = NULL, *var = NULL; 3304 unsigned long flags = HIST_FIELD_FL_VAR; 3305 struct trace_array *tr = file->tr; 3306 struct field_var *field_var; 3307 int ret = 0; 3308 3309 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3310 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3311 ret = -EINVAL; 3312 goto err; 3313 } 3314 3315 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3316 if (IS_ERR(val)) { 3317 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3318 ret = PTR_ERR(val); 3319 goto err; 3320 } 3321 3322 var = create_var(hist_data, file, field_name, val->size, val->type); 3323 if (IS_ERR(var)) { 3324 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3325 destroy_hist_field(val, 0); 3326 ret = PTR_ERR(var); 3327 goto err; 3328 } 3329 var->flags |= val->flags & HIST_FIELD_FL_STACKTRACE; 3330 3331 field_var = kzalloc_obj(struct field_var); 3332 if (!field_var) { 3333 destroy_hist_field(val, 0); 3334 kfree_const(var->type); 3335 kfree(var->var.name); 3336 kfree(var); 3337 ret = -ENOMEM; 3338 goto err; 3339 } 3340 3341 field_var->var = var; 3342 field_var->val = val; 3343 out: 3344 return field_var; 3345 err: 3346 field_var = ERR_PTR(ret); 3347 goto out; 3348 } 3349 3350 /** 3351 * create_target_field_var - Automatically create a variable for a field 3352 * @target_hist_data: The target hist trigger 3353 * @subsys_name: Optional subsystem name 3354 * @event_name: Optional event name 3355 * @var_name: The name of the field (and the resulting variable) 3356 * 3357 * Hist trigger actions fetch data from variables, not directly from 3358 * events. However, for convenience, users are allowed to directly 3359 * specify an event field in an action, which will be automatically 3360 * converted into a variable on their behalf. 3361 * 3362 * This function creates a field variable with the name var_name on 3363 * the hist trigger currently being defined on the target event. If 3364 * subsys_name and event_name are specified, this function simply 3365 * verifies that they do in fact match the target event subsystem and 3366 * event name. 3367 * 3368 * Return: The variable created for the field. 3369 */ 3370 static struct field_var * 3371 create_target_field_var(struct hist_trigger_data *target_hist_data, 3372 char *subsys_name, char *event_name, char *var_name) 3373 { 3374 struct trace_event_file *file = target_hist_data->event_file; 3375 3376 if (subsys_name) { 3377 struct trace_event_call *call; 3378 3379 if (!event_name) 3380 return NULL; 3381 3382 call = file->event_call; 3383 3384 if (strcmp(subsys_name, call->class->system) != 0) 3385 return NULL; 3386 3387 if (strcmp(event_name, trace_event_name(call)) != 0) 3388 return NULL; 3389 } 3390 3391 return create_field_var(target_hist_data, file, var_name); 3392 } 3393 3394 static bool check_track_val_max(u64 track_val, u64 var_val) 3395 { 3396 if (var_val <= track_val) 3397 return false; 3398 3399 return true; 3400 } 3401 3402 static bool check_track_val_changed(u64 track_val, u64 var_val) 3403 { 3404 if (var_val == track_val) 3405 return false; 3406 3407 return true; 3408 } 3409 3410 static u64 get_track_val(struct hist_trigger_data *hist_data, 3411 struct tracing_map_elt *elt, 3412 struct action_data *data) 3413 { 3414 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3415 u64 track_val; 3416 3417 track_val = tracing_map_read_var(elt, track_var_idx); 3418 3419 return track_val; 3420 } 3421 3422 static void save_track_val(struct hist_trigger_data *hist_data, 3423 struct tracing_map_elt *elt, 3424 struct action_data *data, u64 var_val) 3425 { 3426 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3427 3428 tracing_map_set_var(elt, track_var_idx, var_val); 3429 } 3430 3431 static void save_track_data(struct hist_trigger_data *hist_data, 3432 struct tracing_map_elt *elt, 3433 struct trace_buffer *buffer, void *rec, 3434 struct ring_buffer_event *rbe, void *key, 3435 struct action_data *data, u64 *var_ref_vals) 3436 { 3437 if (data->track_data.save_data) 3438 data->track_data.save_data(hist_data, elt, buffer, rec, rbe, 3439 key, data, var_ref_vals); 3440 } 3441 3442 static bool check_track_val(struct tracing_map_elt *elt, 3443 struct action_data *data, 3444 u64 var_val) 3445 { 3446 struct hist_trigger_data *hist_data; 3447 u64 track_val; 3448 3449 hist_data = data->track_data.track_var->hist_data; 3450 track_val = get_track_val(hist_data, elt, data); 3451 3452 return data->track_data.check_val(track_val, var_val); 3453 } 3454 3455 #ifdef CONFIG_TRACER_SNAPSHOT 3456 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3457 { 3458 /* called with tr->max_lock held */ 3459 struct track_data *track_data = tr->cond_snapshot->cond_data; 3460 struct hist_elt_data *elt_data, *track_elt_data; 3461 struct snapshot_context *context = cond_data; 3462 struct action_data *action; 3463 u64 track_val; 3464 3465 if (!track_data) 3466 return false; 3467 3468 action = track_data->action_data; 3469 3470 track_val = get_track_val(track_data->hist_data, context->elt, 3471 track_data->action_data); 3472 3473 if (!action->track_data.check_val(track_data->track_val, track_val)) 3474 return false; 3475 3476 track_data->track_val = track_val; 3477 memcpy(track_data->key, context->key, track_data->key_len); 3478 3479 elt_data = context->elt->private_data; 3480 track_elt_data = track_data->elt.private_data; 3481 if (elt_data->comm) 3482 strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3483 3484 track_data->updated = true; 3485 3486 return true; 3487 } 3488 3489 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3490 struct tracing_map_elt *elt, 3491 struct trace_buffer *buffer, void *rec, 3492 struct ring_buffer_event *rbe, void *key, 3493 struct action_data *data, 3494 u64 *var_ref_vals) 3495 { 3496 struct trace_event_file *file = hist_data->event_file; 3497 struct snapshot_context context; 3498 3499 context.elt = elt; 3500 context.key = key; 3501 3502 tracing_snapshot_cond(file->tr, &context); 3503 } 3504 3505 static void hist_trigger_print_key(struct seq_file *m, 3506 struct hist_trigger_data *hist_data, 3507 void *key, 3508 struct tracing_map_elt *elt); 3509 3510 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3511 { 3512 unsigned int i; 3513 3514 if (!hist_data->n_actions) 3515 return NULL; 3516 3517 for (i = 0; i < hist_data->n_actions; i++) { 3518 struct action_data *data = hist_data->actions[i]; 3519 3520 if (data->action == ACTION_SNAPSHOT) 3521 return data; 3522 } 3523 3524 return NULL; 3525 } 3526 3527 static void track_data_snapshot_print(struct seq_file *m, 3528 struct hist_trigger_data *hist_data) 3529 { 3530 struct trace_event_file *file = hist_data->event_file; 3531 struct track_data *track_data; 3532 struct action_data *action; 3533 3534 track_data = tracing_cond_snapshot_data(file->tr); 3535 if (!track_data) 3536 return; 3537 3538 if (!track_data->updated) 3539 return; 3540 3541 action = snapshot_action(hist_data); 3542 if (!action) 3543 return; 3544 3545 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3546 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3547 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3548 action->track_data.var_str, track_data->track_val); 3549 3550 seq_puts(m, "\ttriggered by event with key: "); 3551 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3552 seq_putc(m, '\n'); 3553 } 3554 #else 3555 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3556 { 3557 return false; 3558 } 3559 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3560 struct tracing_map_elt *elt, 3561 struct trace_buffer *buffer, void *rec, 3562 struct ring_buffer_event *rbe, void *key, 3563 struct action_data *data, 3564 u64 *var_ref_vals) {} 3565 static void track_data_snapshot_print(struct seq_file *m, 3566 struct hist_trigger_data *hist_data) {} 3567 #endif /* CONFIG_TRACER_SNAPSHOT */ 3568 3569 static void track_data_print(struct seq_file *m, 3570 struct hist_trigger_data *hist_data, 3571 struct tracing_map_elt *elt, 3572 struct action_data *data) 3573 { 3574 u64 track_val = get_track_val(hist_data, elt, data); 3575 unsigned int i, save_var_idx; 3576 3577 if (data->handler == HANDLER_ONMAX) 3578 seq_printf(m, "\n\tmax: %10llu", track_val); 3579 else if (data->handler == HANDLER_ONCHANGE) 3580 seq_printf(m, "\n\tchanged: %10llu", track_val); 3581 3582 if (data->action == ACTION_SNAPSHOT) 3583 return; 3584 3585 for (i = 0; i < hist_data->n_save_vars; i++) { 3586 struct hist_field *save_val = hist_data->save_vars[i]->val; 3587 struct hist_field *save_var = hist_data->save_vars[i]->var; 3588 u64 val; 3589 3590 save_var_idx = save_var->var.idx; 3591 3592 val = tracing_map_read_var(elt, save_var_idx); 3593 3594 if (save_val->flags & HIST_FIELD_FL_STRING) { 3595 seq_printf(m, " %s: %-32s", save_var->var.name, 3596 (char *)(uintptr_t)(val)); 3597 } else 3598 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3599 } 3600 } 3601 3602 static void ontrack_action(struct hist_trigger_data *hist_data, 3603 struct tracing_map_elt *elt, 3604 struct trace_buffer *buffer, void *rec, 3605 struct ring_buffer_event *rbe, void *key, 3606 struct action_data *data, u64 *var_ref_vals) 3607 { 3608 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3609 3610 if (check_track_val(elt, data, var_val)) { 3611 save_track_val(hist_data, elt, data, var_val); 3612 save_track_data(hist_data, elt, buffer, rec, rbe, 3613 key, data, var_ref_vals); 3614 } 3615 } 3616 3617 static void action_data_destroy(struct action_data *data) 3618 { 3619 unsigned int i; 3620 3621 lockdep_assert_held(&event_mutex); 3622 3623 kfree(data->action_name); 3624 3625 for (i = 0; i < data->n_params; i++) 3626 kfree(data->params[i]); 3627 3628 if (data->synth_event) 3629 data->synth_event->ref--; 3630 3631 kfree(data->synth_event_name); 3632 3633 kfree(data); 3634 } 3635 3636 static void track_data_destroy(struct hist_trigger_data *hist_data, 3637 struct action_data *data) 3638 { 3639 struct trace_event_file *file = hist_data->event_file; 3640 3641 destroy_hist_field(data->track_data.track_var, 0); 3642 3643 if (data->action == ACTION_SNAPSHOT) { 3644 struct track_data *track_data; 3645 3646 track_data = tracing_cond_snapshot_data(file->tr); 3647 if (track_data && track_data->hist_data == hist_data) { 3648 tracing_snapshot_cond_disable(file->tr); 3649 track_data_free(track_data); 3650 } 3651 } 3652 3653 kfree(data->track_data.var_str); 3654 3655 action_data_destroy(data); 3656 } 3657 3658 static int action_create(struct hist_trigger_data *hist_data, 3659 struct action_data *data); 3660 3661 static int track_data_create(struct hist_trigger_data *hist_data, 3662 struct action_data *data) 3663 { 3664 struct hist_field *var_field, *ref_field, *track_var = NULL; 3665 struct trace_event_file *file = hist_data->event_file; 3666 struct trace_array *tr = file->tr; 3667 char *track_data_var_str; 3668 int ret = 0; 3669 3670 track_data_var_str = data->track_data.var_str; 3671 if (track_data_var_str[0] != '$') { 3672 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3673 return -EINVAL; 3674 } 3675 track_data_var_str++; 3676 3677 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3678 if (!var_field) { 3679 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3680 return -EINVAL; 3681 } 3682 3683 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3684 if (!ref_field) 3685 return -ENOMEM; 3686 3687 data->track_data.var_ref = ref_field; 3688 3689 if (data->handler == HANDLER_ONMAX) 3690 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3691 if (IS_ERR(track_var)) { 3692 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3693 ret = PTR_ERR(track_var); 3694 goto out; 3695 } 3696 3697 if (data->handler == HANDLER_ONCHANGE) 3698 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3699 if (IS_ERR(track_var)) { 3700 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3701 ret = PTR_ERR(track_var); 3702 goto out; 3703 } 3704 data->track_data.track_var = track_var; 3705 3706 ret = action_create(hist_data, data); 3707 out: 3708 return ret; 3709 } 3710 3711 static int parse_action_params(struct trace_array *tr, char *params, 3712 struct action_data *data) 3713 { 3714 char *param, *saved_param; 3715 bool first_param = true; 3716 int ret = 0; 3717 3718 while (params) { 3719 if (data->n_params >= SYNTH_FIELDS_MAX) { 3720 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3721 ret = -EINVAL; 3722 goto out; 3723 } 3724 3725 param = strsep(¶ms, ","); 3726 if (!param) { 3727 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3728 ret = -EINVAL; 3729 goto out; 3730 } 3731 3732 param = strstrip(param); 3733 if (strlen(param) < 2) { 3734 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3735 ret = -EINVAL; 3736 goto out; 3737 } 3738 3739 saved_param = kstrdup(param, GFP_KERNEL); 3740 if (!saved_param) { 3741 ret = -ENOMEM; 3742 goto out; 3743 } 3744 3745 if (first_param && data->use_trace_keyword) { 3746 data->synth_event_name = saved_param; 3747 first_param = false; 3748 continue; 3749 } 3750 first_param = false; 3751 3752 data->params[data->n_params++] = saved_param; 3753 } 3754 out: 3755 return ret; 3756 } 3757 3758 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3759 enum handler_id handler) 3760 { 3761 char *action_name; 3762 int ret = 0; 3763 3764 strsep(&str, "."); 3765 if (!str) { 3766 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3767 ret = -EINVAL; 3768 goto out; 3769 } 3770 3771 action_name = strsep(&str, "("); 3772 if (!action_name || !str) { 3773 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3774 ret = -EINVAL; 3775 goto out; 3776 } 3777 3778 if (str_has_prefix(action_name, "save")) { 3779 char *params = strsep(&str, ")"); 3780 3781 if (!params) { 3782 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3783 ret = -EINVAL; 3784 goto out; 3785 } 3786 3787 ret = parse_action_params(tr, params, data); 3788 if (ret) 3789 goto out; 3790 3791 if (handler == HANDLER_ONMAX) 3792 data->track_data.check_val = check_track_val_max; 3793 else if (handler == HANDLER_ONCHANGE) 3794 data->track_data.check_val = check_track_val_changed; 3795 else { 3796 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3797 ret = -EINVAL; 3798 goto out; 3799 } 3800 3801 data->track_data.save_data = save_track_data_vars; 3802 data->fn = ontrack_action; 3803 data->action = ACTION_SAVE; 3804 } else if (str_has_prefix(action_name, "snapshot")) { 3805 char *params = strsep(&str, ")"); 3806 3807 if (!str) { 3808 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3809 ret = -EINVAL; 3810 goto out; 3811 } 3812 3813 if (handler == HANDLER_ONMAX) 3814 data->track_data.check_val = check_track_val_max; 3815 else if (handler == HANDLER_ONCHANGE) 3816 data->track_data.check_val = check_track_val_changed; 3817 else { 3818 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3819 ret = -EINVAL; 3820 goto out; 3821 } 3822 3823 data->track_data.save_data = save_track_data_snapshot; 3824 data->fn = ontrack_action; 3825 data->action = ACTION_SNAPSHOT; 3826 } else { 3827 char *params = strsep(&str, ")"); 3828 3829 if (str_has_prefix(action_name, "trace")) 3830 data->use_trace_keyword = true; 3831 3832 if (params) { 3833 ret = parse_action_params(tr, params, data); 3834 if (ret) 3835 goto out; 3836 } 3837 3838 if (handler == HANDLER_ONMAX) 3839 data->track_data.check_val = check_track_val_max; 3840 else if (handler == HANDLER_ONCHANGE) 3841 data->track_data.check_val = check_track_val_changed; 3842 3843 if (handler != HANDLER_ONMATCH) { 3844 data->track_data.save_data = action_trace; 3845 data->fn = ontrack_action; 3846 } else 3847 data->fn = action_trace; 3848 3849 data->action = ACTION_TRACE; 3850 } 3851 3852 data->action_name = kstrdup(action_name, GFP_KERNEL); 3853 if (!data->action_name) { 3854 ret = -ENOMEM; 3855 goto out; 3856 } 3857 3858 data->handler = handler; 3859 out: 3860 return ret; 3861 } 3862 3863 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3864 char *str, enum handler_id handler) 3865 { 3866 struct action_data *data; 3867 int ret = -EINVAL; 3868 char *var_str; 3869 3870 data = kzalloc_obj(*data); 3871 if (!data) 3872 return ERR_PTR(-ENOMEM); 3873 3874 var_str = strsep(&str, ")"); 3875 if (!var_str || !str) { 3876 ret = -EINVAL; 3877 goto free; 3878 } 3879 3880 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3881 if (!data->track_data.var_str) { 3882 ret = -ENOMEM; 3883 goto free; 3884 } 3885 3886 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3887 if (ret) 3888 goto free; 3889 out: 3890 return data; 3891 free: 3892 track_data_destroy(hist_data, data); 3893 data = ERR_PTR(ret); 3894 goto out; 3895 } 3896 3897 static void onmatch_destroy(struct action_data *data) 3898 { 3899 kfree(data->match_data.event); 3900 kfree(data->match_data.event_system); 3901 3902 action_data_destroy(data); 3903 } 3904 3905 static void destroy_field_var(struct field_var *field_var) 3906 { 3907 if (!field_var) 3908 return; 3909 3910 destroy_hist_field(field_var->var, 0); 3911 destroy_hist_field(field_var->val, 0); 3912 3913 kfree(field_var); 3914 } 3915 3916 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3917 { 3918 unsigned int i; 3919 3920 for (i = 0; i < hist_data->n_field_vars; i++) 3921 destroy_field_var(hist_data->field_vars[i]); 3922 3923 for (i = 0; i < hist_data->n_save_vars; i++) 3924 destroy_field_var(hist_data->save_vars[i]); 3925 } 3926 3927 static void save_field_var(struct hist_trigger_data *hist_data, 3928 struct field_var *field_var) 3929 { 3930 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3931 3932 /* Stack traces are saved in the string storage too */ 3933 if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 3934 hist_data->n_field_var_str++; 3935 } 3936 3937 3938 static int check_synth_field(struct synth_event *event, 3939 struct hist_field *hist_field, 3940 unsigned int field_pos) 3941 { 3942 struct synth_field *field; 3943 3944 if (field_pos >= event->n_fields) 3945 return -EINVAL; 3946 3947 field = event->fields[field_pos]; 3948 3949 /* 3950 * A dynamic string synth field can accept static or 3951 * dynamic. A static string synth field can only accept a 3952 * same-sized static string, which is checked for later. 3953 */ 3954 if (strstr(hist_field->type, "char[") && field->is_string 3955 && field->is_dynamic) 3956 return 0; 3957 3958 if (!!(hist_field->flags & HIST_FIELD_FL_STACKTRACE) != field->is_stack) 3959 return -EINVAL; 3960 3961 if (field->is_stack) 3962 return 0; 3963 3964 if (strcmp(field->type, hist_field->type) != 0) { 3965 if (field->size != hist_field->size || 3966 (!field->is_string && field->is_signed != hist_field->is_signed)) 3967 return -EINVAL; 3968 } 3969 3970 return 0; 3971 } 3972 3973 static struct hist_field * 3974 trace_action_find_var(struct hist_trigger_data *hist_data, 3975 struct action_data *data, 3976 char *system, char *event, char *var) 3977 { 3978 struct trace_array *tr = hist_data->event_file->tr; 3979 struct hist_field *hist_field; 3980 3981 var++; /* skip '$' */ 3982 3983 hist_field = find_target_event_var(hist_data, system, event, var); 3984 if (!hist_field) { 3985 if (!system && data->handler == HANDLER_ONMATCH) { 3986 system = data->match_data.event_system; 3987 event = data->match_data.event; 3988 } 3989 3990 hist_field = find_event_var(hist_data, system, event, var); 3991 } 3992 3993 if (!hist_field) 3994 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3995 3996 return hist_field; 3997 } 3998 3999 static struct hist_field * 4000 trace_action_create_field_var(struct hist_trigger_data *hist_data, 4001 struct action_data *data, char *system, 4002 char *event, char *var) 4003 { 4004 struct hist_field *hist_field = NULL; 4005 struct field_var *field_var; 4006 4007 /* 4008 * First try to create a field var on the target event (the 4009 * currently being defined). This will create a variable for 4010 * unqualified fields on the target event, or if qualified, 4011 * target fields that have qualified names matching the target. 4012 */ 4013 field_var = create_target_field_var(hist_data, system, event, var); 4014 4015 if (field_var && !IS_ERR(field_var)) { 4016 save_field_var(hist_data, field_var); 4017 hist_field = field_var->var; 4018 } else { 4019 field_var = NULL; 4020 /* 4021 * If no explicit system.event is specified, default to 4022 * looking for fields on the onmatch(system.event.xxx) 4023 * event. 4024 */ 4025 if (!system && data->handler == HANDLER_ONMATCH) { 4026 system = data->match_data.event_system; 4027 event = data->match_data.event; 4028 } 4029 4030 if (!event) 4031 goto free; 4032 /* 4033 * At this point, we're looking at a field on another 4034 * event. Because we can't modify a hist trigger on 4035 * another event to add a variable for a field, we need 4036 * to create a new trigger on that event and create the 4037 * variable at the same time. 4038 */ 4039 hist_field = create_field_var_hist(hist_data, system, event, var); 4040 if (IS_ERR(hist_field)) 4041 goto free; 4042 } 4043 out: 4044 return hist_field; 4045 free: 4046 destroy_field_var(field_var); 4047 hist_field = NULL; 4048 goto out; 4049 } 4050 4051 static int trace_action_create(struct hist_trigger_data *hist_data, 4052 struct action_data *data) 4053 { 4054 struct trace_array *tr = hist_data->event_file->tr; 4055 char *event_name, *param, *system = NULL; 4056 struct hist_field *hist_field, *var_ref; 4057 unsigned int i; 4058 unsigned int field_pos = 0; 4059 struct synth_event *event; 4060 char *synth_event_name; 4061 int var_ref_idx, ret = 0; 4062 4063 lockdep_assert_held(&event_mutex); 4064 4065 /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */ 4066 if (data->n_params > SYNTH_FIELDS_MAX) 4067 return -EINVAL; 4068 4069 if (data->use_trace_keyword) 4070 synth_event_name = data->synth_event_name; 4071 else 4072 synth_event_name = data->action_name; 4073 4074 event = find_synth_event(synth_event_name); 4075 if (!event) { 4076 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4077 return -EINVAL; 4078 } 4079 4080 event->ref++; 4081 4082 for (i = 0; i < data->n_params; i++) { 4083 char *p; 4084 4085 p = param = kstrdup(data->params[i], GFP_KERNEL); 4086 if (!param) { 4087 ret = -ENOMEM; 4088 goto err; 4089 } 4090 4091 system = strsep(¶m, "."); 4092 if (!param) { 4093 param = (char *)system; 4094 system = event_name = NULL; 4095 } else { 4096 event_name = strsep(¶m, "."); 4097 if (!param) { 4098 kfree(p); 4099 ret = -EINVAL; 4100 goto err; 4101 } 4102 } 4103 4104 if (param[0] == '$') 4105 hist_field = trace_action_find_var(hist_data, data, 4106 system, event_name, 4107 param); 4108 else 4109 hist_field = trace_action_create_field_var(hist_data, 4110 data, 4111 system, 4112 event_name, 4113 param); 4114 4115 if (!hist_field) { 4116 kfree(p); 4117 ret = -EINVAL; 4118 goto err; 4119 } 4120 4121 if (check_synth_field(event, hist_field, field_pos) == 0) { 4122 var_ref = create_var_ref(hist_data, hist_field, 4123 system, event_name); 4124 if (!var_ref) { 4125 kfree(p); 4126 ret = -ENOMEM; 4127 goto err; 4128 } 4129 4130 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 4131 if (WARN_ON(var_ref_idx < 0)) { 4132 kfree(p); 4133 ret = var_ref_idx; 4134 goto err; 4135 } 4136 4137 data->var_ref_idx[i] = var_ref_idx; 4138 4139 field_pos++; 4140 kfree(p); 4141 continue; 4142 } 4143 4144 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4145 kfree(p); 4146 ret = -EINVAL; 4147 goto err; 4148 } 4149 4150 if (field_pos != event->n_fields) { 4151 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4152 ret = -EINVAL; 4153 goto err; 4154 } 4155 4156 data->synth_event = event; 4157 out: 4158 return ret; 4159 err: 4160 event->ref--; 4161 4162 goto out; 4163 } 4164 4165 static int action_create(struct hist_trigger_data *hist_data, 4166 struct action_data *data) 4167 { 4168 struct trace_event_file *file = hist_data->event_file; 4169 struct trace_array *tr = file->tr; 4170 struct track_data *track_data; 4171 struct field_var *field_var; 4172 unsigned int i; 4173 char *param; 4174 int ret = 0; 4175 4176 if (data->action == ACTION_TRACE) 4177 return trace_action_create(hist_data, data); 4178 4179 if (data->action == ACTION_SNAPSHOT) { 4180 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4181 if (IS_ERR(track_data)) { 4182 ret = PTR_ERR(track_data); 4183 goto out; 4184 } 4185 4186 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4187 cond_snapshot_update); 4188 if (ret) 4189 track_data_free(track_data); 4190 4191 goto out; 4192 } 4193 4194 if (data->action == ACTION_SAVE) { 4195 if (hist_data->n_save_vars) { 4196 ret = -EEXIST; 4197 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4198 goto out; 4199 } 4200 4201 for (i = 0; i < data->n_params; i++) { 4202 param = kstrdup(data->params[i], GFP_KERNEL); 4203 if (!param) { 4204 ret = -ENOMEM; 4205 goto out; 4206 } 4207 4208 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4209 if (IS_ERR(field_var)) { 4210 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4211 errpos(param)); 4212 ret = PTR_ERR(field_var); 4213 kfree(param); 4214 goto out; 4215 } 4216 4217 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4218 if (field_var->val->flags & 4219 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4220 hist_data->n_save_var_str++; 4221 kfree(param); 4222 } 4223 } 4224 out: 4225 return ret; 4226 } 4227 4228 static int onmatch_create(struct hist_trigger_data *hist_data, 4229 struct action_data *data) 4230 { 4231 return action_create(hist_data, data); 4232 } 4233 4234 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4235 { 4236 char *match_event, *match_event_system; 4237 struct action_data *data; 4238 int ret = -EINVAL; 4239 4240 data = kzalloc_obj(*data); 4241 if (!data) 4242 return ERR_PTR(-ENOMEM); 4243 4244 match_event = strsep(&str, ")"); 4245 if (!match_event || !str) { 4246 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4247 goto free; 4248 } 4249 4250 match_event_system = strsep(&match_event, "."); 4251 if (!match_event) { 4252 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4253 goto free; 4254 } 4255 4256 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4257 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4258 goto free; 4259 } 4260 4261 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4262 if (!data->match_data.event) { 4263 ret = -ENOMEM; 4264 goto free; 4265 } 4266 4267 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4268 if (!data->match_data.event_system) { 4269 ret = -ENOMEM; 4270 goto free; 4271 } 4272 4273 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4274 if (ret) 4275 goto free; 4276 out: 4277 return data; 4278 free: 4279 onmatch_destroy(data); 4280 data = ERR_PTR(ret); 4281 goto out; 4282 } 4283 4284 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4285 { 4286 hist_data->fields[HITCOUNT_IDX] = 4287 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4288 if (!hist_data->fields[HITCOUNT_IDX]) 4289 return -ENOMEM; 4290 4291 hist_data->n_vals++; 4292 hist_data->n_fields++; 4293 4294 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4295 return -EINVAL; 4296 4297 return 0; 4298 } 4299 4300 static int __create_val_field(struct hist_trigger_data *hist_data, 4301 unsigned int val_idx, 4302 struct trace_event_file *file, 4303 char *var_name, char *field_str, 4304 unsigned long flags) 4305 { 4306 struct hist_field *hist_field; 4307 int ret = 0, n_subexprs = 0; 4308 4309 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs); 4310 if (IS_ERR(hist_field)) { 4311 ret = PTR_ERR(hist_field); 4312 goto out; 4313 } 4314 4315 /* values and variables should not have some modifiers */ 4316 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4317 /* Variable */ 4318 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4319 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2)) 4320 goto err; 4321 } else { 4322 /* Value */ 4323 if (hist_field->flags & (HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | 4324 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | 4325 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) 4326 goto err; 4327 } 4328 4329 hist_data->fields[val_idx] = hist_field; 4330 4331 ++hist_data->n_vals; 4332 ++hist_data->n_fields; 4333 4334 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4335 ret = -EINVAL; 4336 out: 4337 return ret; 4338 err: 4339 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 4340 destroy_hist_field(hist_field, 0); 4341 return -EINVAL; 4342 } 4343 4344 static int create_val_field(struct hist_trigger_data *hist_data, 4345 unsigned int val_idx, 4346 struct trace_event_file *file, 4347 char *field_str) 4348 { 4349 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4350 return -EINVAL; 4351 4352 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4353 } 4354 4355 static const char no_comm[] = "(no comm)"; 4356 4357 static u64 hist_field_execname(struct hist_field *hist_field, 4358 struct tracing_map_elt *elt, 4359 struct trace_buffer *buffer, 4360 struct ring_buffer_event *rbe, 4361 void *event) 4362 { 4363 struct hist_elt_data *elt_data; 4364 4365 if (WARN_ON_ONCE(!elt)) 4366 return (u64)(unsigned long)no_comm; 4367 4368 elt_data = elt->private_data; 4369 4370 if (WARN_ON_ONCE(!elt_data->comm)) 4371 return (u64)(unsigned long)no_comm; 4372 4373 return (u64)(unsigned long)(elt_data->comm); 4374 } 4375 4376 static u64 hist_field_stack(struct hist_field *hist_field, 4377 struct tracing_map_elt *elt, 4378 struct trace_buffer *buffer, 4379 struct ring_buffer_event *rbe, 4380 void *event) 4381 { 4382 u32 str_item = *(u32 *)(event + hist_field->field->offset); 4383 int str_loc = str_item & 0xffff; 4384 char *addr = (char *)(event + str_loc); 4385 4386 return (u64)(unsigned long)addr; 4387 } 4388 4389 static u64 hist_fn_call(struct hist_field *hist_field, 4390 struct tracing_map_elt *elt, 4391 struct trace_buffer *buffer, 4392 struct ring_buffer_event *rbe, 4393 void *event) 4394 { 4395 switch (hist_field->fn_num) { 4396 case HIST_FIELD_FN_VAR_REF: 4397 return hist_field_var_ref(hist_field, elt, buffer, rbe, event); 4398 case HIST_FIELD_FN_COUNTER: 4399 return hist_field_counter(hist_field, elt, buffer, rbe, event); 4400 case HIST_FIELD_FN_CONST: 4401 return hist_field_const(hist_field, elt, buffer, rbe, event); 4402 case HIST_FIELD_FN_LOG2: 4403 return hist_field_log2(hist_field, elt, buffer, rbe, event); 4404 case HIST_FIELD_FN_BUCKET: 4405 return hist_field_bucket(hist_field, elt, buffer, rbe, event); 4406 case HIST_FIELD_FN_TIMESTAMP: 4407 return hist_field_timestamp(hist_field, elt, buffer, rbe, event); 4408 case HIST_FIELD_FN_CPU: 4409 return hist_field_cpu(hist_field, elt, buffer, rbe, event); 4410 case HIST_FIELD_FN_COMM: 4411 return hist_field_comm(hist_field, elt, buffer, rbe, event); 4412 case HIST_FIELD_FN_STRING: 4413 return hist_field_string(hist_field, elt, buffer, rbe, event); 4414 case HIST_FIELD_FN_DYNSTRING: 4415 return hist_field_dynstring(hist_field, elt, buffer, rbe, event); 4416 case HIST_FIELD_FN_RELDYNSTRING: 4417 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event); 4418 case HIST_FIELD_FN_PSTRING: 4419 return hist_field_pstring(hist_field, elt, buffer, rbe, event); 4420 case HIST_FIELD_FN_S64: 4421 return hist_field_s64(hist_field, elt, buffer, rbe, event); 4422 case HIST_FIELD_FN_U64: 4423 return hist_field_u64(hist_field, elt, buffer, rbe, event); 4424 case HIST_FIELD_FN_S32: 4425 return hist_field_s32(hist_field, elt, buffer, rbe, event); 4426 case HIST_FIELD_FN_U32: 4427 return hist_field_u32(hist_field, elt, buffer, rbe, event); 4428 case HIST_FIELD_FN_S16: 4429 return hist_field_s16(hist_field, elt, buffer, rbe, event); 4430 case HIST_FIELD_FN_U16: 4431 return hist_field_u16(hist_field, elt, buffer, rbe, event); 4432 case HIST_FIELD_FN_S8: 4433 return hist_field_s8(hist_field, elt, buffer, rbe, event); 4434 case HIST_FIELD_FN_U8: 4435 return hist_field_u8(hist_field, elt, buffer, rbe, event); 4436 case HIST_FIELD_FN_UMINUS: 4437 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event); 4438 case HIST_FIELD_FN_MINUS: 4439 return hist_field_minus(hist_field, elt, buffer, rbe, event); 4440 case HIST_FIELD_FN_PLUS: 4441 return hist_field_plus(hist_field, elt, buffer, rbe, event); 4442 case HIST_FIELD_FN_DIV: 4443 return hist_field_div(hist_field, elt, buffer, rbe, event); 4444 case HIST_FIELD_FN_MULT: 4445 return hist_field_mult(hist_field, elt, buffer, rbe, event); 4446 case HIST_FIELD_FN_DIV_POWER2: 4447 return div_by_power_of_two(hist_field, elt, buffer, rbe, event); 4448 case HIST_FIELD_FN_DIV_NOT_POWER2: 4449 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event); 4450 case HIST_FIELD_FN_DIV_MULT_SHIFT: 4451 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event); 4452 case HIST_FIELD_FN_EXECNAME: 4453 return hist_field_execname(hist_field, elt, buffer, rbe, event); 4454 case HIST_FIELD_FN_STACK: 4455 return hist_field_stack(hist_field, elt, buffer, rbe, event); 4456 default: 4457 return 0; 4458 } 4459 } 4460 4461 /* Convert a var that points to common_pid.execname to a string */ 4462 static void update_var_execname(struct hist_field *hist_field) 4463 { 4464 hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR | 4465 HIST_FIELD_FL_EXECNAME; 4466 hist_field->size = MAX_FILTER_STR_VAL; 4467 hist_field->is_signed = 0; 4468 4469 kfree_const(hist_field->type); 4470 hist_field->type = "char[]"; 4471 4472 hist_field->fn_num = HIST_FIELD_FN_EXECNAME; 4473 } 4474 4475 static int create_var_field(struct hist_trigger_data *hist_data, 4476 unsigned int val_idx, 4477 struct trace_event_file *file, 4478 char *var_name, char *expr_str) 4479 { 4480 struct trace_array *tr = hist_data->event_file->tr; 4481 unsigned long flags = 0; 4482 int ret; 4483 4484 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4485 return -EINVAL; 4486 4487 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4488 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4489 return -EINVAL; 4490 } 4491 4492 flags |= HIST_FIELD_FL_VAR; 4493 hist_data->n_vars++; 4494 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4495 return -EINVAL; 4496 4497 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4498 4499 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME) 4500 update_var_execname(hist_data->fields[val_idx]); 4501 4502 if (!ret && !(hist_data->fields[val_idx]->flags & HIST_FIELD_FL_ALIAS) && 4503 hist_data->fields[val_idx]->flags & 4504 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4505 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; 4506 4507 return ret; 4508 } 4509 4510 static int create_val_fields(struct hist_trigger_data *hist_data, 4511 struct trace_event_file *file) 4512 { 4513 unsigned int i, j = 1, n_hitcount = 0; 4514 char *fields_str, *field_str; 4515 int ret; 4516 4517 ret = create_hitcount_val(hist_data); 4518 if (ret) 4519 goto out; 4520 4521 fields_str = hist_data->attrs->vals_str; 4522 if (!fields_str) 4523 goto out; 4524 4525 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4526 j < TRACING_MAP_VALS_MAX; i++) { 4527 field_str = strsep(&fields_str, ","); 4528 if (!field_str) 4529 break; 4530 4531 if (strcmp(field_str, "hitcount") == 0) { 4532 if (!n_hitcount++) 4533 continue; 4534 } 4535 4536 ret = create_val_field(hist_data, j++, file, field_str); 4537 if (ret) 4538 goto out; 4539 } 4540 4541 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4542 ret = -EINVAL; 4543 out: 4544 /* There is only raw hitcount but nohitcount suppresses it. */ 4545 if (j == 1 && hist_data->attrs->no_hitcount) { 4546 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0); 4547 ret = -ENOENT; 4548 } 4549 4550 return ret; 4551 } 4552 4553 static int create_key_field(struct hist_trigger_data *hist_data, 4554 unsigned int key_idx, 4555 unsigned int key_offset, 4556 struct trace_event_file *file, 4557 char *field_str) 4558 { 4559 struct trace_array *tr = hist_data->event_file->tr; 4560 struct hist_field *hist_field = NULL; 4561 unsigned long flags = 0; 4562 unsigned int key_size; 4563 int ret = 0, n_subexprs = 0; 4564 4565 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4566 return -EINVAL; 4567 4568 flags |= HIST_FIELD_FL_KEY; 4569 4570 if (strcmp(field_str, "stacktrace") == 0) { 4571 flags |= HIST_FIELD_FL_STACKTRACE; 4572 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4573 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4574 } else { 4575 hist_field = parse_expr(hist_data, file, field_str, flags, 4576 NULL, &n_subexprs); 4577 if (IS_ERR(hist_field)) { 4578 ret = PTR_ERR(hist_field); 4579 goto out; 4580 } 4581 4582 if (field_has_hist_vars(hist_field, 0)) { 4583 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4584 destroy_hist_field(hist_field, 0); 4585 ret = -EINVAL; 4586 goto out; 4587 } 4588 4589 key_size = hist_field->size; 4590 } 4591 4592 hist_data->fields[key_idx] = hist_field; 4593 4594 key_size = ALIGN(key_size, sizeof(u64)); 4595 hist_data->fields[key_idx]->size = key_size; 4596 hist_data->fields[key_idx]->offset = key_offset; 4597 4598 hist_data->key_size += key_size; 4599 4600 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4601 ret = -EINVAL; 4602 goto out; 4603 } 4604 4605 hist_data->n_keys++; 4606 hist_data->n_fields++; 4607 4608 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4609 return -EINVAL; 4610 4611 ret = key_size; 4612 out: 4613 return ret; 4614 } 4615 4616 static int create_key_fields(struct hist_trigger_data *hist_data, 4617 struct trace_event_file *file) 4618 { 4619 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4620 char *fields_str, *field_str; 4621 int ret = -EINVAL; 4622 4623 fields_str = hist_data->attrs->keys_str; 4624 if (!fields_str) 4625 goto out; 4626 4627 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4628 field_str = strsep(&fields_str, ","); 4629 if (!field_str) 4630 break; 4631 ret = create_key_field(hist_data, i, key_offset, 4632 file, field_str); 4633 if (ret < 0) 4634 goto out; 4635 key_offset += ret; 4636 } 4637 if (fields_str) { 4638 ret = -EINVAL; 4639 goto out; 4640 } 4641 ret = 0; 4642 out: 4643 return ret; 4644 } 4645 4646 static int create_var_fields(struct hist_trigger_data *hist_data, 4647 struct trace_event_file *file) 4648 { 4649 unsigned int i, j = hist_data->n_vals; 4650 int ret = 0; 4651 4652 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4653 4654 for (i = 0; i < n_vars; i++) { 4655 char *var_name = hist_data->attrs->var_defs.name[i]; 4656 char *expr = hist_data->attrs->var_defs.expr[i]; 4657 4658 ret = create_var_field(hist_data, j++, file, var_name, expr); 4659 if (ret) 4660 goto out; 4661 } 4662 out: 4663 return ret; 4664 } 4665 4666 static void free_var_defs(struct hist_trigger_data *hist_data) 4667 { 4668 unsigned int i; 4669 4670 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4671 kfree(hist_data->attrs->var_defs.name[i]); 4672 kfree(hist_data->attrs->var_defs.expr[i]); 4673 } 4674 4675 hist_data->attrs->var_defs.n_vars = 0; 4676 } 4677 4678 static int parse_var_defs(struct hist_trigger_data *hist_data) 4679 { 4680 struct trace_array *tr = hist_data->event_file->tr; 4681 char *s, *str, *var_name, *field_str; 4682 unsigned int i, j, n_vars = 0; 4683 int ret = 0; 4684 4685 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4686 str = hist_data->attrs->assignment_str[i]; 4687 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4688 field_str = strsep(&str, ","); 4689 if (!field_str) 4690 break; 4691 4692 var_name = strsep(&field_str, "="); 4693 if (!var_name || !field_str) { 4694 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4695 errpos(var_name)); 4696 ret = -EINVAL; 4697 goto free; 4698 } 4699 4700 if (n_vars == TRACING_MAP_VARS_MAX) { 4701 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4702 ret = -EINVAL; 4703 goto free; 4704 } 4705 4706 s = kstrdup(var_name, GFP_KERNEL); 4707 if (!s) { 4708 ret = -ENOMEM; 4709 goto free; 4710 } 4711 hist_data->attrs->var_defs.name[n_vars] = s; 4712 4713 s = kstrdup(field_str, GFP_KERNEL); 4714 if (!s) { 4715 kfree(hist_data->attrs->var_defs.name[n_vars]); 4716 hist_data->attrs->var_defs.name[n_vars] = NULL; 4717 ret = -ENOMEM; 4718 goto free; 4719 } 4720 hist_data->attrs->var_defs.expr[n_vars++] = s; 4721 4722 hist_data->attrs->var_defs.n_vars = n_vars; 4723 } 4724 } 4725 4726 return ret; 4727 free: 4728 free_var_defs(hist_data); 4729 4730 return ret; 4731 } 4732 4733 static int create_hist_fields(struct hist_trigger_data *hist_data, 4734 struct trace_event_file *file) 4735 { 4736 int ret; 4737 4738 ret = parse_var_defs(hist_data); 4739 if (ret) 4740 return ret; 4741 4742 ret = create_val_fields(hist_data, file); 4743 if (ret) 4744 goto out; 4745 4746 ret = create_var_fields(hist_data, file); 4747 if (ret) 4748 goto out; 4749 4750 ret = create_key_fields(hist_data, file); 4751 4752 out: 4753 free_var_defs(hist_data); 4754 4755 return ret; 4756 } 4757 4758 static int is_descending(struct trace_array *tr, const char *str) 4759 { 4760 if (!str) 4761 return 0; 4762 4763 if (strcmp(str, "descending") == 0) 4764 return 1; 4765 4766 if (strcmp(str, "ascending") == 0) 4767 return 0; 4768 4769 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 4770 4771 return -EINVAL; 4772 } 4773 4774 static int create_sort_keys(struct hist_trigger_data *hist_data) 4775 { 4776 struct trace_array *tr = hist_data->event_file->tr; 4777 char *fields_str = hist_data->attrs->sort_key_str; 4778 struct tracing_map_sort_key *sort_key; 4779 int descending, ret = 0; 4780 unsigned int i, j, k; 4781 4782 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4783 4784 if (!fields_str) 4785 goto out; 4786 4787 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4788 struct hist_field *hist_field; 4789 char *field_str, *field_name; 4790 const char *test_name; 4791 4792 sort_key = &hist_data->sort_keys[i]; 4793 4794 field_str = strsep(&fields_str, ","); 4795 if (!field_str) 4796 break; 4797 4798 if (!*field_str) { 4799 ret = -EINVAL; 4800 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4801 break; 4802 } 4803 4804 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4805 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 4806 ret = -EINVAL; 4807 break; 4808 } 4809 4810 field_name = strsep(&field_str, "."); 4811 if (!field_name || !*field_name) { 4812 ret = -EINVAL; 4813 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4814 break; 4815 } 4816 4817 if (strcmp(field_name, "hitcount") == 0) { 4818 descending = is_descending(tr, field_str); 4819 if (descending < 0) { 4820 ret = descending; 4821 break; 4822 } 4823 sort_key->descending = descending; 4824 continue; 4825 } 4826 4827 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4828 unsigned int idx; 4829 4830 hist_field = hist_data->fields[j]; 4831 if (hist_field->flags & HIST_FIELD_FL_VAR) 4832 continue; 4833 4834 idx = k++; 4835 4836 test_name = hist_field_name(hist_field, 0); 4837 4838 if (strcmp(field_name, test_name) == 0) { 4839 sort_key->field_idx = idx; 4840 descending = is_descending(tr, field_str); 4841 if (descending < 0) { 4842 ret = descending; 4843 goto out; 4844 } 4845 sort_key->descending = descending; 4846 break; 4847 } 4848 } 4849 if (j == hist_data->n_fields) { 4850 ret = -EINVAL; 4851 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4852 break; 4853 } 4854 } 4855 4856 hist_data->n_sort_keys = i; 4857 out: 4858 return ret; 4859 } 4860 4861 static void destroy_actions(struct hist_trigger_data *hist_data) 4862 { 4863 unsigned int i; 4864 4865 for (i = 0; i < hist_data->n_actions; i++) { 4866 struct action_data *data = hist_data->actions[i]; 4867 4868 if (data->handler == HANDLER_ONMATCH) 4869 onmatch_destroy(data); 4870 else if (data->handler == HANDLER_ONMAX || 4871 data->handler == HANDLER_ONCHANGE) 4872 track_data_destroy(hist_data, data); 4873 else 4874 kfree(data); 4875 } 4876 } 4877 4878 static int parse_actions(struct hist_trigger_data *hist_data) 4879 { 4880 struct trace_array *tr = hist_data->event_file->tr; 4881 struct action_data *data; 4882 unsigned int i; 4883 int ret = 0; 4884 char *str; 4885 int len; 4886 4887 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4888 enum handler_id hid = 0; 4889 char *action_str; 4890 4891 str = hist_data->attrs->action_str[i]; 4892 4893 if ((len = str_has_prefix(str, "onmatch("))) 4894 hid = HANDLER_ONMATCH; 4895 else if ((len = str_has_prefix(str, "onmax("))) 4896 hid = HANDLER_ONMAX; 4897 else if ((len = str_has_prefix(str, "onchange("))) 4898 hid = HANDLER_ONCHANGE; 4899 4900 action_str = str + len; 4901 4902 switch (hid) { 4903 case HANDLER_ONMATCH: 4904 data = onmatch_parse(tr, action_str); 4905 break; 4906 case HANDLER_ONMAX: 4907 case HANDLER_ONCHANGE: 4908 data = track_data_parse(hist_data, action_str, hid); 4909 break; 4910 default: 4911 data = ERR_PTR(-EINVAL); 4912 break; 4913 } 4914 4915 if (IS_ERR(data)) { 4916 ret = PTR_ERR(data); 4917 break; 4918 } 4919 4920 hist_data->actions[hist_data->n_actions++] = data; 4921 } 4922 4923 return ret; 4924 } 4925 4926 static int create_actions(struct hist_trigger_data *hist_data) 4927 { 4928 struct action_data *data; 4929 unsigned int i; 4930 int ret = 0; 4931 4932 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4933 data = hist_data->actions[i]; 4934 4935 if (data->handler == HANDLER_ONMATCH) { 4936 ret = onmatch_create(hist_data, data); 4937 if (ret) 4938 break; 4939 } else if (data->handler == HANDLER_ONMAX || 4940 data->handler == HANDLER_ONCHANGE) { 4941 ret = track_data_create(hist_data, data); 4942 if (ret) 4943 break; 4944 } else { 4945 ret = -EINVAL; 4946 break; 4947 } 4948 } 4949 4950 return ret; 4951 } 4952 4953 static void print_actions(struct seq_file *m, 4954 struct hist_trigger_data *hist_data, 4955 struct tracing_map_elt *elt) 4956 { 4957 unsigned int i; 4958 4959 for (i = 0; i < hist_data->n_actions; i++) { 4960 struct action_data *data = hist_data->actions[i]; 4961 4962 if (data->action == ACTION_SNAPSHOT) 4963 continue; 4964 4965 if (data->handler == HANDLER_ONMAX || 4966 data->handler == HANDLER_ONCHANGE) 4967 track_data_print(m, hist_data, elt, data); 4968 } 4969 } 4970 4971 static void print_action_spec(struct seq_file *m, 4972 struct hist_trigger_data *hist_data, 4973 struct action_data *data) 4974 { 4975 unsigned int i; 4976 4977 if (data->action == ACTION_SAVE) { 4978 for (i = 0; i < hist_data->n_save_vars; i++) { 4979 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4980 if (i < hist_data->n_save_vars - 1) 4981 seq_puts(m, ","); 4982 } 4983 } else if (data->action == ACTION_TRACE) { 4984 if (data->use_trace_keyword) 4985 seq_printf(m, "%s", data->synth_event_name); 4986 for (i = 0; i < data->n_params; i++) { 4987 if (i || data->use_trace_keyword) 4988 seq_puts(m, ","); 4989 seq_printf(m, "%s", data->params[i]); 4990 } 4991 } 4992 } 4993 4994 static void print_track_data_spec(struct seq_file *m, 4995 struct hist_trigger_data *hist_data, 4996 struct action_data *data) 4997 { 4998 if (data->handler == HANDLER_ONMAX) 4999 seq_puts(m, ":onmax("); 5000 else if (data->handler == HANDLER_ONCHANGE) 5001 seq_puts(m, ":onchange("); 5002 seq_printf(m, "%s", data->track_data.var_str); 5003 seq_printf(m, ").%s(", data->action_name); 5004 5005 print_action_spec(m, hist_data, data); 5006 5007 seq_puts(m, ")"); 5008 } 5009 5010 static void print_onmatch_spec(struct seq_file *m, 5011 struct hist_trigger_data *hist_data, 5012 struct action_data *data) 5013 { 5014 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 5015 data->match_data.event); 5016 5017 seq_printf(m, "%s(", data->action_name); 5018 5019 print_action_spec(m, hist_data, data); 5020 5021 seq_puts(m, ")"); 5022 } 5023 5024 static bool actions_match(struct hist_trigger_data *hist_data, 5025 struct hist_trigger_data *hist_data_test) 5026 { 5027 unsigned int i, j; 5028 5029 if (hist_data->n_actions != hist_data_test->n_actions) 5030 return false; 5031 5032 for (i = 0; i < hist_data->n_actions; i++) { 5033 struct action_data *data = hist_data->actions[i]; 5034 struct action_data *data_test = hist_data_test->actions[i]; 5035 char *action_name, *action_name_test; 5036 5037 if (data->handler != data_test->handler) 5038 return false; 5039 if (data->action != data_test->action) 5040 return false; 5041 5042 if (data->n_params != data_test->n_params) 5043 return false; 5044 5045 for (j = 0; j < data->n_params; j++) { 5046 if (strcmp(data->params[j], data_test->params[j]) != 0) 5047 return false; 5048 } 5049 5050 if (data->use_trace_keyword) 5051 action_name = data->synth_event_name; 5052 else 5053 action_name = data->action_name; 5054 5055 if (data_test->use_trace_keyword) 5056 action_name_test = data_test->synth_event_name; 5057 else 5058 action_name_test = data_test->action_name; 5059 5060 if (strcmp(action_name, action_name_test) != 0) 5061 return false; 5062 5063 if (data->handler == HANDLER_ONMATCH) { 5064 if (strcmp(data->match_data.event_system, 5065 data_test->match_data.event_system) != 0) 5066 return false; 5067 if (strcmp(data->match_data.event, 5068 data_test->match_data.event) != 0) 5069 return false; 5070 } else if (data->handler == HANDLER_ONMAX || 5071 data->handler == HANDLER_ONCHANGE) { 5072 if (strcmp(data->track_data.var_str, 5073 data_test->track_data.var_str) != 0) 5074 return false; 5075 } 5076 } 5077 5078 return true; 5079 } 5080 5081 5082 static void print_actions_spec(struct seq_file *m, 5083 struct hist_trigger_data *hist_data) 5084 { 5085 unsigned int i; 5086 5087 for (i = 0; i < hist_data->n_actions; i++) { 5088 struct action_data *data = hist_data->actions[i]; 5089 5090 if (data->handler == HANDLER_ONMATCH) 5091 print_onmatch_spec(m, hist_data, data); 5092 else if (data->handler == HANDLER_ONMAX || 5093 data->handler == HANDLER_ONCHANGE) 5094 print_track_data_spec(m, hist_data, data); 5095 } 5096 } 5097 5098 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5099 { 5100 unsigned int i; 5101 5102 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5103 kfree(hist_data->field_var_hists[i]->cmd); 5104 kfree(hist_data->field_var_hists[i]); 5105 } 5106 } 5107 5108 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5109 { 5110 if (!hist_data) 5111 return; 5112 5113 destroy_hist_trigger_attrs(hist_data->attrs); 5114 destroy_hist_fields(hist_data); 5115 tracing_map_destroy(hist_data->map); 5116 5117 destroy_actions(hist_data); 5118 destroy_field_vars(hist_data); 5119 destroy_field_var_hists(hist_data); 5120 5121 kfree(hist_data); 5122 } 5123 5124 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5125 { 5126 struct tracing_map *map = hist_data->map; 5127 struct ftrace_event_field *field; 5128 struct hist_field *hist_field; 5129 int i, idx = 0; 5130 5131 for_each_hist_field(i, hist_data) { 5132 hist_field = hist_data->fields[i]; 5133 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5134 tracing_map_cmp_fn_t cmp_fn; 5135 5136 field = hist_field->field; 5137 5138 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5139 cmp_fn = tracing_map_cmp_none; 5140 else if (!field || hist_field->flags & HIST_FIELD_FL_CPU) 5141 cmp_fn = tracing_map_cmp_num(hist_field->size, 5142 hist_field->is_signed); 5143 else if (is_string_field(field)) 5144 cmp_fn = tracing_map_cmp_string; 5145 else 5146 cmp_fn = tracing_map_cmp_num(field->size, 5147 field->is_signed); 5148 idx = tracing_map_add_key_field(map, 5149 hist_field->offset, 5150 cmp_fn); 5151 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5152 idx = tracing_map_add_sum_field(map); 5153 5154 if (idx < 0) 5155 return idx; 5156 5157 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5158 idx = tracing_map_add_var(map); 5159 if (idx < 0) 5160 return idx; 5161 hist_field->var.idx = idx; 5162 hist_field->var.hist_data = hist_data; 5163 } 5164 } 5165 5166 return 0; 5167 } 5168 5169 static struct hist_trigger_data * 5170 create_hist_data(unsigned int map_bits, 5171 struct hist_trigger_attrs *attrs, 5172 struct trace_event_file *file, 5173 bool remove) 5174 { 5175 const struct tracing_map_ops *map_ops = NULL; 5176 struct hist_trigger_data *hist_data; 5177 int ret = 0; 5178 5179 hist_data = kzalloc_obj(*hist_data); 5180 if (!hist_data) 5181 return ERR_PTR(-ENOMEM); 5182 5183 hist_data->attrs = attrs; 5184 hist_data->remove = remove; 5185 hist_data->event_file = file; 5186 5187 ret = parse_actions(hist_data); 5188 if (ret) 5189 goto free; 5190 5191 ret = create_hist_fields(hist_data, file); 5192 if (ret) 5193 goto free; 5194 5195 ret = create_sort_keys(hist_data); 5196 if (ret) 5197 goto free; 5198 5199 map_ops = &hist_trigger_elt_data_ops; 5200 5201 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5202 map_ops, hist_data); 5203 if (IS_ERR(hist_data->map)) { 5204 ret = PTR_ERR(hist_data->map); 5205 hist_data->map = NULL; 5206 goto free; 5207 } 5208 5209 ret = create_tracing_map_fields(hist_data); 5210 if (ret) 5211 goto free; 5212 out: 5213 return hist_data; 5214 free: 5215 hist_data->attrs = NULL; 5216 5217 destroy_hist_data(hist_data); 5218 5219 hist_data = ERR_PTR(ret); 5220 5221 goto out; 5222 } 5223 5224 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5225 struct tracing_map_elt *elt, 5226 struct trace_buffer *buffer, void *rec, 5227 struct ring_buffer_event *rbe, 5228 u64 *var_ref_vals) 5229 { 5230 struct hist_elt_data *elt_data; 5231 struct hist_field *hist_field; 5232 unsigned int i, var_idx; 5233 u64 hist_val; 5234 5235 elt_data = elt->private_data; 5236 elt_data->var_ref_vals = var_ref_vals; 5237 5238 for_each_hist_val_field(i, hist_data) { 5239 hist_field = hist_data->fields[i]; 5240 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5241 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5242 var_idx = hist_field->var.idx; 5243 5244 if (!(hist_field->flags & HIST_FIELD_FL_ALIAS) && 5245 hist_field->flags & 5246 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) { 5247 unsigned int str_start, var_str_idx, idx; 5248 char *str, *val_str; 5249 unsigned int size; 5250 5251 str_start = hist_data->n_field_var_str + 5252 hist_data->n_save_var_str; 5253 var_str_idx = hist_field->var_str_idx; 5254 idx = str_start + var_str_idx; 5255 5256 str = elt_data->field_var_str[idx]; 5257 val_str = (char *)(uintptr_t)hist_val; 5258 5259 if (hist_field->flags & HIST_FIELD_FL_STRING) { 5260 size = min(hist_field->size, STR_VAR_LEN_MAX); 5261 strscpy(str, val_str, size); 5262 } else { 5263 char *stack_start = str + sizeof(unsigned long); 5264 int e; 5265 5266 e = stack_trace_save((void *)stack_start, 5267 HIST_STACKTRACE_DEPTH, 5268 HIST_STACKTRACE_SKIP); 5269 if (e < HIST_STACKTRACE_DEPTH - 1) 5270 ((unsigned long *)stack_start)[e] = 0; 5271 *((unsigned long *)str) = e; 5272 } 5273 hist_val = (u64)(uintptr_t)str; 5274 } 5275 tracing_map_set_var(elt, var_idx, hist_val); 5276 continue; 5277 } 5278 tracing_map_update_sum(elt, i, hist_val); 5279 } 5280 5281 for_each_hist_key_field(i, hist_data) { 5282 hist_field = hist_data->fields[i]; 5283 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5284 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5285 var_idx = hist_field->var.idx; 5286 tracing_map_set_var(elt, var_idx, hist_val); 5287 } 5288 } 5289 5290 update_field_vars(hist_data, elt, buffer, rbe, rec); 5291 } 5292 5293 static inline void add_to_key(char *compound_key, void *key, 5294 struct hist_field *key_field, void *rec) 5295 { 5296 size_t size = key_field->size; 5297 5298 if (key_field->flags & HIST_FIELD_FL_STRING) { 5299 5300 if (key_field->flags & HIST_FIELD_FL_COMM) { 5301 size = strlen((char *)key); 5302 } else { 5303 struct ftrace_event_field *field; 5304 5305 field = key_field->field; 5306 if (field->filter_type == FILTER_DYN_STRING || 5307 field->filter_type == FILTER_RDYN_STRING) 5308 size = *(u32 *)(rec + field->offset) >> 16; 5309 else if (field->filter_type == FILTER_STATIC_STRING) 5310 size = field->size; 5311 } 5312 5313 /* ensure NULL-termination */ 5314 if (size > key_field->size - 1) 5315 size = key_field->size - 1; 5316 } 5317 memcpy(compound_key + key_field->offset, key, size); 5318 } 5319 5320 static void 5321 hist_trigger_actions(struct hist_trigger_data *hist_data, 5322 struct tracing_map_elt *elt, 5323 struct trace_buffer *buffer, void *rec, 5324 struct ring_buffer_event *rbe, void *key, 5325 u64 *var_ref_vals) 5326 { 5327 struct action_data *data; 5328 unsigned int i; 5329 5330 for (i = 0; i < hist_data->n_actions; i++) { 5331 data = hist_data->actions[i]; 5332 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals); 5333 } 5334 } 5335 5336 /* 5337 * The hist_pad structure is used to save information to create 5338 * a histogram from the histogram trigger. It's too big to store 5339 * on the stack, so when the histogram trigger is initialized 5340 * a percpu array of 4 hist_pad structures is allocated. 5341 * This will cover every context from normal, softirq, irq and NMI 5342 * in the very unlikely event that a trigger happens at each of 5343 * these contexts and interrupts a currently active trigger. 5344 */ 5345 struct hist_pad { 5346 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5347 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5348 char compound_key[HIST_KEY_SIZE_MAX]; 5349 }; 5350 5351 static struct hist_pad __percpu *hist_pads; 5352 static DEFINE_PER_CPU(int, hist_pad_cnt); 5353 static refcount_t hist_pad_ref; 5354 5355 /* One hist_pad for every context (normal, softirq, irq, NMI) */ 5356 #define MAX_HIST_CNT 4 5357 5358 static int alloc_hist_pad(void) 5359 { 5360 lockdep_assert_held(&event_mutex); 5361 5362 if (refcount_read(&hist_pad_ref)) { 5363 refcount_inc(&hist_pad_ref); 5364 return 0; 5365 } 5366 5367 hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT, 5368 __alignof__(struct hist_pad)); 5369 if (!hist_pads) 5370 return -ENOMEM; 5371 5372 refcount_set(&hist_pad_ref, 1); 5373 return 0; 5374 } 5375 5376 static void free_hist_pad(void) 5377 { 5378 lockdep_assert_held(&event_mutex); 5379 5380 if (!refcount_dec_and_test(&hist_pad_ref)) 5381 return; 5382 5383 free_percpu(hist_pads); 5384 hist_pads = NULL; 5385 } 5386 5387 static struct hist_pad *get_hist_pad(void) 5388 { 5389 struct hist_pad *hist_pad; 5390 int cnt; 5391 5392 if (WARN_ON_ONCE(!hist_pads)) 5393 return NULL; 5394 5395 preempt_disable(); 5396 5397 hist_pad = per_cpu_ptr(hist_pads, smp_processor_id()); 5398 5399 if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) { 5400 preempt_enable(); 5401 return NULL; 5402 } 5403 5404 cnt = this_cpu_inc_return(hist_pad_cnt) - 1; 5405 5406 return &hist_pad[cnt]; 5407 } 5408 5409 static void put_hist_pad(void) 5410 { 5411 this_cpu_dec(hist_pad_cnt); 5412 preempt_enable(); 5413 } 5414 5415 static void event_hist_trigger(struct event_trigger_data *data, 5416 struct trace_buffer *buffer, void *rec, 5417 struct ring_buffer_event *rbe) 5418 { 5419 struct hist_trigger_data *hist_data = data->private_data; 5420 bool use_compound_key = (hist_data->n_keys > 1); 5421 struct tracing_map_elt *elt = NULL; 5422 struct hist_field *key_field; 5423 struct hist_pad *hist_pad; 5424 u64 field_contents; 5425 void *key = NULL; 5426 unsigned int i; 5427 5428 if (unlikely(!rbe)) 5429 return; 5430 5431 hist_pad = get_hist_pad(); 5432 if (!hist_pad) 5433 return; 5434 5435 memset(hist_pad->compound_key, 0, hist_data->key_size); 5436 5437 for_each_hist_key_field(i, hist_data) { 5438 key_field = hist_data->fields[i]; 5439 5440 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5441 unsigned long *entries = hist_pad->entries; 5442 5443 memset(entries, 0, HIST_STACKTRACE_SIZE); 5444 if (key_field->field) { 5445 unsigned long *stack, n_entries; 5446 5447 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5448 stack = (unsigned long *)(long)field_contents; 5449 n_entries = *stack; 5450 memcpy(entries, ++stack, n_entries * sizeof(unsigned long)); 5451 } else { 5452 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5453 HIST_STACKTRACE_SKIP); 5454 } 5455 key = entries; 5456 } else { 5457 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5458 if (key_field->flags & HIST_FIELD_FL_STRING) { 5459 key = (void *)(unsigned long)field_contents; 5460 use_compound_key = true; 5461 } else 5462 key = (void *)&field_contents; 5463 } 5464 5465 if (use_compound_key) 5466 add_to_key(hist_pad->compound_key, key, key_field, rec); 5467 } 5468 5469 if (use_compound_key) 5470 key = hist_pad->compound_key; 5471 5472 if (hist_data->n_var_refs && 5473 !resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false)) 5474 goto out; 5475 5476 elt = tracing_map_insert(hist_data->map, key); 5477 if (!elt) 5478 goto out; 5479 5480 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals); 5481 5482 if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) { 5483 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, 5484 key, hist_pad->var_ref_vals); 5485 } 5486 5487 hist_poll_wakeup(); 5488 5489 out: 5490 put_hist_pad(); 5491 } 5492 5493 static void hist_trigger_stacktrace_print(struct seq_file *m, 5494 unsigned long *stacktrace_entries, 5495 unsigned int max_entries) 5496 { 5497 unsigned int spaces = 8; 5498 unsigned int i; 5499 5500 for (i = 0; i < max_entries; i++) { 5501 if (!stacktrace_entries[i]) 5502 return; 5503 5504 seq_printf(m, "%*c", 1 + spaces, ' '); 5505 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]); 5506 } 5507 } 5508 5509 static void hist_trigger_print_key(struct seq_file *m, 5510 struct hist_trigger_data *hist_data, 5511 void *key, 5512 struct tracing_map_elt *elt) 5513 { 5514 struct hist_field *key_field; 5515 bool multiline = false; 5516 const char *field_name; 5517 unsigned int i; 5518 u64 uval; 5519 5520 seq_puts(m, "{ "); 5521 5522 for_each_hist_key_field(i, hist_data) { 5523 key_field = hist_data->fields[i]; 5524 5525 if (i > hist_data->n_vals) 5526 seq_puts(m, ", "); 5527 5528 field_name = hist_field_name(key_field, 0); 5529 5530 if (key_field->flags & HIST_FIELD_FL_HEX) { 5531 uval = *(u64 *)(key + key_field->offset); 5532 seq_printf(m, "%s: %llx", field_name, uval); 5533 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5534 uval = *(u64 *)(key + key_field->offset); 5535 seq_printf(m, "%s: [%llx] %-45ps", field_name, 5536 uval, (void *)(uintptr_t)uval); 5537 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5538 uval = *(u64 *)(key + key_field->offset); 5539 seq_printf(m, "%s: [%llx] %-55pS", field_name, 5540 uval, (void *)(uintptr_t)uval); 5541 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5542 struct hist_elt_data *elt_data = elt->private_data; 5543 char *comm; 5544 5545 if (WARN_ON_ONCE(!elt_data)) 5546 return; 5547 5548 comm = elt_data->comm; 5549 5550 uval = *(u64 *)(key + key_field->offset); 5551 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5552 comm, uval); 5553 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5554 const char *syscall_name; 5555 5556 uval = *(u64 *)(key + key_field->offset); 5557 syscall_name = get_syscall_name(uval); 5558 if (!syscall_name) 5559 syscall_name = "unknown_syscall"; 5560 5561 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5562 syscall_name, uval); 5563 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5564 if (key_field->field) 5565 seq_printf(m, "%s.stacktrace", key_field->field->name); 5566 else 5567 seq_puts(m, "common_stacktrace:\n"); 5568 hist_trigger_stacktrace_print(m, 5569 key + key_field->offset, 5570 HIST_STACKTRACE_DEPTH); 5571 multiline = true; 5572 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5573 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5574 *(u64 *)(key + key_field->offset)); 5575 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) { 5576 unsigned long buckets = key_field->buckets; 5577 u64 end; 5578 5579 uval = *(u64 *)(key + key_field->offset); 5580 end = uval + buckets - 1; 5581 if (end < uval) 5582 end = U64_MAX; 5583 seq_printf(m, "%s: ~ %llu-%llu", field_name, 5584 uval, end); 5585 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5586 seq_printf(m, "%s: %-50s", field_name, 5587 (char *)(key + key_field->offset)); 5588 } else { 5589 uval = *(u64 *)(key + key_field->offset); 5590 if (hist_field_is_signed(key_field)) 5591 seq_printf(m, "%s: %10lld", field_name, 5592 (s64)uval); 5593 else 5594 seq_printf(m, "%s: %10llu", field_name, uval); 5595 } 5596 } 5597 5598 if (!multiline) 5599 seq_puts(m, " "); 5600 5601 seq_puts(m, "}"); 5602 } 5603 5604 /* Get the 100 times of the percentage of @val in @total */ 5605 static inline unsigned int __get_percentage(u64 val, u64 total) 5606 { 5607 if (!total) 5608 goto div0; 5609 5610 if (val < (U64_MAX / 10000)) 5611 return (unsigned int)div64_ul(val * 10000, total); 5612 5613 total = div64_u64(total, 10000); 5614 if (!total) 5615 goto div0; 5616 5617 return (unsigned int)div64_ul(val, total); 5618 div0: 5619 return val ? UINT_MAX : 0; 5620 } 5621 5622 #define BAR_CHAR '#' 5623 5624 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max) 5625 { 5626 unsigned int len = __get_percentage(val, max); 5627 int i; 5628 5629 if (len == UINT_MAX) { 5630 snprintf(buf, size, "[ERROR]"); 5631 return buf; 5632 } 5633 5634 len = len * size / 10000; 5635 for (i = 0; i < len && i < size; i++) 5636 buf[i] = BAR_CHAR; 5637 while (i < size) 5638 buf[i++] = ' '; 5639 buf[size] = '\0'; 5640 5641 return buf; 5642 } 5643 5644 struct hist_val_stat { 5645 u64 max; 5646 u64 total; 5647 }; 5648 5649 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx, 5650 const char *field_name, unsigned long flags, 5651 struct hist_val_stat *stats, 5652 struct tracing_map_elt *elt) 5653 { 5654 u64 val = tracing_map_read_sum(elt, idx); 5655 unsigned int pc; 5656 char bar[21]; 5657 5658 if (flags & HIST_FIELD_FL_PERCENT) { 5659 pc = __get_percentage(val, stats[idx].total); 5660 if (pc == UINT_MAX) 5661 seq_printf(m, " %s (%%):[ERROR]", field_name); 5662 else 5663 seq_printf(m, " %s (%%): %3u.%02u", field_name, 5664 pc / 100, pc % 100); 5665 } else if (flags & HIST_FIELD_FL_GRAPH) { 5666 seq_printf(m, " %s: %20s", field_name, 5667 __fill_bar_str(bar, 20, val, stats[idx].max)); 5668 } else if (flags & HIST_FIELD_FL_HEX) { 5669 seq_printf(m, " %s: %10llx", field_name, val); 5670 } else { 5671 seq_printf(m, " %s: %10llu", field_name, val); 5672 } 5673 } 5674 5675 static void hist_trigger_entry_print(struct seq_file *m, 5676 struct hist_trigger_data *hist_data, 5677 struct hist_val_stat *stats, 5678 void *key, 5679 struct tracing_map_elt *elt) 5680 { 5681 const char *field_name; 5682 unsigned int i = HITCOUNT_IDX; 5683 unsigned long flags; 5684 5685 hist_trigger_print_key(m, hist_data, key, elt); 5686 5687 /* At first, show the raw hitcount if !nohitcount */ 5688 if (!hist_data->attrs->no_hitcount) 5689 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt); 5690 5691 for (i = 1; i < hist_data->n_vals; i++) { 5692 field_name = hist_field_name(hist_data->fields[i], 0); 5693 flags = hist_data->fields[i]->flags; 5694 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR) 5695 continue; 5696 5697 seq_puts(m, " "); 5698 hist_trigger_print_val(m, i, field_name, flags, stats, elt); 5699 } 5700 5701 print_actions(m, hist_data, elt); 5702 5703 seq_puts(m, "\n"); 5704 } 5705 5706 static int print_entries(struct seq_file *m, 5707 struct hist_trigger_data *hist_data) 5708 { 5709 struct tracing_map_sort_entry **sort_entries = NULL; 5710 struct tracing_map *map = hist_data->map; 5711 int i, j, n_entries, ret; 5712 struct hist_val_stat *stats = NULL; 5713 u64 val; 5714 5715 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5716 hist_data->n_sort_keys, 5717 &sort_entries); 5718 if (n_entries < 0) 5719 return n_entries; 5720 5721 ret = n_entries; 5722 5723 /* Calculate the max and the total for each field if needed. */ 5724 for (j = 0; j < hist_data->n_vals; j++) { 5725 if (!(hist_data->fields[j]->flags & 5726 (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH))) 5727 continue; 5728 if (!stats) { 5729 stats = kzalloc_objs(*stats, hist_data->n_vals); 5730 if (!stats) { 5731 ret = -ENOMEM; 5732 goto out; 5733 } 5734 } 5735 for (i = 0; i < n_entries; i++) { 5736 val = tracing_map_read_sum(sort_entries[i]->elt, j); 5737 stats[j].total += val; 5738 if (stats[j].max < val) 5739 stats[j].max = val; 5740 } 5741 } 5742 5743 for (i = 0; i < n_entries; i++) 5744 hist_trigger_entry_print(m, hist_data, stats, 5745 sort_entries[i]->key, 5746 sort_entries[i]->elt); 5747 5748 kfree(stats); 5749 out: 5750 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5751 5752 return ret; 5753 } 5754 5755 static void hist_trigger_show(struct seq_file *m, 5756 struct event_trigger_data *data, int n) 5757 { 5758 struct hist_trigger_data *hist_data; 5759 int n_entries; 5760 5761 if (n > 0) 5762 seq_puts(m, "\n\n"); 5763 5764 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5765 data->cmd_ops->print(m, data); 5766 seq_puts(m, "#\n\n"); 5767 5768 hist_data = data->private_data; 5769 n_entries = print_entries(m, hist_data); 5770 if (n_entries < 0) 5771 n_entries = 0; 5772 5773 track_data_snapshot_print(m, hist_data); 5774 5775 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5776 (u64)atomic64_read(&hist_data->map->hits), 5777 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5778 } 5779 5780 struct hist_file_data { 5781 struct file *file; 5782 u64 last_read; 5783 u64 last_act; 5784 }; 5785 5786 static u64 get_hist_hit_count(struct trace_event_file *event_file) 5787 { 5788 struct hist_trigger_data *hist_data; 5789 struct event_trigger_data *data; 5790 u64 ret = 0; 5791 5792 list_for_each_entry(data, &event_file->triggers, list) { 5793 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5794 hist_data = data->private_data; 5795 ret += atomic64_read(&hist_data->map->hits); 5796 } 5797 } 5798 return ret; 5799 } 5800 5801 static int hist_show(struct seq_file *m, void *v) 5802 { 5803 struct hist_file_data *hist_file = m->private; 5804 struct event_trigger_data *data; 5805 struct trace_event_file *event_file; 5806 int n = 0; 5807 5808 guard(mutex)(&event_mutex); 5809 5810 event_file = event_file_file(hist_file->file); 5811 if (unlikely(!event_file)) 5812 return -ENODEV; 5813 5814 list_for_each_entry(data, &event_file->triggers, list) { 5815 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5816 hist_trigger_show(m, data, n++); 5817 } 5818 hist_file->last_read = get_hist_hit_count(event_file); 5819 /* 5820 * Update last_act too so that poll()/POLLPRI can wait for the next 5821 * event after any syscall on hist file. 5822 */ 5823 hist_file->last_act = hist_file->last_read; 5824 5825 return 0; 5826 } 5827 5828 static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wait) 5829 { 5830 struct trace_event_file *event_file; 5831 struct seq_file *m = file->private_data; 5832 struct hist_file_data *hist_file = m->private; 5833 __poll_t ret = 0; 5834 u64 cnt; 5835 5836 guard(mutex)(&event_mutex); 5837 5838 event_file = event_file_file(file); 5839 if (!event_file) 5840 return EPOLLERR; 5841 5842 hist_poll_wait(file, wait); 5843 5844 cnt = get_hist_hit_count(event_file); 5845 if (hist_file->last_read != cnt) 5846 ret |= EPOLLIN | EPOLLRDNORM; 5847 if (hist_file->last_act != cnt) { 5848 hist_file->last_act = cnt; 5849 ret |= EPOLLPRI; 5850 } 5851 5852 return ret; 5853 } 5854 5855 static int event_hist_release(struct inode *inode, struct file *file) 5856 { 5857 struct seq_file *m = file->private_data; 5858 struct hist_file_data *hist_file = m->private; 5859 5860 kfree(hist_file); 5861 return tracing_single_release_file_tr(inode, file); 5862 } 5863 5864 static int event_hist_open(struct inode *inode, struct file *file) 5865 { 5866 struct trace_event_file *event_file; 5867 struct hist_file_data *hist_file; 5868 int ret; 5869 5870 ret = tracing_open_file_tr(inode, file); 5871 if (ret) 5872 return ret; 5873 5874 guard(mutex)(&event_mutex); 5875 5876 event_file = event_file_file(file); 5877 if (!event_file) { 5878 ret = -ENODEV; 5879 goto err; 5880 } 5881 5882 hist_file = kzalloc_obj(*hist_file); 5883 if (!hist_file) { 5884 ret = -ENOMEM; 5885 goto err; 5886 } 5887 5888 hist_file->file = file; 5889 hist_file->last_act = get_hist_hit_count(event_file); 5890 5891 ret = single_open(file, hist_show, hist_file); 5892 if (ret) { 5893 kfree(hist_file); 5894 goto err; 5895 } 5896 5897 return 0; 5898 err: 5899 tracing_release_file_tr(inode, file); 5900 return ret; 5901 } 5902 5903 const struct file_operations event_hist_fops = { 5904 .open = event_hist_open, 5905 .read = seq_read, 5906 .llseek = seq_lseek, 5907 .release = event_hist_release, 5908 .poll = event_hist_poll, 5909 }; 5910 5911 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 5912 5913 #undef C 5914 #define C(a, b) b 5915 5916 static const char * const field_funcs[] = { FIELD_FUNCS }; 5917 5918 static void hist_field_debug_show_flags(struct seq_file *m, 5919 unsigned long flags) 5920 { 5921 seq_puts(m, " flags:\n"); 5922 5923 if (flags & HIST_FIELD_FL_KEY) 5924 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 5925 else if (flags & HIST_FIELD_FL_HITCOUNT) 5926 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 5927 else if (flags & HIST_FIELD_FL_VAR) 5928 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 5929 else if (flags & HIST_FIELD_FL_VAR_REF) 5930 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 5931 else 5932 seq_puts(m, " VAL: normal u64 value\n"); 5933 5934 if (flags & HIST_FIELD_FL_ALIAS) 5935 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 5936 else if (flags & HIST_FIELD_FL_CONST) 5937 seq_puts(m, " HIST_FIELD_FL_CONST\n"); 5938 } 5939 5940 static int hist_field_debug_show(struct seq_file *m, 5941 struct hist_field *field, unsigned long flags) 5942 { 5943 if ((field->flags & flags) != flags) { 5944 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 5945 return -EINVAL; 5946 } 5947 5948 hist_field_debug_show_flags(m, field->flags); 5949 if (field->field) 5950 seq_printf(m, " ftrace_event_field name: %s\n", 5951 field->field->name); 5952 5953 if (field->flags & HIST_FIELD_FL_VAR) { 5954 seq_printf(m, " var.name: %s\n", field->var.name); 5955 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5956 field->var.idx); 5957 } 5958 5959 if (field->flags & HIST_FIELD_FL_CONST) 5960 seq_printf(m, " constant: %llu\n", field->constant); 5961 5962 if (field->flags & HIST_FIELD_FL_ALIAS) 5963 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5964 field->var_ref_idx); 5965 5966 if (field->flags & HIST_FIELD_FL_VAR_REF) { 5967 seq_printf(m, " name: %s\n", field->name); 5968 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5969 field->var.idx); 5970 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 5971 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5972 field->var_ref_idx); 5973 if (field->system) 5974 seq_printf(m, " system: %s\n", field->system); 5975 if (field->event_name) 5976 seq_printf(m, " event_name: %s\n", field->event_name); 5977 } 5978 5979 seq_printf(m, " type: %s\n", field->type); 5980 seq_printf(m, " size: %u\n", field->size); 5981 seq_printf(m, " is_signed: %u\n", field->is_signed); 5982 seq_printf(m, " function: hist_field_%s()\n", field_funcs[field->fn_num]); 5983 5984 return 0; 5985 } 5986 5987 static int field_var_debug_show(struct seq_file *m, 5988 struct field_var *field_var, unsigned int i, 5989 bool save_vars) 5990 { 5991 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 5992 struct hist_field *field; 5993 int ret = 0; 5994 5995 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 5996 5997 field = field_var->var; 5998 5999 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 6000 6001 hist_field_debug_show_flags(m, field->flags); 6002 seq_printf(m, " var.name: %s\n", field->var.name); 6003 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 6004 field->var.idx); 6005 6006 field = field_var->val; 6007 6008 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 6009 if (field->field) 6010 seq_printf(m, " ftrace_event_field name: %s\n", 6011 field->field->name); 6012 else { 6013 ret = -EINVAL; 6014 goto out; 6015 } 6016 6017 seq_printf(m, " type: %s\n", field->type); 6018 seq_printf(m, " size: %u\n", field->size); 6019 seq_printf(m, " is_signed: %u\n", field->is_signed); 6020 out: 6021 return ret; 6022 } 6023 6024 static int hist_action_debug_show(struct seq_file *m, 6025 struct action_data *data, int i) 6026 { 6027 int ret = 0; 6028 6029 if (data->handler == HANDLER_ONMAX || 6030 data->handler == HANDLER_ONCHANGE) { 6031 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 6032 ret = hist_field_debug_show(m, data->track_data.var_ref, 6033 HIST_FIELD_FL_VAR_REF); 6034 if (ret) 6035 goto out; 6036 6037 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 6038 ret = hist_field_debug_show(m, data->track_data.track_var, 6039 HIST_FIELD_FL_VAR); 6040 if (ret) 6041 goto out; 6042 } 6043 6044 if (data->handler == HANDLER_ONMATCH) { 6045 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 6046 i, data->match_data.event_system); 6047 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 6048 i, data->match_data.event); 6049 } 6050 out: 6051 return ret; 6052 } 6053 6054 static int hist_actions_debug_show(struct seq_file *m, 6055 struct hist_trigger_data *hist_data) 6056 { 6057 int i, ret = 0; 6058 6059 if (hist_data->n_actions) 6060 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 6061 6062 for (i = 0; i < hist_data->n_actions; i++) { 6063 struct action_data *action = hist_data->actions[i]; 6064 6065 ret = hist_action_debug_show(m, action, i); 6066 if (ret) 6067 goto out; 6068 } 6069 6070 if (hist_data->n_save_vars) 6071 seq_puts(m, "\n save action variables (save() params):\n"); 6072 6073 for (i = 0; i < hist_data->n_save_vars; i++) { 6074 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 6075 if (ret) 6076 goto out; 6077 } 6078 out: 6079 return ret; 6080 } 6081 6082 static void hist_trigger_debug_show(struct seq_file *m, 6083 struct event_trigger_data *data, int n) 6084 { 6085 struct hist_trigger_data *hist_data; 6086 int i, ret; 6087 6088 if (n > 0) 6089 seq_puts(m, "\n\n"); 6090 6091 seq_puts(m, "# event histogram\n#\n# trigger info: "); 6092 data->cmd_ops->print(m, data); 6093 seq_puts(m, "#\n\n"); 6094 6095 hist_data = data->private_data; 6096 6097 seq_printf(m, "hist_data: %p\n\n", hist_data); 6098 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 6099 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 6100 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 6101 6102 seq_puts(m, "\n val fields:\n\n"); 6103 6104 seq_puts(m, " hist_data->fields[0]:\n"); 6105 ret = hist_field_debug_show(m, hist_data->fields[0], 6106 HIST_FIELD_FL_HITCOUNT); 6107 if (ret) 6108 return; 6109 6110 for (i = 1; i < hist_data->n_vals; i++) { 6111 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6112 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 6113 if (ret) 6114 return; 6115 } 6116 6117 seq_puts(m, "\n key fields:\n"); 6118 6119 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 6120 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6121 ret = hist_field_debug_show(m, hist_data->fields[i], 6122 HIST_FIELD_FL_KEY); 6123 if (ret) 6124 return; 6125 } 6126 6127 if (hist_data->n_var_refs) 6128 seq_puts(m, "\n variable reference fields:\n"); 6129 6130 for (i = 0; i < hist_data->n_var_refs; i++) { 6131 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 6132 ret = hist_field_debug_show(m, hist_data->var_refs[i], 6133 HIST_FIELD_FL_VAR_REF); 6134 if (ret) 6135 return; 6136 } 6137 6138 if (hist_data->n_field_vars) 6139 seq_puts(m, "\n field variables:\n"); 6140 6141 for (i = 0; i < hist_data->n_field_vars; i++) { 6142 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 6143 if (ret) 6144 return; 6145 } 6146 6147 ret = hist_actions_debug_show(m, hist_data); 6148 if (ret) 6149 return; 6150 } 6151 6152 static int hist_debug_show(struct seq_file *m, void *v) 6153 { 6154 struct event_trigger_data *data; 6155 struct trace_event_file *event_file; 6156 int n = 0; 6157 6158 guard(mutex)(&event_mutex); 6159 6160 event_file = event_file_file(m->private); 6161 if (unlikely(!event_file)) 6162 return -ENODEV; 6163 6164 list_for_each_entry(data, &event_file->triggers, list) { 6165 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 6166 hist_trigger_debug_show(m, data, n++); 6167 } 6168 return 0; 6169 } 6170 6171 static int event_hist_debug_open(struct inode *inode, struct file *file) 6172 { 6173 int ret; 6174 6175 ret = tracing_open_file_tr(inode, file); 6176 if (ret) 6177 return ret; 6178 6179 ret = single_open(file, hist_debug_show, file); 6180 if (ret) 6181 tracing_release_file_tr(inode, file); 6182 return ret; 6183 } 6184 6185 const struct file_operations event_hist_debug_fops = { 6186 .open = event_hist_debug_open, 6187 .read = seq_read, 6188 .llseek = seq_lseek, 6189 .release = tracing_single_release_file_tr, 6190 }; 6191 #endif 6192 6193 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 6194 { 6195 const char *field_name = hist_field_name(hist_field, 0); 6196 6197 if (hist_field->var.name) 6198 seq_printf(m, "%s=", hist_field->var.name); 6199 6200 if (hist_field->flags & HIST_FIELD_FL_CPU) 6201 seq_puts(m, "common_cpu"); 6202 if (hist_field->flags & HIST_FIELD_FL_COMM) 6203 seq_puts(m, "common_comm"); 6204 else if (hist_field->flags & HIST_FIELD_FL_CONST) 6205 seq_printf(m, "%llu", hist_field->constant); 6206 else if (field_name) { 6207 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 6208 hist_field->flags & HIST_FIELD_FL_ALIAS) 6209 if (!hist_field->system) 6210 seq_putc(m, '$'); 6211 seq_printf(m, "%s", field_name); 6212 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 6213 seq_puts(m, "common_timestamp"); 6214 6215 if (hist_field->flags) { 6216 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 6217 !(hist_field->flags & HIST_FIELD_FL_EXPR) && 6218 !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) { 6219 const char *flags = get_hist_field_flags(hist_field); 6220 6221 if (flags) 6222 seq_printf(m, ".%s", flags); 6223 } 6224 } 6225 if (hist_field->buckets) 6226 seq_printf(m, "=%lu", hist_field->buckets); 6227 } 6228 6229 static int event_hist_trigger_print(struct seq_file *m, 6230 struct event_trigger_data *data) 6231 { 6232 struct hist_trigger_data *hist_data = data->private_data; 6233 struct hist_field *field; 6234 bool have_var = false; 6235 bool show_val = false; 6236 unsigned int i; 6237 6238 seq_puts(m, HIST_PREFIX); 6239 6240 if (data->name) 6241 seq_printf(m, "%s:", data->name); 6242 6243 seq_puts(m, "keys="); 6244 6245 for_each_hist_key_field(i, hist_data) { 6246 field = hist_data->fields[i]; 6247 6248 if (i > hist_data->n_vals) 6249 seq_puts(m, ","); 6250 6251 if (field->flags & HIST_FIELD_FL_STACKTRACE) { 6252 if (field->field) 6253 seq_printf(m, "%s.stacktrace", field->field->name); 6254 else 6255 seq_puts(m, "common_stacktrace"); 6256 } else 6257 hist_field_print(m, field); 6258 } 6259 6260 seq_puts(m, ":vals="); 6261 6262 for_each_hist_val_field(i, hist_data) { 6263 field = hist_data->fields[i]; 6264 if (field->flags & HIST_FIELD_FL_VAR) { 6265 have_var = true; 6266 continue; 6267 } 6268 6269 if (i == HITCOUNT_IDX) { 6270 if (hist_data->attrs->no_hitcount) 6271 continue; 6272 seq_puts(m, "hitcount"); 6273 } else { 6274 if (show_val) 6275 seq_puts(m, ","); 6276 hist_field_print(m, field); 6277 } 6278 show_val = true; 6279 } 6280 6281 if (have_var) { 6282 unsigned int n = 0; 6283 6284 seq_puts(m, ":"); 6285 6286 for_each_hist_val_field(i, hist_data) { 6287 field = hist_data->fields[i]; 6288 6289 if (field->flags & HIST_FIELD_FL_VAR) { 6290 if (n++) 6291 seq_puts(m, ","); 6292 hist_field_print(m, field); 6293 } 6294 } 6295 } 6296 6297 seq_puts(m, ":sort="); 6298 6299 for (i = 0; i < hist_data->n_sort_keys; i++) { 6300 struct tracing_map_sort_key *sort_key; 6301 unsigned int idx, first_key_idx; 6302 6303 /* skip VAR vals */ 6304 first_key_idx = hist_data->n_vals - hist_data->n_vars; 6305 6306 sort_key = &hist_data->sort_keys[i]; 6307 idx = sort_key->field_idx; 6308 6309 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 6310 return -EINVAL; 6311 6312 if (i > 0) 6313 seq_puts(m, ","); 6314 6315 if (idx == HITCOUNT_IDX) 6316 seq_puts(m, "hitcount"); 6317 else { 6318 if (idx >= first_key_idx) 6319 idx += hist_data->n_vars; 6320 hist_field_print(m, hist_data->fields[idx]); 6321 } 6322 6323 if (sort_key->descending) 6324 seq_puts(m, ".descending"); 6325 } 6326 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 6327 if (hist_data->enable_timestamps) 6328 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 6329 if (hist_data->attrs->no_hitcount) 6330 seq_puts(m, ":nohitcount"); 6331 6332 print_actions_spec(m, hist_data); 6333 6334 if (data->filter_str) 6335 seq_printf(m, " if %s", data->filter_str); 6336 6337 if (data->paused) 6338 seq_puts(m, " [paused]"); 6339 else 6340 seq_puts(m, " [active]"); 6341 6342 seq_putc(m, '\n'); 6343 6344 return 0; 6345 } 6346 6347 static int event_hist_trigger_init(struct event_trigger_data *data) 6348 { 6349 struct hist_trigger_data *hist_data = data->private_data; 6350 6351 if (alloc_hist_pad() < 0) 6352 return -ENOMEM; 6353 6354 if (!data->ref && hist_data->attrs->name) 6355 save_named_trigger(hist_data->attrs->name, data); 6356 6357 data->ref++; 6358 6359 return 0; 6360 } 6361 6362 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 6363 { 6364 struct trace_event_file *file; 6365 unsigned int i; 6366 char *cmd; 6367 int ret; 6368 6369 for (i = 0; i < hist_data->n_field_var_hists; i++) { 6370 file = hist_data->field_var_hists[i]->file; 6371 cmd = hist_data->field_var_hists[i]->cmd; 6372 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 6373 "!hist", "hist", cmd); 6374 WARN_ON_ONCE(ret < 0); 6375 } 6376 } 6377 6378 static void event_hist_trigger_free(struct event_trigger_data *data) 6379 { 6380 struct hist_trigger_data *hist_data = data->private_data; 6381 6382 if (WARN_ON_ONCE(data->ref <= 0)) 6383 return; 6384 6385 data->ref--; 6386 if (!data->ref) { 6387 if (data->name) 6388 del_named_trigger(data); 6389 6390 trigger_data_free(data); 6391 6392 tracepoint_synchronize_unregister(); 6393 remove_hist_vars(hist_data); 6394 6395 unregister_field_var_hists(hist_data); 6396 6397 destroy_hist_data(hist_data); 6398 } 6399 free_hist_pad(); 6400 } 6401 6402 static int event_hist_trigger_named_init(struct event_trigger_data *data) 6403 { 6404 int ret; 6405 6406 ret = event_hist_trigger_init(data->named_data); 6407 if (ret < 0) { 6408 kfree(data->cmd_ops); 6409 data->cmd_ops = &trigger_hist_cmd; 6410 return ret; 6411 } 6412 6413 data->ref++; 6414 6415 save_named_trigger(data->named_data->name, data); 6416 6417 return 0; 6418 } 6419 6420 static void event_hist_trigger_named_free(struct event_trigger_data *data) 6421 { 6422 if (WARN_ON_ONCE(data->ref <= 0)) 6423 return; 6424 6425 event_hist_trigger_free(data->named_data); 6426 6427 data->ref--; 6428 if (!data->ref) { 6429 struct event_command *cmd_ops = data->cmd_ops; 6430 6431 del_named_trigger(data); 6432 trigger_data_free(data); 6433 tracepoint_synchronize_unregister(); 6434 kfree(cmd_ops); 6435 } 6436 } 6437 6438 static void hist_clear(struct event_trigger_data *data) 6439 { 6440 struct hist_trigger_data *hist_data = data->private_data; 6441 6442 if (data->name) 6443 pause_named_trigger(data); 6444 6445 tracepoint_synchronize_unregister(); 6446 6447 tracing_map_clear(hist_data->map); 6448 6449 if (data->name) 6450 unpause_named_trigger(data); 6451 } 6452 6453 static bool compatible_field(struct ftrace_event_field *field, 6454 struct ftrace_event_field *test_field) 6455 { 6456 if (field == test_field) 6457 return true; 6458 if (field == NULL || test_field == NULL) 6459 return false; 6460 if (strcmp(field->name, test_field->name) != 0) 6461 return false; 6462 if (strcmp(field->type, test_field->type) != 0) 6463 return false; 6464 if (field->size != test_field->size) 6465 return false; 6466 if (field->is_signed != test_field->is_signed) 6467 return false; 6468 6469 return true; 6470 } 6471 6472 static bool hist_trigger_match(struct event_trigger_data *data, 6473 struct event_trigger_data *data_test, 6474 struct event_trigger_data *named_data, 6475 bool ignore_filter) 6476 { 6477 struct tracing_map_sort_key *sort_key, *sort_key_test; 6478 struct hist_trigger_data *hist_data, *hist_data_test; 6479 struct hist_field *key_field, *key_field_test; 6480 unsigned int i; 6481 6482 if (named_data && (named_data != data_test) && 6483 (named_data != data_test->named_data)) 6484 return false; 6485 6486 if (!named_data && is_named_trigger(data_test)) 6487 return false; 6488 6489 hist_data = data->private_data; 6490 hist_data_test = data_test->private_data; 6491 6492 if (hist_data->n_vals != hist_data_test->n_vals || 6493 hist_data->n_fields != hist_data_test->n_fields || 6494 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 6495 return false; 6496 6497 if (!ignore_filter) { 6498 if ((data->filter_str && !data_test->filter_str) || 6499 (!data->filter_str && data_test->filter_str)) 6500 return false; 6501 } 6502 6503 for_each_hist_field(i, hist_data) { 6504 key_field = hist_data->fields[i]; 6505 key_field_test = hist_data_test->fields[i]; 6506 6507 if (key_field->flags != key_field_test->flags) 6508 return false; 6509 if (!compatible_field(key_field->field, key_field_test->field)) 6510 return false; 6511 if (key_field->offset != key_field_test->offset) 6512 return false; 6513 if (key_field->size != key_field_test->size) 6514 return false; 6515 if (key_field->is_signed != key_field_test->is_signed) 6516 return false; 6517 if (!!key_field->var.name != !!key_field_test->var.name) 6518 return false; 6519 if (key_field->var.name && 6520 strcmp(key_field->var.name, key_field_test->var.name) != 0) 6521 return false; 6522 } 6523 6524 for (i = 0; i < hist_data->n_sort_keys; i++) { 6525 sort_key = &hist_data->sort_keys[i]; 6526 sort_key_test = &hist_data_test->sort_keys[i]; 6527 6528 if (sort_key->field_idx != sort_key_test->field_idx || 6529 sort_key->descending != sort_key_test->descending) 6530 return false; 6531 } 6532 6533 if (!ignore_filter && data->filter_str && 6534 (strcmp(data->filter_str, data_test->filter_str) != 0)) 6535 return false; 6536 6537 if (!actions_match(hist_data, hist_data_test)) 6538 return false; 6539 6540 return true; 6541 } 6542 6543 static bool existing_hist_update_only(char *glob, 6544 struct event_trigger_data *data, 6545 struct trace_event_file *file) 6546 { 6547 struct hist_trigger_data *hist_data = data->private_data; 6548 struct event_trigger_data *test, *named_data = NULL; 6549 bool updated = false; 6550 6551 if (!hist_data->attrs->pause && !hist_data->attrs->cont && 6552 !hist_data->attrs->clear) 6553 goto out; 6554 6555 if (hist_data->attrs->name) { 6556 named_data = find_named_trigger(hist_data->attrs->name); 6557 if (named_data) { 6558 if (!hist_trigger_match(data, named_data, named_data, 6559 true)) 6560 goto out; 6561 } 6562 } 6563 6564 if (hist_data->attrs->name && !named_data) 6565 goto out; 6566 6567 list_for_each_entry(test, &file->triggers, list) { 6568 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6569 if (!hist_trigger_match(data, test, named_data, false)) 6570 continue; 6571 if (hist_data->attrs->pause) 6572 test->paused = true; 6573 else if (hist_data->attrs->cont) 6574 test->paused = false; 6575 else if (hist_data->attrs->clear) 6576 hist_clear(test); 6577 updated = true; 6578 goto out; 6579 } 6580 } 6581 out: 6582 return updated; 6583 } 6584 6585 /* 6586 * Set or disable using the per CPU trace_buffer_event when possible. 6587 */ 6588 static int tracing_set_filter_buffering(struct trace_array *tr, bool set) 6589 { 6590 guard(mutex)(&trace_types_lock); 6591 6592 if (set && tr->no_filter_buffering_ref++) 6593 return 0; 6594 6595 if (!set) { 6596 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) 6597 return -EINVAL; 6598 6599 --tr->no_filter_buffering_ref; 6600 } 6601 6602 return 0; 6603 } 6604 6605 static int hist_register_trigger(char *glob, 6606 struct event_trigger_data *data, 6607 struct trace_event_file *file) 6608 { 6609 struct hist_trigger_data *hist_data = data->private_data; 6610 struct event_trigger_data *test, *named_data = NULL; 6611 struct trace_array *tr = file->tr; 6612 int ret = 0; 6613 6614 if (hist_data->attrs->name) { 6615 named_data = find_named_trigger(hist_data->attrs->name); 6616 if (named_data) { 6617 if (!hist_trigger_match(data, named_data, named_data, 6618 true)) { 6619 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 6620 ret = -EINVAL; 6621 goto out; 6622 } 6623 } 6624 } 6625 6626 if (hist_data->attrs->name && !named_data) 6627 goto new; 6628 6629 lockdep_assert_held(&event_mutex); 6630 6631 list_for_each_entry(test, &file->triggers, list) { 6632 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6633 if (hist_trigger_match(data, test, named_data, false)) { 6634 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 6635 ret = -EEXIST; 6636 goto out; 6637 } 6638 } 6639 } 6640 new: 6641 if (hist_data->attrs->cont || hist_data->attrs->clear) { 6642 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 6643 ret = -ENOENT; 6644 goto out; 6645 } 6646 6647 if (hist_data->attrs->pause) 6648 data->paused = true; 6649 6650 if (named_data) { 6651 struct event_command *cmd_ops; 6652 6653 data->private_data = named_data->private_data; 6654 set_named_trigger_data(data, named_data); 6655 /* Copy the command ops and update some of the functions */ 6656 cmd_ops = kmalloc_obj(*cmd_ops); 6657 if (!cmd_ops) { 6658 ret = -ENOMEM; 6659 goto out; 6660 } 6661 *cmd_ops = *data->cmd_ops; 6662 cmd_ops->init = event_hist_trigger_named_init; 6663 cmd_ops->free = event_hist_trigger_named_free; 6664 data->cmd_ops = cmd_ops; 6665 } 6666 6667 if (hist_data->enable_timestamps) { 6668 char *clock = hist_data->attrs->clock; 6669 6670 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 6671 if (ret) { 6672 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 6673 goto out; 6674 } 6675 6676 tracing_set_filter_buffering(file->tr, true); 6677 } 6678 6679 if (data->cmd_ops->init) { 6680 ret = data->cmd_ops->init(data); 6681 if (ret < 0) { 6682 if (hist_data->enable_timestamps) 6683 tracing_set_filter_buffering(file->tr, false); 6684 goto out; 6685 } 6686 } 6687 6688 if (named_data) { 6689 remove_hist_vars(hist_data); 6690 destroy_hist_data(hist_data); 6691 } 6692 out: 6693 return ret; 6694 } 6695 6696 static int hist_trigger_enable(struct event_trigger_data *data, 6697 struct trace_event_file *file) 6698 { 6699 int ret = 0; 6700 6701 list_add_tail_rcu(&data->list, &file->triggers); 6702 6703 update_cond_flag(file); 6704 6705 /* 6706 * On failure the caller undoes the registration, and 6707 * hist_unregister_trigger() can only find the trigger here. 6708 */ 6709 if (trace_event_trigger_enable_disable(file, 1) < 0) 6710 ret--; 6711 6712 return ret; 6713 } 6714 6715 static bool have_hist_trigger_match(struct event_trigger_data *data, 6716 struct trace_event_file *file) 6717 { 6718 struct hist_trigger_data *hist_data = data->private_data; 6719 struct event_trigger_data *test, *named_data = NULL; 6720 bool match = false; 6721 6722 lockdep_assert_held(&event_mutex); 6723 6724 if (hist_data->attrs->name) 6725 named_data = find_named_trigger(hist_data->attrs->name); 6726 6727 list_for_each_entry(test, &file->triggers, list) { 6728 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6729 if (hist_trigger_match(data, test, named_data, false)) { 6730 match = true; 6731 break; 6732 } 6733 } 6734 } 6735 6736 return match; 6737 } 6738 6739 static bool hist_trigger_check_refs(struct event_trigger_data *data, 6740 struct trace_event_file *file) 6741 { 6742 struct hist_trigger_data *hist_data = data->private_data; 6743 struct event_trigger_data *test, *named_data = NULL; 6744 6745 lockdep_assert_held(&event_mutex); 6746 6747 if (hist_data->attrs->name) 6748 named_data = find_named_trigger(hist_data->attrs->name); 6749 6750 list_for_each_entry(test, &file->triggers, list) { 6751 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6752 if (!hist_trigger_match(data, test, named_data, false)) 6753 continue; 6754 hist_data = test->private_data; 6755 if (check_var_refs(hist_data)) 6756 return true; 6757 break; 6758 } 6759 } 6760 6761 return false; 6762 } 6763 6764 static void hist_unregister_trigger(char *glob, 6765 struct event_trigger_data *data, 6766 struct trace_event_file *file) 6767 { 6768 struct event_trigger_data *test = NULL, *iter, *named_data = NULL; 6769 struct hist_trigger_data *hist_data = data->private_data; 6770 6771 lockdep_assert_held(&event_mutex); 6772 6773 if (hist_data->attrs->name) 6774 named_data = find_named_trigger(hist_data->attrs->name); 6775 6776 list_for_each_entry(iter, &file->triggers, list) { 6777 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6778 if (!hist_trigger_match(data, iter, named_data, false)) 6779 continue; 6780 test = iter; 6781 list_del_rcu(&test->list); 6782 trace_event_trigger_enable_disable(file, 0); 6783 update_cond_flag(file); 6784 break; 6785 } 6786 } 6787 6788 if (hist_data->enable_timestamps) { 6789 if (!hist_data->remove || test) 6790 tracing_set_filter_buffering(file->tr, false); 6791 } 6792 6793 if (test && test->cmd_ops->free) 6794 test->cmd_ops->free(test); 6795 } 6796 6797 static bool hist_file_check_refs(struct trace_event_file *file) 6798 { 6799 struct hist_trigger_data *hist_data; 6800 struct event_trigger_data *test; 6801 6802 lockdep_assert_held(&event_mutex); 6803 6804 list_for_each_entry(test, &file->triggers, list) { 6805 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6806 hist_data = test->private_data; 6807 if (check_var_refs(hist_data)) 6808 return true; 6809 } 6810 } 6811 6812 return false; 6813 } 6814 6815 static void hist_unreg_all(struct trace_event_file *file) 6816 { 6817 struct event_trigger_data *test, *n; 6818 struct hist_trigger_data *hist_data; 6819 struct synth_event *se; 6820 6821 lockdep_assert_held(&event_mutex); 6822 6823 if (hist_file_check_refs(file)) 6824 return; 6825 6826 list_for_each_entry_safe(test, n, &file->triggers, list) { 6827 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6828 hist_data = test->private_data; 6829 list_del_rcu(&test->list); 6830 trace_event_trigger_enable_disable(file, 0); 6831 6832 se = find_synth_event_call(file->event_call); 6833 if (se) 6834 se->ref--; 6835 6836 update_cond_flag(file); 6837 if (hist_data->enable_timestamps) 6838 tracing_set_filter_buffering(file->tr, false); 6839 if (test->cmd_ops->free) 6840 test->cmd_ops->free(test); 6841 } 6842 } 6843 } 6844 6845 static int event_hist_trigger_parse(struct event_command *cmd_ops, 6846 struct trace_event_file *file, 6847 char *glob, char *cmd, 6848 char *param_and_filter) 6849 { 6850 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6851 struct event_trigger_data *trigger_data; 6852 struct hist_trigger_attrs *attrs; 6853 struct hist_trigger_data *hist_data; 6854 char *param, *filter, *p, *start; 6855 struct synth_event *se; 6856 bool remove; 6857 int ret = 0; 6858 6859 lockdep_assert_held(&event_mutex); 6860 6861 if (WARN_ON(!glob)) 6862 return -EINVAL; 6863 6864 if (glob[0]) { 6865 hist_err_clear(); 6866 last_cmd_set(file, param_and_filter); 6867 } 6868 6869 remove = event_trigger_check_remove(glob); 6870 6871 if (event_trigger_empty_param(param_and_filter)) 6872 return -EINVAL; 6873 6874 /* 6875 * separate the trigger from the filter (k:v [if filter]) 6876 * allowing for whitespace in the trigger 6877 */ 6878 p = param = param_and_filter; 6879 do { 6880 p = strstr(p, "if"); 6881 if (!p) 6882 break; 6883 if (p == param_and_filter) 6884 return -EINVAL; 6885 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6886 p++; 6887 continue; 6888 } 6889 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1) 6890 return -EINVAL; 6891 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6892 p++; 6893 continue; 6894 } 6895 break; 6896 } while (1); 6897 6898 if (!p) 6899 filter = NULL; 6900 else { 6901 *(p - 1) = '\0'; 6902 filter = strstrip(p); 6903 param = strstrip(param); 6904 } 6905 6906 /* 6907 * To simplify arithmetic expression parsing, replace occurrences of 6908 * '.sym-offset' modifier with '.symXoffset' 6909 */ 6910 start = strstr(param, ".sym-offset"); 6911 while (start) { 6912 *(start + 4) = 'X'; 6913 start = strstr(start + 11, ".sym-offset"); 6914 } 6915 6916 attrs = parse_hist_trigger_attrs(file->tr, param); 6917 if (IS_ERR(attrs)) 6918 return PTR_ERR(attrs); 6919 6920 if (attrs->map_bits) 6921 hist_trigger_bits = attrs->map_bits; 6922 6923 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6924 if (IS_ERR(hist_data)) { 6925 destroy_hist_trigger_attrs(attrs); 6926 return PTR_ERR(hist_data); 6927 } 6928 6929 trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data); 6930 if (!trigger_data) { 6931 ret = -ENOMEM; 6932 goto out_free; 6933 } 6934 6935 ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data); 6936 if (ret < 0) 6937 goto out_free; 6938 6939 if (remove) { 6940 if (!have_hist_trigger_match(trigger_data, file)) 6941 goto out_free; 6942 6943 if (hist_trigger_check_refs(trigger_data, file)) { 6944 ret = -EBUSY; 6945 goto out_free; 6946 } 6947 6948 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6949 se = find_synth_event_call(file->event_call); 6950 if (se) 6951 se->ref--; 6952 ret = 0; 6953 goto out_free; 6954 } 6955 6956 if (existing_hist_update_only(glob, trigger_data, file)) 6957 goto out_free; 6958 6959 if (!get_named_trigger_data(trigger_data)) { 6960 6961 ret = create_actions(hist_data); 6962 if (ret) 6963 goto out_free; 6964 6965 if (has_hist_vars(hist_data) || hist_data->n_var_refs) { 6966 ret = save_hist_vars(hist_data); 6967 if (ret) 6968 goto out_free; 6969 } 6970 6971 ret = tracing_map_init(hist_data->map); 6972 if (ret) 6973 goto out_free; 6974 } 6975 6976 ret = event_trigger_register(cmd_ops, file, glob, trigger_data); 6977 if (ret < 0) 6978 goto out_free; 6979 6980 ret = hist_trigger_enable(trigger_data, file); 6981 if (ret) 6982 goto out_unreg; 6983 6984 se = find_synth_event_call(file->event_call); 6985 if (se) 6986 se->ref++; 6987 out: 6988 if (ret == 0 && glob[0]) 6989 hist_err_clear(); 6990 6991 return ret; 6992 out_unreg: 6993 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6994 /* The unregister frees trigger_data, skip out_free */ 6995 goto out; 6996 out_free: 6997 remove_hist_vars(hist_data); 6998 6999 trigger_data_free(trigger_data); 7000 7001 unregister_field_var_hists(hist_data); 7002 7003 destroy_hist_data(hist_data); 7004 goto out; 7005 } 7006 7007 static struct event_command trigger_hist_cmd = { 7008 .name = "hist", 7009 .trigger_type = ETT_EVENT_HIST, 7010 .flags = EVENT_CMD_FL_NEEDS_REC, 7011 .parse = event_hist_trigger_parse, 7012 .reg = hist_register_trigger, 7013 .unreg = hist_unregister_trigger, 7014 .unreg_all = hist_unreg_all, 7015 .set_filter = set_trigger_filter, 7016 .trigger = event_hist_trigger, 7017 .print = event_hist_trigger_print, 7018 .init = event_hist_trigger_init, 7019 .free = event_hist_trigger_free, 7020 }; 7021 7022 __init int register_trigger_hist_cmd(void) 7023 { 7024 int ret; 7025 7026 ret = register_event_command(&trigger_hist_cmd); 7027 WARN_ON(ret < 0); 7028 7029 return ret; 7030 } 7031 7032 static void 7033 hist_enable_trigger(struct event_trigger_data *data, 7034 struct trace_buffer *buffer, void *rec, 7035 struct ring_buffer_event *event) 7036 { 7037 struct enable_trigger_data *enable_data = data->private_data; 7038 struct event_trigger_data *test; 7039 7040 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 7041 lockdep_is_held(&event_mutex)) { 7042 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 7043 if (enable_data->enable) 7044 test->paused = false; 7045 else 7046 test->paused = true; 7047 } 7048 } 7049 } 7050 7051 static void hist_enable_unreg_all(struct trace_event_file *file) 7052 { 7053 struct event_trigger_data *test, *n; 7054 7055 list_for_each_entry_safe(test, n, &file->triggers, list) { 7056 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 7057 list_del_rcu(&test->list); 7058 update_cond_flag(file); 7059 trace_event_trigger_enable_disable(file, 0); 7060 if (test->cmd_ops->free) 7061 test->cmd_ops->free(test); 7062 } 7063 } 7064 } 7065 7066 static struct event_command trigger_hist_enable_cmd = { 7067 .name = ENABLE_HIST_STR, 7068 .trigger_type = ETT_HIST_ENABLE, 7069 .parse = event_enable_trigger_parse, 7070 .reg = event_enable_register_trigger, 7071 .unreg = event_enable_unregister_trigger, 7072 .unreg_all = hist_enable_unreg_all, 7073 .set_filter = set_trigger_filter, 7074 .trigger = hist_enable_trigger, 7075 .count_func = event_trigger_count, 7076 .print = event_enable_trigger_print, 7077 .init = event_trigger_init, 7078 .free = event_enable_trigger_free, 7079 }; 7080 7081 static struct event_command trigger_hist_disable_cmd = { 7082 .name = DISABLE_HIST_STR, 7083 .trigger_type = ETT_HIST_ENABLE, 7084 .parse = event_enable_trigger_parse, 7085 .reg = event_enable_register_trigger, 7086 .unreg = event_enable_unregister_trigger, 7087 .unreg_all = hist_enable_unreg_all, 7088 .set_filter = set_trigger_filter, 7089 .trigger = hist_enable_trigger, 7090 .count_func = event_trigger_count, 7091 .print = event_enable_trigger_print, 7092 .init = event_trigger_init, 7093 .free = event_enable_trigger_free, 7094 }; 7095 7096 static __init void unregister_trigger_hist_enable_disable_cmds(void) 7097 { 7098 unregister_event_command(&trigger_hist_enable_cmd); 7099 unregister_event_command(&trigger_hist_disable_cmd); 7100 } 7101 7102 __init int register_trigger_hist_enable_disable_cmds(void) 7103 { 7104 int ret; 7105 7106 ret = register_event_command(&trigger_hist_enable_cmd); 7107 if (WARN_ON(ret < 0)) 7108 return ret; 7109 ret = register_event_command(&trigger_hist_disable_cmd); 7110 if (WARN_ON(ret < 0)) 7111 unregister_trigger_hist_enable_disable_cmds(); 7112 7113 return ret; 7114 } 7115