1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * trace_events_hist - trace event hist triggers 4 * 5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kallsyms.h> 10 #include <linux/security.h> 11 #include <linux/mutex.h> 12 #include <linux/slab.h> 13 #include <linux/stacktrace.h> 14 #include <linux/rculist.h> 15 #include <linux/tracefs.h> 16 17 /* for gfp flag names */ 18 #include <linux/trace_events.h> 19 #include <trace/events/mmflags.h> 20 21 #include "tracing_map.h" 22 #include "trace_synth.h" 23 24 #define ERRORS \ 25 C(NONE, "No error"), \ 26 C(DUPLICATE_VAR, "Variable already defined"), \ 27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \ 28 C(TOO_MANY_VARS, "Too many variables defined"), \ 29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \ 30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \ 31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \ 32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \ 33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \ 34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \ 35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \ 36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \ 37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \ 38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \ 39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \ 40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \ 41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \ 42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \ 43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \ 44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \ 45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \ 46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \ 47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \ 48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \ 49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \ 50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \ 51 C(TOO_MANY_PARAMS, "Too many action params"), \ 52 C(PARAM_NOT_FOUND, "Couldn't find param"), \ 53 C(INVALID_PARAM, "Invalid action param"), \ 54 C(ACTION_NOT_FOUND, "No action found"), \ 55 C(NO_SAVE_PARAMS, "No params found for save()"), \ 56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \ 57 C(ACTION_MISMATCH, "Handler doesn't support action"), \ 58 C(NO_CLOSING_PAREN, "No closing paren found"), \ 59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \ 60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \ 61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \ 62 C(VAR_NOT_FOUND, "Couldn't find variable"), \ 63 C(FIELD_NOT_FOUND, "Couldn't find field"), \ 64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \ 65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \ 66 C(EMPTY_SORT_FIELD, "Empty sort field"), \ 67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \ 68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \ 69 C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \ 70 C(EXPECT_NUMBER, "Expecting numeric literal"), \ 71 C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \ 72 C(DIVISION_BY_ZERO, "Division by zero"), \ 73 C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"), 74 75 #undef C 76 #define C(a, b) HIST_ERR_##a 77 78 enum { ERRORS }; 79 80 #undef C 81 #define C(a, b) b 82 83 static const char *err_text[] = { ERRORS }; 84 85 struct hist_field; 86 87 typedef u64 (*hist_field_fn_t) (struct hist_field *field, 88 struct tracing_map_elt *elt, 89 struct trace_buffer *buffer, 90 struct ring_buffer_event *rbe, 91 void *event); 92 93 #define HIST_FIELD_OPERANDS_MAX 2 94 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX) 95 #define HIST_ACTIONS_MAX 8 96 #define HIST_CONST_DIGITS_MAX 21 97 #define HIST_DIV_SHIFT 20 /* For optimizing division by constants */ 98 99 enum field_op_id { 100 FIELD_OP_NONE, 101 FIELD_OP_PLUS, 102 FIELD_OP_MINUS, 103 FIELD_OP_UNARY_MINUS, 104 FIELD_OP_DIV, 105 FIELD_OP_MULT, 106 }; 107 108 enum hist_field_fn { 109 HIST_FIELD_FN_NOP, 110 HIST_FIELD_FN_VAR_REF, 111 HIST_FIELD_FN_COUNTER, 112 HIST_FIELD_FN_CONST, 113 HIST_FIELD_FN_LOG2, 114 HIST_FIELD_FN_BUCKET, 115 HIST_FIELD_FN_TIMESTAMP, 116 HIST_FIELD_FN_CPU, 117 HIST_FIELD_FN_COMM, 118 HIST_FIELD_FN_STRING, 119 HIST_FIELD_FN_DYNSTRING, 120 HIST_FIELD_FN_RELDYNSTRING, 121 HIST_FIELD_FN_PSTRING, 122 HIST_FIELD_FN_S64, 123 HIST_FIELD_FN_U64, 124 HIST_FIELD_FN_S32, 125 HIST_FIELD_FN_U32, 126 HIST_FIELD_FN_S16, 127 HIST_FIELD_FN_U16, 128 HIST_FIELD_FN_S8, 129 HIST_FIELD_FN_U8, 130 HIST_FIELD_FN_UMINUS, 131 HIST_FIELD_FN_MINUS, 132 HIST_FIELD_FN_PLUS, 133 HIST_FIELD_FN_DIV, 134 HIST_FIELD_FN_MULT, 135 HIST_FIELD_FN_DIV_POWER2, 136 HIST_FIELD_FN_DIV_NOT_POWER2, 137 HIST_FIELD_FN_DIV_MULT_SHIFT, 138 HIST_FIELD_FN_EXECNAME, 139 HIST_FIELD_FN_STACK, 140 }; 141 142 /* 143 * A hist_var (histogram variable) contains variable information for 144 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF 145 * flag set. A hist_var has a variable name e.g. ts0, and is 146 * associated with a given histogram trigger, as specified by 147 * hist_data. The hist_var idx is the unique index assigned to the 148 * variable by the hist trigger's tracing_map. The idx is what is 149 * used to set a variable's value and, by a variable reference, to 150 * retrieve it. 151 */ 152 struct hist_var { 153 char *name; 154 struct hist_trigger_data *hist_data; 155 unsigned int idx; 156 }; 157 158 struct hist_field { 159 struct ftrace_event_field *field; 160 unsigned long flags; 161 unsigned long buckets; 162 const char *type; 163 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; 164 struct hist_trigger_data *hist_data; 165 enum hist_field_fn fn_num; 166 unsigned int ref; 167 unsigned int size; 168 unsigned int offset; 169 unsigned int is_signed; 170 171 /* 172 * Variable fields contain variable-specific info in var. 173 */ 174 struct hist_var var; 175 enum field_op_id operator; 176 char *system; 177 char *event_name; 178 179 /* 180 * The name field is used for EXPR and VAR_REF fields. VAR 181 * fields contain the variable name in var.name. 182 */ 183 char *name; 184 185 /* 186 * When a histogram trigger is hit, if it has any references 187 * to variables, the values of those variables are collected 188 * into a var_ref_vals array by resolve_var_refs(). The 189 * current value of each variable is read from the tracing_map 190 * using the hist field's hist_var.idx and entered into the 191 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx]. 192 */ 193 unsigned int var_ref_idx; 194 bool read_once; 195 196 unsigned int var_str_idx; 197 198 /* Numeric literals are represented as u64 */ 199 u64 constant; 200 /* Used to optimize division by constants */ 201 u64 div_multiplier; 202 }; 203 204 static u64 hist_fn_call(struct hist_field *hist_field, 205 struct tracing_map_elt *elt, 206 struct trace_buffer *buffer, 207 struct ring_buffer_event *rbe, 208 void *event); 209 210 static u64 hist_field_const(struct hist_field *field, 211 struct tracing_map_elt *elt, 212 struct trace_buffer *buffer, 213 struct ring_buffer_event *rbe, 214 void *event) 215 { 216 return field->constant; 217 } 218 219 static u64 hist_field_counter(struct hist_field *field, 220 struct tracing_map_elt *elt, 221 struct trace_buffer *buffer, 222 struct ring_buffer_event *rbe, 223 void *event) 224 { 225 return 1; 226 } 227 228 static u64 hist_field_string(struct hist_field *hist_field, 229 struct tracing_map_elt *elt, 230 struct trace_buffer *buffer, 231 struct ring_buffer_event *rbe, 232 void *event) 233 { 234 char *addr = (char *)(event + hist_field->field->offset); 235 236 return (u64)(unsigned long)addr; 237 } 238 239 static u64 hist_field_dynstring(struct hist_field *hist_field, 240 struct tracing_map_elt *elt, 241 struct trace_buffer *buffer, 242 struct ring_buffer_event *rbe, 243 void *event) 244 { 245 u32 str_item = *(u32 *)(event + hist_field->field->offset); 246 int str_loc = str_item & 0xffff; 247 char *addr = (char *)(event + str_loc); 248 249 return (u64)(unsigned long)addr; 250 } 251 252 static u64 hist_field_reldynstring(struct hist_field *hist_field, 253 struct tracing_map_elt *elt, 254 struct trace_buffer *buffer, 255 struct ring_buffer_event *rbe, 256 void *event) 257 { 258 u32 *item = event + hist_field->field->offset; 259 u32 str_item = *item; 260 int str_loc = str_item & 0xffff; 261 char *addr = (char *)&item[1] + str_loc; 262 263 return (u64)(unsigned long)addr; 264 } 265 266 static u64 hist_field_pstring(struct hist_field *hist_field, 267 struct tracing_map_elt *elt, 268 struct trace_buffer *buffer, 269 struct ring_buffer_event *rbe, 270 void *event) 271 { 272 char **addr = (char **)(event + hist_field->field->offset); 273 274 return (u64)(unsigned long)*addr; 275 } 276 277 static u64 hist_field_log2(struct hist_field *hist_field, 278 struct tracing_map_elt *elt, 279 struct trace_buffer *buffer, 280 struct ring_buffer_event *rbe, 281 void *event) 282 { 283 struct hist_field *operand = hist_field->operands[0]; 284 285 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 286 287 return (u64) ilog2(roundup_pow_of_two(val)); 288 } 289 290 static u64 hist_field_bucket(struct hist_field *hist_field, 291 struct tracing_map_elt *elt, 292 struct trace_buffer *buffer, 293 struct ring_buffer_event *rbe, 294 void *event) 295 { 296 struct hist_field *operand = hist_field->operands[0]; 297 unsigned long buckets = hist_field->buckets; 298 299 u64 val = hist_fn_call(operand, elt, buffer, rbe, event); 300 301 if (WARN_ON_ONCE(!buckets)) 302 return val; 303 304 if (val >= LONG_MAX) 305 val = div64_ul(val, buckets); 306 else 307 val = (u64)((unsigned long)val / buckets); 308 return val * buckets; 309 } 310 311 static u64 hist_field_plus(struct hist_field *hist_field, 312 struct tracing_map_elt *elt, 313 struct trace_buffer *buffer, 314 struct ring_buffer_event *rbe, 315 void *event) 316 { 317 struct hist_field *operand1 = hist_field->operands[0]; 318 struct hist_field *operand2 = hist_field->operands[1]; 319 320 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 321 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 322 323 return val1 + val2; 324 } 325 326 static u64 hist_field_minus(struct hist_field *hist_field, 327 struct tracing_map_elt *elt, 328 struct trace_buffer *buffer, 329 struct ring_buffer_event *rbe, 330 void *event) 331 { 332 struct hist_field *operand1 = hist_field->operands[0]; 333 struct hist_field *operand2 = hist_field->operands[1]; 334 335 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 336 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 337 338 return val1 - val2; 339 } 340 341 static u64 hist_field_div(struct hist_field *hist_field, 342 struct tracing_map_elt *elt, 343 struct trace_buffer *buffer, 344 struct ring_buffer_event *rbe, 345 void *event) 346 { 347 struct hist_field *operand1 = hist_field->operands[0]; 348 struct hist_field *operand2 = hist_field->operands[1]; 349 350 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 351 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 352 353 /* Return -1 for the undefined case */ 354 if (!val2) 355 return -1; 356 357 /* Use shift if the divisor is a power of 2 */ 358 if (!(val2 & (val2 - 1))) 359 return val1 >> __ffs64(val2); 360 361 return div64_u64(val1, val2); 362 } 363 364 static u64 div_by_power_of_two(struct hist_field *hist_field, 365 struct tracing_map_elt *elt, 366 struct trace_buffer *buffer, 367 struct ring_buffer_event *rbe, 368 void *event) 369 { 370 struct hist_field *operand1 = hist_field->operands[0]; 371 struct hist_field *operand2 = hist_field->operands[1]; 372 373 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 374 375 return val1 >> __ffs64(operand2->constant); 376 } 377 378 static u64 div_by_not_power_of_two(struct hist_field *hist_field, 379 struct tracing_map_elt *elt, 380 struct trace_buffer *buffer, 381 struct ring_buffer_event *rbe, 382 void *event) 383 { 384 struct hist_field *operand1 = hist_field->operands[0]; 385 struct hist_field *operand2 = hist_field->operands[1]; 386 387 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 388 389 return div64_u64(val1, operand2->constant); 390 } 391 392 static u64 div_by_mult_and_shift(struct hist_field *hist_field, 393 struct tracing_map_elt *elt, 394 struct trace_buffer *buffer, 395 struct ring_buffer_event *rbe, 396 void *event) 397 { 398 struct hist_field *operand1 = hist_field->operands[0]; 399 struct hist_field *operand2 = hist_field->operands[1]; 400 401 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 402 403 /* 404 * If the divisor is a constant, do a multiplication and shift instead. 405 * 406 * Choose Z = some power of 2. If Y <= Z, then: 407 * X / Y = (X * (Z / Y)) / Z 408 * 409 * (Z / Y) is a constant (mult) which is calculated at parse time, so: 410 * X / Y = (X * mult) / Z 411 * 412 * The division by Z can be replaced by a shift since Z is a power of 2: 413 * X / Y = (X * mult) >> HIST_DIV_SHIFT 414 * 415 * As long, as X < Z the results will not be off by more than 1. 416 */ 417 if (val1 < (1 << HIST_DIV_SHIFT)) { 418 u64 mult = operand2->div_multiplier; 419 420 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT; 421 } 422 423 return div64_u64(val1, operand2->constant); 424 } 425 426 static u64 hist_field_mult(struct hist_field *hist_field, 427 struct tracing_map_elt *elt, 428 struct trace_buffer *buffer, 429 struct ring_buffer_event *rbe, 430 void *event) 431 { 432 struct hist_field *operand1 = hist_field->operands[0]; 433 struct hist_field *operand2 = hist_field->operands[1]; 434 435 u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); 436 u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); 437 438 return val1 * val2; 439 } 440 441 static u64 hist_field_unary_minus(struct hist_field *hist_field, 442 struct tracing_map_elt *elt, 443 struct trace_buffer *buffer, 444 struct ring_buffer_event *rbe, 445 void *event) 446 { 447 struct hist_field *operand = hist_field->operands[0]; 448 449 s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event); 450 u64 val = (u64)-sval; 451 452 return val; 453 } 454 455 #define DEFINE_HIST_FIELD_FN(type) \ 456 static u64 hist_field_##type(struct hist_field *hist_field, \ 457 struct tracing_map_elt *elt, \ 458 struct trace_buffer *buffer, \ 459 struct ring_buffer_event *rbe, \ 460 void *event) \ 461 { \ 462 type *addr = (type *)(event + hist_field->field->offset); \ 463 \ 464 return (u64)(unsigned long)*addr; \ 465 } 466 467 DEFINE_HIST_FIELD_FN(s64); 468 DEFINE_HIST_FIELD_FN(u64); 469 DEFINE_HIST_FIELD_FN(s32); 470 DEFINE_HIST_FIELD_FN(u32); 471 DEFINE_HIST_FIELD_FN(s16); 472 DEFINE_HIST_FIELD_FN(u16); 473 DEFINE_HIST_FIELD_FN(s8); 474 DEFINE_HIST_FIELD_FN(u8); 475 476 #define for_each_hist_field(i, hist_data) \ 477 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++) 478 479 #define for_each_hist_val_field(i, hist_data) \ 480 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++) 481 482 #define for_each_hist_key_field(i, hist_data) \ 483 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++) 484 485 #define HITCOUNT_IDX 0 486 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE) 487 488 enum hist_field_flags { 489 HIST_FIELD_FL_HITCOUNT = 1 << 0, 490 HIST_FIELD_FL_KEY = 1 << 1, 491 HIST_FIELD_FL_STRING = 1 << 2, 492 HIST_FIELD_FL_HEX = 1 << 3, 493 HIST_FIELD_FL_SYM = 1 << 4, 494 HIST_FIELD_FL_SYM_OFFSET = 1 << 5, 495 HIST_FIELD_FL_EXECNAME = 1 << 6, 496 HIST_FIELD_FL_SYSCALL = 1 << 7, 497 HIST_FIELD_FL_STACKTRACE = 1 << 8, 498 HIST_FIELD_FL_LOG2 = 1 << 9, 499 HIST_FIELD_FL_TIMESTAMP = 1 << 10, 500 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11, 501 HIST_FIELD_FL_VAR = 1 << 12, 502 HIST_FIELD_FL_EXPR = 1 << 13, 503 HIST_FIELD_FL_VAR_REF = 1 << 14, 504 HIST_FIELD_FL_CPU = 1 << 15, 505 HIST_FIELD_FL_ALIAS = 1 << 16, 506 HIST_FIELD_FL_BUCKET = 1 << 17, 507 HIST_FIELD_FL_CONST = 1 << 18, 508 HIST_FIELD_FL_PERCENT = 1 << 19, 509 HIST_FIELD_FL_GRAPH = 1 << 20, 510 HIST_FIELD_FL_COMM = 1 << 21, 511 }; 512 513 struct var_defs { 514 unsigned int n_vars; 515 char *name[TRACING_MAP_VARS_MAX]; 516 char *expr[TRACING_MAP_VARS_MAX]; 517 }; 518 519 struct hist_trigger_attrs { 520 char *keys_str; 521 char *vals_str; 522 char *sort_key_str; 523 char *name; 524 char *clock; 525 bool pause; 526 bool cont; 527 bool clear; 528 bool ts_in_usecs; 529 bool no_hitcount; 530 unsigned int map_bits; 531 532 char *assignment_str[TRACING_MAP_VARS_MAX]; 533 unsigned int n_assignments; 534 535 char *action_str[HIST_ACTIONS_MAX]; 536 unsigned int n_actions; 537 538 struct var_defs var_defs; 539 }; 540 541 struct field_var { 542 struct hist_field *var; 543 struct hist_field *val; 544 }; 545 546 struct field_var_hist { 547 struct hist_trigger_data *hist_data; 548 char *cmd; 549 }; 550 551 struct hist_trigger_data { 552 struct hist_field *fields[HIST_FIELDS_MAX]; 553 unsigned int n_vals; 554 unsigned int n_keys; 555 unsigned int n_fields; 556 unsigned int n_vars; 557 unsigned int n_var_str; 558 unsigned int key_size; 559 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX]; 560 unsigned int n_sort_keys; 561 struct trace_event_file *event_file; 562 struct hist_trigger_attrs *attrs; 563 struct tracing_map *map; 564 bool enable_timestamps; 565 bool remove; 566 struct hist_field *var_refs[TRACING_MAP_VARS_MAX]; 567 unsigned int n_var_refs; 568 569 struct action_data *actions[HIST_ACTIONS_MAX]; 570 unsigned int n_actions; 571 572 struct field_var *field_vars[SYNTH_FIELDS_MAX]; 573 unsigned int n_field_vars; 574 unsigned int n_field_var_str; 575 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX]; 576 unsigned int n_field_var_hists; 577 578 struct field_var *save_vars[SYNTH_FIELDS_MAX]; 579 unsigned int n_save_vars; 580 unsigned int n_save_var_str; 581 }; 582 583 struct action_data; 584 585 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data, 586 struct tracing_map_elt *elt, 587 struct trace_buffer *buffer, void *rec, 588 struct ring_buffer_event *rbe, void *key, 589 struct action_data *data, u64 *var_ref_vals); 590 591 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val); 592 593 enum handler_id { 594 HANDLER_ONMATCH = 1, 595 HANDLER_ONMAX, 596 HANDLER_ONCHANGE, 597 }; 598 599 enum action_id { 600 ACTION_SAVE = 1, 601 ACTION_TRACE, 602 ACTION_SNAPSHOT, 603 }; 604 605 struct action_data { 606 enum handler_id handler; 607 enum action_id action; 608 char *action_name; 609 action_fn_t fn; 610 611 unsigned int n_params; 612 char *params[SYNTH_FIELDS_MAX]; 613 614 /* 615 * When a histogram trigger is hit, the values of any 616 * references to variables, including variables being passed 617 * as parameters to synthetic events, are collected into a 618 * var_ref_vals array. This var_ref_idx array is an array of 619 * indices into the var_ref_vals array, one for each synthetic 620 * event param, and is passed to the synthetic event 621 * invocation. 622 */ 623 unsigned int var_ref_idx[SYNTH_FIELDS_MAX]; 624 struct synth_event *synth_event; 625 bool use_trace_keyword; 626 char *synth_event_name; 627 628 union { 629 struct { 630 char *event; 631 char *event_system; 632 } match_data; 633 634 struct { 635 /* 636 * var_str contains the $-unstripped variable 637 * name referenced by var_ref, and used when 638 * printing the action. Because var_ref 639 * creation is deferred to create_actions(), 640 * we need a per-action way to save it until 641 * then, thus var_str. 642 */ 643 char *var_str; 644 645 /* 646 * var_ref refers to the variable being 647 * tracked e.g onmax($var). 648 */ 649 struct hist_field *var_ref; 650 651 /* 652 * track_var contains the 'invisible' tracking 653 * variable created to keep the current 654 * e.g. max value. 655 */ 656 struct hist_field *track_var; 657 658 check_track_val_fn_t check_val; 659 action_fn_t save_data; 660 } track_data; 661 }; 662 }; 663 664 struct track_data { 665 u64 track_val; 666 bool updated; 667 668 unsigned int key_len; 669 void *key; 670 struct tracing_map_elt elt; 671 672 struct action_data *action_data; 673 struct hist_trigger_data *hist_data; 674 }; 675 676 struct hist_elt_data { 677 char *comm; 678 u64 *var_ref_vals; 679 char **field_var_str; 680 int n_field_var_str; 681 }; 682 683 struct snapshot_context { 684 struct tracing_map_elt *elt; 685 void *key; 686 }; 687 688 /* 689 * Returns the specific division function to use if the divisor 690 * is constant. This avoids extra branches when the trigger is hit. 691 */ 692 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor) 693 { 694 u64 div = divisor->constant; 695 696 if (!(div & (div - 1))) 697 return HIST_FIELD_FN_DIV_POWER2; 698 699 /* If the divisor is too large, do a regular division */ 700 if (div > (1 << HIST_DIV_SHIFT)) 701 return HIST_FIELD_FN_DIV_NOT_POWER2; 702 703 divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div); 704 return HIST_FIELD_FN_DIV_MULT_SHIFT; 705 } 706 707 static void track_data_free(struct track_data *track_data) 708 { 709 struct hist_elt_data *elt_data; 710 711 if (!track_data) 712 return; 713 714 kfree(track_data->key); 715 716 elt_data = track_data->elt.private_data; 717 if (elt_data) { 718 kfree(elt_data->comm); 719 kfree(elt_data); 720 } 721 722 kfree(track_data); 723 } 724 725 static struct track_data *track_data_alloc(unsigned int key_len, 726 struct action_data *action_data, 727 struct hist_trigger_data *hist_data) 728 { 729 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL); 730 struct hist_elt_data *elt_data; 731 732 if (!data) 733 return ERR_PTR(-ENOMEM); 734 735 data->key = kzalloc(key_len, GFP_KERNEL); 736 if (!data->key) { 737 track_data_free(data); 738 return ERR_PTR(-ENOMEM); 739 } 740 741 data->key_len = key_len; 742 data->action_data = action_data; 743 data->hist_data = hist_data; 744 745 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 746 if (!elt_data) { 747 track_data_free(data); 748 return ERR_PTR(-ENOMEM); 749 } 750 751 data->elt.private_data = elt_data; 752 753 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL); 754 if (!elt_data->comm) { 755 track_data_free(data); 756 return ERR_PTR(-ENOMEM); 757 } 758 759 return data; 760 } 761 762 #define HIST_PREFIX "hist:" 763 764 static char *last_cmd; 765 static char last_cmd_loc[MAX_FILTER_STR_VAL]; 766 767 static int errpos(char *str) 768 { 769 if (!str || !last_cmd) 770 return 0; 771 772 return err_pos(last_cmd, str); 773 } 774 775 static void last_cmd_set(struct trace_event_file *file, char *str) 776 { 777 const char *system = NULL, *name = NULL; 778 struct trace_event_call *call; 779 780 if (!str) 781 return; 782 783 kfree(last_cmd); 784 785 last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str); 786 if (!last_cmd) 787 return; 788 789 if (file) { 790 call = file->event_call; 791 system = call->class->system; 792 if (system) { 793 name = trace_event_name(call); 794 if (!name) 795 system = NULL; 796 } 797 } 798 799 if (system) 800 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name); 801 } 802 803 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos) 804 { 805 if (!last_cmd) 806 return; 807 808 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text, 809 err_type, err_pos); 810 } 811 812 static void hist_err_clear(void) 813 { 814 if (last_cmd) 815 last_cmd[0] = '\0'; 816 last_cmd_loc[0] = '\0'; 817 } 818 819 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals, 820 unsigned int *var_ref_idx); 821 822 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals, 823 unsigned int *var_ref_idx) 824 { 825 struct tracepoint *tp = event->tp; 826 827 if (unlikely(static_key_enabled(&tp->key))) { 828 struct tracepoint_func *probe_func_ptr; 829 synth_probe_func_t probe_func; 830 void *__data; 831 832 if (!(cpu_online(raw_smp_processor_id()))) 833 return; 834 835 probe_func_ptr = rcu_dereference_sched((tp)->funcs); 836 if (probe_func_ptr) { 837 do { 838 probe_func = probe_func_ptr->func; 839 __data = probe_func_ptr->data; 840 probe_func(__data, var_ref_vals, var_ref_idx); 841 } while ((++probe_func_ptr)->func); 842 } 843 } 844 } 845 846 static void action_trace(struct hist_trigger_data *hist_data, 847 struct tracing_map_elt *elt, 848 struct trace_buffer *buffer, void *rec, 849 struct ring_buffer_event *rbe, void *key, 850 struct action_data *data, u64 *var_ref_vals) 851 { 852 struct synth_event *event = data->synth_event; 853 854 trace_synth(event, var_ref_vals, data->var_ref_idx); 855 } 856 857 struct hist_var_data { 858 struct list_head list; 859 struct hist_trigger_data *hist_data; 860 }; 861 862 static u64 hist_field_timestamp(struct hist_field *hist_field, 863 struct tracing_map_elt *elt, 864 struct trace_buffer *buffer, 865 struct ring_buffer_event *rbe, 866 void *event) 867 { 868 struct hist_trigger_data *hist_data = hist_field->hist_data; 869 struct trace_array *tr = hist_data->event_file->tr; 870 871 u64 ts = ring_buffer_event_time_stamp(buffer, rbe); 872 873 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr)) 874 ts = ns2usecs(ts); 875 876 return ts; 877 } 878 879 static u64 hist_field_cpu(struct hist_field *hist_field, 880 struct tracing_map_elt *elt, 881 struct trace_buffer *buffer, 882 struct ring_buffer_event *rbe, 883 void *event) 884 { 885 int cpu = smp_processor_id(); 886 887 return cpu; 888 } 889 890 static u64 hist_field_comm(struct hist_field *hist_field, 891 struct tracing_map_elt *elt, 892 struct trace_buffer *buffer, 893 struct ring_buffer_event *rbe, 894 void *event) 895 { 896 return (u64)(unsigned long)current->comm; 897 } 898 899 /** 900 * check_field_for_var_ref - Check if a VAR_REF field references a variable 901 * @hist_field: The VAR_REF field to check 902 * @var_data: The hist trigger that owns the variable 903 * @var_idx: The trigger variable identifier 904 * 905 * Check the given VAR_REF field to see whether or not it references 906 * the given variable associated with the given trigger. 907 * 908 * Return: The VAR_REF field if it does reference the variable, NULL if not 909 */ 910 static struct hist_field * 911 check_field_for_var_ref(struct hist_field *hist_field, 912 struct hist_trigger_data *var_data, 913 unsigned int var_idx) 914 { 915 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF)); 916 917 if (hist_field && hist_field->var.idx == var_idx && 918 hist_field->var.hist_data == var_data) 919 return hist_field; 920 921 return NULL; 922 } 923 924 /** 925 * find_var_ref - Check if a trigger has a reference to a trigger variable 926 * @hist_data: The hist trigger that might have a reference to the variable 927 * @var_data: The hist trigger that owns the variable 928 * @var_idx: The trigger variable identifier 929 * 930 * Check the list of var_refs[] on the first hist trigger to see 931 * whether any of them are references to the variable on the second 932 * trigger. 933 * 934 * Return: The VAR_REF field referencing the variable if so, NULL if not 935 */ 936 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data, 937 struct hist_trigger_data *var_data, 938 unsigned int var_idx) 939 { 940 struct hist_field *hist_field; 941 unsigned int i; 942 943 for (i = 0; i < hist_data->n_var_refs; i++) { 944 hist_field = hist_data->var_refs[i]; 945 if (check_field_for_var_ref(hist_field, var_data, var_idx)) 946 return hist_field; 947 } 948 949 return NULL; 950 } 951 952 /** 953 * find_any_var_ref - Check if there is a reference to a given trigger variable 954 * @hist_data: The hist trigger 955 * @var_idx: The trigger variable identifier 956 * 957 * Check to see whether the given variable is currently referenced by 958 * any other trigger. 959 * 960 * The trigger the variable is defined on is explicitly excluded - the 961 * assumption being that a self-reference doesn't prevent a trigger 962 * from being removed. 963 * 964 * Return: The VAR_REF field referencing the variable if so, NULL if not 965 */ 966 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data, 967 unsigned int var_idx) 968 { 969 struct trace_array *tr = hist_data->event_file->tr; 970 struct hist_field *found = NULL; 971 struct hist_var_data *var_data; 972 973 list_for_each_entry(var_data, &tr->hist_vars, list) { 974 if (var_data->hist_data == hist_data) 975 continue; 976 found = find_var_ref(var_data->hist_data, hist_data, var_idx); 977 if (found) 978 break; 979 } 980 981 return found; 982 } 983 984 /** 985 * check_var_refs - Check if there is a reference to any of trigger's variables 986 * @hist_data: The hist trigger 987 * 988 * A trigger can define one or more variables. If any one of them is 989 * currently referenced by any other trigger, this function will 990 * determine that. 991 * 992 * Typically used to determine whether or not a trigger can be removed 993 * - if there are any references to a trigger's variables, it cannot. 994 * 995 * Return: True if there is a reference to any of trigger's variables 996 */ 997 static bool check_var_refs(struct hist_trigger_data *hist_data) 998 { 999 struct hist_field *field; 1000 bool found = false; 1001 int i; 1002 1003 for_each_hist_field(i, hist_data) { 1004 field = hist_data->fields[i]; 1005 if (field && field->flags & HIST_FIELD_FL_VAR) { 1006 if (find_any_var_ref(hist_data, field->var.idx)) { 1007 found = true; 1008 break; 1009 } 1010 } 1011 } 1012 1013 return found; 1014 } 1015 1016 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data) 1017 { 1018 struct trace_array *tr = hist_data->event_file->tr; 1019 struct hist_var_data *var_data, *found = NULL; 1020 1021 list_for_each_entry(var_data, &tr->hist_vars, list) { 1022 if (var_data->hist_data == hist_data) { 1023 found = var_data; 1024 break; 1025 } 1026 } 1027 1028 return found; 1029 } 1030 1031 static bool field_has_hist_vars(struct hist_field *hist_field, 1032 unsigned int level) 1033 { 1034 int i; 1035 1036 if (level > 3) 1037 return false; 1038 1039 if (!hist_field) 1040 return false; 1041 1042 if (hist_field->flags & HIST_FIELD_FL_VAR || 1043 hist_field->flags & HIST_FIELD_FL_VAR_REF) 1044 return true; 1045 1046 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) { 1047 struct hist_field *operand; 1048 1049 operand = hist_field->operands[i]; 1050 if (field_has_hist_vars(operand, level + 1)) 1051 return true; 1052 } 1053 1054 return false; 1055 } 1056 1057 static bool has_hist_vars(struct hist_trigger_data *hist_data) 1058 { 1059 struct hist_field *hist_field; 1060 int i; 1061 1062 for_each_hist_field(i, hist_data) { 1063 hist_field = hist_data->fields[i]; 1064 if (field_has_hist_vars(hist_field, 0)) 1065 return true; 1066 } 1067 1068 return false; 1069 } 1070 1071 static int save_hist_vars(struct hist_trigger_data *hist_data) 1072 { 1073 struct trace_array *tr = hist_data->event_file->tr; 1074 struct hist_var_data *var_data; 1075 1076 var_data = find_hist_vars(hist_data); 1077 if (var_data) 1078 return 0; 1079 1080 if (tracing_check_open_get_tr(tr)) 1081 return -ENODEV; 1082 1083 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL); 1084 if (!var_data) { 1085 trace_array_put(tr); 1086 return -ENOMEM; 1087 } 1088 1089 var_data->hist_data = hist_data; 1090 list_add(&var_data->list, &tr->hist_vars); 1091 1092 return 0; 1093 } 1094 1095 static void remove_hist_vars(struct hist_trigger_data *hist_data) 1096 { 1097 struct trace_array *tr = hist_data->event_file->tr; 1098 struct hist_var_data *var_data; 1099 1100 var_data = find_hist_vars(hist_data); 1101 if (!var_data) 1102 return; 1103 1104 if (WARN_ON(check_var_refs(hist_data))) 1105 return; 1106 1107 list_del(&var_data->list); 1108 1109 kfree(var_data); 1110 1111 trace_array_put(tr); 1112 } 1113 1114 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data, 1115 const char *var_name) 1116 { 1117 struct hist_field *hist_field, *found = NULL; 1118 int i; 1119 1120 for_each_hist_field(i, hist_data) { 1121 hist_field = hist_data->fields[i]; 1122 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR && 1123 strcmp(hist_field->var.name, var_name) == 0) { 1124 found = hist_field; 1125 break; 1126 } 1127 } 1128 1129 return found; 1130 } 1131 1132 static struct hist_field *find_var(struct hist_trigger_data *hist_data, 1133 struct trace_event_file *file, 1134 const char *var_name) 1135 { 1136 struct hist_trigger_data *test_data; 1137 struct event_trigger_data *test; 1138 struct hist_field *hist_field; 1139 1140 lockdep_assert_held(&event_mutex); 1141 1142 hist_field = find_var_field(hist_data, var_name); 1143 if (hist_field) 1144 return hist_field; 1145 1146 list_for_each_entry(test, &file->triggers, list) { 1147 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1148 test_data = test->private_data; 1149 hist_field = find_var_field(test_data, var_name); 1150 if (hist_field) 1151 return hist_field; 1152 } 1153 } 1154 1155 return NULL; 1156 } 1157 1158 static struct trace_event_file *find_var_file(struct trace_array *tr, 1159 char *system, 1160 char *event_name, 1161 char *var_name) 1162 { 1163 struct hist_trigger_data *var_hist_data; 1164 struct hist_var_data *var_data; 1165 struct trace_event_file *file, *found = NULL; 1166 1167 if (system) 1168 return find_event_file(tr, system, event_name); 1169 1170 list_for_each_entry(var_data, &tr->hist_vars, list) { 1171 var_hist_data = var_data->hist_data; 1172 file = var_hist_data->event_file; 1173 if (file == found) 1174 continue; 1175 1176 if (find_var_field(var_hist_data, var_name)) { 1177 if (found) { 1178 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name)); 1179 return NULL; 1180 } 1181 1182 found = file; 1183 } 1184 } 1185 1186 return found; 1187 } 1188 1189 static struct hist_field *find_file_var(struct trace_event_file *file, 1190 const char *var_name) 1191 { 1192 struct hist_trigger_data *test_data; 1193 struct event_trigger_data *test; 1194 struct hist_field *hist_field; 1195 1196 lockdep_assert_held(&event_mutex); 1197 1198 list_for_each_entry(test, &file->triggers, list) { 1199 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 1200 test_data = test->private_data; 1201 hist_field = find_var_field(test_data, var_name); 1202 if (hist_field) 1203 return hist_field; 1204 } 1205 } 1206 1207 return NULL; 1208 } 1209 1210 static struct hist_field * 1211 find_match_var(struct hist_trigger_data *hist_data, char *var_name) 1212 { 1213 struct trace_array *tr = hist_data->event_file->tr; 1214 struct hist_field *hist_field, *found = NULL; 1215 struct trace_event_file *file; 1216 unsigned int i; 1217 1218 for (i = 0; i < hist_data->n_actions; i++) { 1219 struct action_data *data = hist_data->actions[i]; 1220 1221 if (data->handler == HANDLER_ONMATCH) { 1222 char *system = data->match_data.event_system; 1223 char *event_name = data->match_data.event; 1224 1225 file = find_var_file(tr, system, event_name, var_name); 1226 if (!file) 1227 continue; 1228 hist_field = find_file_var(file, var_name); 1229 if (hist_field) { 1230 if (found) { 1231 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, 1232 errpos(var_name)); 1233 return ERR_PTR(-EINVAL); 1234 } 1235 1236 found = hist_field; 1237 } 1238 } 1239 } 1240 return found; 1241 } 1242 1243 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data, 1244 char *system, 1245 char *event_name, 1246 char *var_name) 1247 { 1248 struct trace_array *tr = hist_data->event_file->tr; 1249 struct hist_field *hist_field = NULL; 1250 struct trace_event_file *file; 1251 1252 if (!system || !event_name) { 1253 hist_field = find_match_var(hist_data, var_name); 1254 if (IS_ERR(hist_field)) 1255 return NULL; 1256 if (hist_field) 1257 return hist_field; 1258 } 1259 1260 file = find_var_file(tr, system, event_name, var_name); 1261 if (!file) 1262 return NULL; 1263 1264 hist_field = find_file_var(file, var_name); 1265 1266 return hist_field; 1267 } 1268 1269 static u64 hist_field_var_ref(struct hist_field *hist_field, 1270 struct tracing_map_elt *elt, 1271 struct trace_buffer *buffer, 1272 struct ring_buffer_event *rbe, 1273 void *event) 1274 { 1275 struct hist_elt_data *elt_data; 1276 u64 var_val = 0; 1277 1278 if (WARN_ON_ONCE(!elt)) 1279 return var_val; 1280 1281 elt_data = elt->private_data; 1282 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx]; 1283 1284 return var_val; 1285 } 1286 1287 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key, 1288 u64 *var_ref_vals, bool self) 1289 { 1290 struct hist_trigger_data *var_data; 1291 struct tracing_map_elt *var_elt; 1292 struct hist_field *hist_field; 1293 unsigned int i, var_idx; 1294 bool resolved = true; 1295 u64 var_val = 0; 1296 1297 for (i = 0; i < hist_data->n_var_refs; i++) { 1298 hist_field = hist_data->var_refs[i]; 1299 var_idx = hist_field->var.idx; 1300 var_data = hist_field->var.hist_data; 1301 1302 if (var_data == NULL) { 1303 resolved = false; 1304 break; 1305 } 1306 1307 if ((self && var_data != hist_data) || 1308 (!self && var_data == hist_data)) 1309 continue; 1310 1311 var_elt = tracing_map_lookup(var_data->map, key); 1312 if (!var_elt) { 1313 resolved = false; 1314 break; 1315 } 1316 1317 if (!tracing_map_var_set(var_elt, var_idx)) { 1318 resolved = false; 1319 break; 1320 } 1321 1322 if (self || !hist_field->read_once) 1323 var_val = tracing_map_read_var(var_elt, var_idx); 1324 else 1325 var_val = tracing_map_read_var_once(var_elt, var_idx); 1326 1327 var_ref_vals[i] = var_val; 1328 } 1329 1330 return resolved; 1331 } 1332 1333 static const char *hist_field_name(struct hist_field *field, 1334 unsigned int level) 1335 { 1336 const char *field_name = ""; 1337 1338 if (WARN_ON_ONCE(!field)) 1339 return field_name; 1340 1341 if (level > 1) 1342 return field_name; 1343 1344 if (field->field) 1345 field_name = field->field->name; 1346 else if (field->flags & HIST_FIELD_FL_LOG2 || 1347 field->flags & HIST_FIELD_FL_ALIAS || 1348 field->flags & HIST_FIELD_FL_BUCKET) 1349 field_name = hist_field_name(field->operands[0], ++level); 1350 else if (field->flags & HIST_FIELD_FL_CPU) 1351 field_name = "common_cpu"; 1352 else if (field->flags & HIST_FIELD_FL_COMM) 1353 field_name = "common_comm"; 1354 else if (field->flags & HIST_FIELD_FL_EXPR || 1355 field->flags & HIST_FIELD_FL_VAR_REF) { 1356 if (field->system) { 1357 static char full_name[MAX_FILTER_STR_VAL]; 1358 1359 strcat(full_name, field->system); 1360 strcat(full_name, "."); 1361 strcat(full_name, field->event_name); 1362 strcat(full_name, "."); 1363 strcat(full_name, field->name); 1364 field_name = full_name; 1365 } else 1366 field_name = field->name; 1367 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP) 1368 field_name = "common_timestamp"; 1369 else if (field->flags & HIST_FIELD_FL_STACKTRACE) { 1370 field_name = "common_stacktrace"; 1371 } else if (field->flags & HIST_FIELD_FL_HITCOUNT) 1372 field_name = "hitcount"; 1373 1374 if (field_name == NULL) 1375 field_name = ""; 1376 1377 return field_name; 1378 } 1379 1380 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed) 1381 { 1382 switch (field_size) { 1383 case 8: 1384 if (field_is_signed) 1385 return HIST_FIELD_FN_S64; 1386 else 1387 return HIST_FIELD_FN_U64; 1388 case 4: 1389 if (field_is_signed) 1390 return HIST_FIELD_FN_S32; 1391 else 1392 return HIST_FIELD_FN_U32; 1393 case 2: 1394 if (field_is_signed) 1395 return HIST_FIELD_FN_S16; 1396 else 1397 return HIST_FIELD_FN_U16; 1398 case 1: 1399 if (field_is_signed) 1400 return HIST_FIELD_FN_S8; 1401 else 1402 return HIST_FIELD_FN_U8; 1403 } 1404 1405 return HIST_FIELD_FN_NOP; 1406 } 1407 1408 static int parse_map_size(char *str) 1409 { 1410 unsigned long size, map_bits; 1411 int ret; 1412 1413 ret = kstrtoul(str, 0, &size); 1414 if (ret) 1415 goto out; 1416 1417 map_bits = ilog2(roundup_pow_of_two(size)); 1418 if (map_bits < TRACING_MAP_BITS_MIN || 1419 map_bits > TRACING_MAP_BITS_MAX) 1420 ret = -EINVAL; 1421 else 1422 ret = map_bits; 1423 out: 1424 return ret; 1425 } 1426 1427 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs) 1428 { 1429 unsigned int i; 1430 1431 if (!attrs) 1432 return; 1433 1434 for (i = 0; i < attrs->n_assignments; i++) 1435 kfree(attrs->assignment_str[i]); 1436 1437 for (i = 0; i < attrs->n_actions; i++) 1438 kfree(attrs->action_str[i]); 1439 1440 kfree(attrs->name); 1441 kfree(attrs->sort_key_str); 1442 kfree(attrs->keys_str); 1443 kfree(attrs->vals_str); 1444 kfree(attrs->clock); 1445 kfree(attrs); 1446 } 1447 1448 static int parse_action(char *str, struct hist_trigger_attrs *attrs) 1449 { 1450 int ret = -EINVAL; 1451 1452 if (attrs->n_actions >= HIST_ACTIONS_MAX) 1453 return ret; 1454 1455 if ((str_has_prefix(str, "onmatch(")) || 1456 (str_has_prefix(str, "onmax(")) || 1457 (str_has_prefix(str, "onchange("))) { 1458 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL); 1459 if (!attrs->action_str[attrs->n_actions]) { 1460 ret = -ENOMEM; 1461 return ret; 1462 } 1463 attrs->n_actions++; 1464 ret = 0; 1465 } 1466 return ret; 1467 } 1468 1469 static int parse_assignment(struct trace_array *tr, 1470 char *str, struct hist_trigger_attrs *attrs) 1471 { 1472 int len, ret = 0; 1473 1474 if ((len = str_has_prefix(str, "key=")) || 1475 (len = str_has_prefix(str, "keys="))) { 1476 attrs->keys_str = kstrdup(str + len, GFP_KERNEL); 1477 if (!attrs->keys_str) { 1478 ret = -ENOMEM; 1479 goto out; 1480 } 1481 } else if ((len = str_has_prefix(str, "val=")) || 1482 (len = str_has_prefix(str, "vals=")) || 1483 (len = str_has_prefix(str, "values="))) { 1484 attrs->vals_str = kstrdup(str + len, GFP_KERNEL); 1485 if (!attrs->vals_str) { 1486 ret = -ENOMEM; 1487 goto out; 1488 } 1489 } else if ((len = str_has_prefix(str, "sort="))) { 1490 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL); 1491 if (!attrs->sort_key_str) { 1492 ret = -ENOMEM; 1493 goto out; 1494 } 1495 } else if (str_has_prefix(str, "name=")) { 1496 attrs->name = kstrdup(str, GFP_KERNEL); 1497 if (!attrs->name) { 1498 ret = -ENOMEM; 1499 goto out; 1500 } 1501 } else if ((len = str_has_prefix(str, "clock="))) { 1502 str += len; 1503 1504 str = strstrip(str); 1505 attrs->clock = kstrdup(str, GFP_KERNEL); 1506 if (!attrs->clock) { 1507 ret = -ENOMEM; 1508 goto out; 1509 } 1510 } else if ((len = str_has_prefix(str, "size="))) { 1511 int map_bits = parse_map_size(str + len); 1512 1513 if (map_bits < 0) { 1514 ret = map_bits; 1515 goto out; 1516 } 1517 attrs->map_bits = map_bits; 1518 } else { 1519 char *assignment; 1520 1521 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) { 1522 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str)); 1523 ret = -EINVAL; 1524 goto out; 1525 } 1526 1527 assignment = kstrdup(str, GFP_KERNEL); 1528 if (!assignment) { 1529 ret = -ENOMEM; 1530 goto out; 1531 } 1532 1533 attrs->assignment_str[attrs->n_assignments++] = assignment; 1534 } 1535 out: 1536 return ret; 1537 } 1538 1539 static struct hist_trigger_attrs * 1540 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str) 1541 { 1542 struct hist_trigger_attrs *attrs; 1543 int ret = 0; 1544 1545 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL); 1546 if (!attrs) 1547 return ERR_PTR(-ENOMEM); 1548 1549 while (trigger_str) { 1550 char *str = strsep(&trigger_str, ":"); 1551 char *rhs; 1552 1553 rhs = strchr(str, '='); 1554 if (rhs) { 1555 if (!strlen(++rhs)) { 1556 ret = -EINVAL; 1557 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str)); 1558 goto free; 1559 } 1560 ret = parse_assignment(tr, str, attrs); 1561 if (ret) 1562 goto free; 1563 } else if (strcmp(str, "nohitcount") == 0 || 1564 strcmp(str, "NOHC") == 0) 1565 attrs->no_hitcount = true; 1566 else if (strcmp(str, "pause") == 0) 1567 attrs->pause = true; 1568 else if ((strcmp(str, "cont") == 0) || 1569 (strcmp(str, "continue") == 0)) 1570 attrs->cont = true; 1571 else if (strcmp(str, "clear") == 0) 1572 attrs->clear = true; 1573 else { 1574 ret = parse_action(str, attrs); 1575 if (ret) 1576 goto free; 1577 } 1578 } 1579 1580 if (!attrs->keys_str) { 1581 ret = -EINVAL; 1582 goto free; 1583 } 1584 1585 if (!attrs->clock) { 1586 attrs->clock = kstrdup("global", GFP_KERNEL); 1587 if (!attrs->clock) { 1588 ret = -ENOMEM; 1589 goto free; 1590 } 1591 } 1592 1593 return attrs; 1594 free: 1595 destroy_hist_trigger_attrs(attrs); 1596 1597 return ERR_PTR(ret); 1598 } 1599 1600 static inline void save_comm(char *comm, struct task_struct *task) 1601 { 1602 if (!task->pid) { 1603 strcpy(comm, "<idle>"); 1604 return; 1605 } 1606 1607 if (WARN_ON_ONCE(task->pid < 0)) { 1608 strcpy(comm, "<XXX>"); 1609 return; 1610 } 1611 1612 strscpy(comm, task->comm, TASK_COMM_LEN); 1613 } 1614 1615 static void hist_elt_data_free(struct hist_elt_data *elt_data) 1616 { 1617 unsigned int i; 1618 1619 for (i = 0; i < elt_data->n_field_var_str; i++) 1620 kfree(elt_data->field_var_str[i]); 1621 1622 kfree(elt_data->field_var_str); 1623 1624 kfree(elt_data->comm); 1625 kfree(elt_data); 1626 } 1627 1628 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt) 1629 { 1630 struct hist_elt_data *elt_data = elt->private_data; 1631 1632 hist_elt_data_free(elt_data); 1633 } 1634 1635 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt) 1636 { 1637 struct hist_trigger_data *hist_data = elt->map->private_data; 1638 unsigned int size = TASK_COMM_LEN; 1639 struct hist_elt_data *elt_data; 1640 struct hist_field *hist_field; 1641 unsigned int i, n_str; 1642 1643 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL); 1644 if (!elt_data) 1645 return -ENOMEM; 1646 1647 for_each_hist_field(i, hist_data) { 1648 hist_field = hist_data->fields[i]; 1649 1650 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) { 1651 elt_data->comm = kzalloc(size, GFP_KERNEL); 1652 if (!elt_data->comm) { 1653 kfree(elt_data); 1654 return -ENOMEM; 1655 } 1656 break; 1657 } 1658 } 1659 1660 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str + 1661 hist_data->n_var_str; 1662 if (n_str > SYNTH_FIELDS_MAX) { 1663 hist_elt_data_free(elt_data); 1664 return -EINVAL; 1665 } 1666 1667 BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1)); 1668 1669 size = STR_VAR_LEN_MAX; 1670 1671 elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL); 1672 if (!elt_data->field_var_str) { 1673 hist_elt_data_free(elt_data); 1674 return -EINVAL; 1675 } 1676 elt_data->n_field_var_str = n_str; 1677 1678 for (i = 0; i < n_str; i++) { 1679 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL); 1680 if (!elt_data->field_var_str[i]) { 1681 hist_elt_data_free(elt_data); 1682 return -ENOMEM; 1683 } 1684 } 1685 1686 elt->private_data = elt_data; 1687 1688 return 0; 1689 } 1690 1691 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt) 1692 { 1693 struct hist_elt_data *elt_data = elt->private_data; 1694 1695 if (elt_data->comm) 1696 save_comm(elt_data->comm, current); 1697 } 1698 1699 static const struct tracing_map_ops hist_trigger_elt_data_ops = { 1700 .elt_alloc = hist_trigger_elt_data_alloc, 1701 .elt_free = hist_trigger_elt_data_free, 1702 .elt_init = hist_trigger_elt_data_init, 1703 }; 1704 1705 static const char *get_hist_field_flags(struct hist_field *hist_field) 1706 { 1707 const char *flags_str = NULL; 1708 1709 if (hist_field->flags & HIST_FIELD_FL_HEX) 1710 flags_str = "hex"; 1711 else if (hist_field->flags & HIST_FIELD_FL_SYM) 1712 flags_str = "sym"; 1713 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET) 1714 flags_str = "sym-offset"; 1715 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME) 1716 flags_str = "execname"; 1717 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL) 1718 flags_str = "syscall"; 1719 else if (hist_field->flags & HIST_FIELD_FL_LOG2) 1720 flags_str = "log2"; 1721 else if (hist_field->flags & HIST_FIELD_FL_BUCKET) 1722 flags_str = "buckets"; 1723 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS) 1724 flags_str = "usecs"; 1725 else if (hist_field->flags & HIST_FIELD_FL_PERCENT) 1726 flags_str = "percent"; 1727 else if (hist_field->flags & HIST_FIELD_FL_GRAPH) 1728 flags_str = "graph"; 1729 else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 1730 flags_str = "stacktrace"; 1731 1732 return flags_str; 1733 } 1734 1735 static void expr_field_str(struct hist_field *field, char *expr) 1736 { 1737 if (field->flags & HIST_FIELD_FL_VAR_REF) 1738 strcat(expr, "$"); 1739 else if (field->flags & HIST_FIELD_FL_CONST) { 1740 char str[HIST_CONST_DIGITS_MAX]; 1741 1742 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant); 1743 strcat(expr, str); 1744 } 1745 1746 strcat(expr, hist_field_name(field, 0)); 1747 1748 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) { 1749 const char *flags_str = get_hist_field_flags(field); 1750 1751 if (flags_str) { 1752 strcat(expr, "."); 1753 strcat(expr, flags_str); 1754 } 1755 } 1756 } 1757 1758 static char *expr_str(struct hist_field *field, unsigned int level) 1759 { 1760 char *expr; 1761 1762 if (level > 1) 1763 return NULL; 1764 1765 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 1766 if (!expr) 1767 return NULL; 1768 1769 if (!field->operands[0]) { 1770 expr_field_str(field, expr); 1771 return expr; 1772 } 1773 1774 if (field->operator == FIELD_OP_UNARY_MINUS) { 1775 char *subexpr; 1776 1777 strcat(expr, "-("); 1778 subexpr = expr_str(field->operands[0], ++level); 1779 if (!subexpr) { 1780 kfree(expr); 1781 return NULL; 1782 } 1783 strcat(expr, subexpr); 1784 strcat(expr, ")"); 1785 1786 kfree(subexpr); 1787 1788 return expr; 1789 } 1790 1791 expr_field_str(field->operands[0], expr); 1792 1793 switch (field->operator) { 1794 case FIELD_OP_MINUS: 1795 strcat(expr, "-"); 1796 break; 1797 case FIELD_OP_PLUS: 1798 strcat(expr, "+"); 1799 break; 1800 case FIELD_OP_DIV: 1801 strcat(expr, "/"); 1802 break; 1803 case FIELD_OP_MULT: 1804 strcat(expr, "*"); 1805 break; 1806 default: 1807 kfree(expr); 1808 return NULL; 1809 } 1810 1811 expr_field_str(field->operands[1], expr); 1812 1813 return expr; 1814 } 1815 1816 /* 1817 * If field_op != FIELD_OP_NONE, *sep points to the root operator 1818 * of the expression tree to be evaluated. 1819 */ 1820 static int contains_operator(char *str, char **sep) 1821 { 1822 enum field_op_id field_op = FIELD_OP_NONE; 1823 char *minus_op, *plus_op, *div_op, *mult_op; 1824 1825 1826 /* 1827 * Report the last occurrence of the operators first, so that the 1828 * expression is evaluated left to right. This is important since 1829 * subtraction and division are not associative. 1830 * 1831 * e.g 1832 * 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2 1833 * 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2 1834 */ 1835 1836 /* 1837 * First, find lower precedence addition and subtraction 1838 * since the expression will be evaluated recursively. 1839 */ 1840 minus_op = strrchr(str, '-'); 1841 if (minus_op) { 1842 /* 1843 * Unary minus is not supported in sub-expressions. If 1844 * present, it is always the next root operator. 1845 */ 1846 if (minus_op == str) { 1847 field_op = FIELD_OP_UNARY_MINUS; 1848 goto out; 1849 } 1850 1851 field_op = FIELD_OP_MINUS; 1852 } 1853 1854 plus_op = strrchr(str, '+'); 1855 if (plus_op || minus_op) { 1856 /* 1857 * For operators of the same precedence use to rightmost as the 1858 * root, so that the expression is evaluated left to right. 1859 */ 1860 if (plus_op > minus_op) 1861 field_op = FIELD_OP_PLUS; 1862 goto out; 1863 } 1864 1865 /* 1866 * Multiplication and division have higher precedence than addition and 1867 * subtraction. 1868 */ 1869 div_op = strrchr(str, '/'); 1870 if (div_op) 1871 field_op = FIELD_OP_DIV; 1872 1873 mult_op = strrchr(str, '*'); 1874 /* 1875 * For operators of the same precedence use to rightmost as the 1876 * root, so that the expression is evaluated left to right. 1877 */ 1878 if (mult_op > div_op) 1879 field_op = FIELD_OP_MULT; 1880 1881 out: 1882 if (sep) { 1883 switch (field_op) { 1884 case FIELD_OP_UNARY_MINUS: 1885 case FIELD_OP_MINUS: 1886 *sep = minus_op; 1887 break; 1888 case FIELD_OP_PLUS: 1889 *sep = plus_op; 1890 break; 1891 case FIELD_OP_DIV: 1892 *sep = div_op; 1893 break; 1894 case FIELD_OP_MULT: 1895 *sep = mult_op; 1896 break; 1897 case FIELD_OP_NONE: 1898 default: 1899 *sep = NULL; 1900 break; 1901 } 1902 } 1903 1904 return field_op; 1905 } 1906 1907 static void get_hist_field(struct hist_field *hist_field) 1908 { 1909 hist_field->ref++; 1910 } 1911 1912 static void __destroy_hist_field(struct hist_field *hist_field) 1913 { 1914 if (--hist_field->ref > 1) 1915 return; 1916 1917 kfree(hist_field->var.name); 1918 kfree(hist_field->name); 1919 1920 /* Can likely be a const */ 1921 kfree_const(hist_field->type); 1922 1923 kfree(hist_field->system); 1924 kfree(hist_field->event_name); 1925 1926 kfree(hist_field); 1927 } 1928 1929 static void destroy_hist_field(struct hist_field *hist_field, 1930 unsigned int level) 1931 { 1932 unsigned int i; 1933 1934 if (level > 3) 1935 return; 1936 1937 if (!hist_field) 1938 return; 1939 1940 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) 1941 return; /* var refs will be destroyed separately */ 1942 1943 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) 1944 destroy_hist_field(hist_field->operands[i], level + 1); 1945 1946 __destroy_hist_field(hist_field); 1947 } 1948 1949 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, 1950 struct ftrace_event_field *field, 1951 unsigned long flags, 1952 char *var_name) 1953 { 1954 struct hist_field *hist_field; 1955 1956 if (field && is_function_field(field)) 1957 return NULL; 1958 1959 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 1960 if (!hist_field) 1961 return NULL; 1962 1963 hist_field->ref = 1; 1964 1965 hist_field->hist_data = hist_data; 1966 1967 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) 1968 goto out; /* caller will populate */ 1969 1970 if (flags & HIST_FIELD_FL_VAR_REF) { 1971 hist_field->fn_num = HIST_FIELD_FN_VAR_REF; 1972 goto out; 1973 } 1974 1975 if (flags & HIST_FIELD_FL_HITCOUNT) { 1976 hist_field->fn_num = HIST_FIELD_FN_COUNTER; 1977 hist_field->size = sizeof(u64); 1978 hist_field->type = "u64"; 1979 goto out; 1980 } 1981 1982 if (flags & HIST_FIELD_FL_CONST) { 1983 hist_field->fn_num = HIST_FIELD_FN_CONST; 1984 hist_field->size = sizeof(u64); 1985 hist_field->type = kstrdup("u64", GFP_KERNEL); 1986 if (!hist_field->type) 1987 goto free; 1988 goto out; 1989 } 1990 1991 if (flags & HIST_FIELD_FL_STACKTRACE) { 1992 if (field) 1993 hist_field->fn_num = HIST_FIELD_FN_STACK; 1994 else 1995 hist_field->fn_num = HIST_FIELD_FN_NOP; 1996 hist_field->size = HIST_STACKTRACE_SIZE; 1997 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL); 1998 if (!hist_field->type) 1999 goto free; 2000 goto out; 2001 } 2002 2003 if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) { 2004 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET); 2005 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 : 2006 HIST_FIELD_FN_BUCKET; 2007 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); 2008 if (!hist_field->operands[0]) 2009 goto free; 2010 hist_field->size = hist_field->operands[0]->size; 2011 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL); 2012 if (!hist_field->type) 2013 goto free; 2014 goto out; 2015 } 2016 2017 if (flags & HIST_FIELD_FL_TIMESTAMP) { 2018 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP; 2019 hist_field->size = sizeof(u64); 2020 hist_field->type = "u64"; 2021 goto out; 2022 } 2023 2024 if (flags & HIST_FIELD_FL_CPU) { 2025 hist_field->fn_num = HIST_FIELD_FN_CPU; 2026 hist_field->size = sizeof(int); 2027 hist_field->type = "unsigned int"; 2028 goto out; 2029 } 2030 2031 if (flags & HIST_FIELD_FL_COMM) { 2032 hist_field->fn_num = HIST_FIELD_FN_COMM; 2033 hist_field->size = MAX_FILTER_STR_VAL; 2034 hist_field->type = "char[]"; 2035 goto out; 2036 } 2037 2038 if (WARN_ON_ONCE(!field)) 2039 goto out; 2040 2041 /* Pointers to strings are just pointers and dangerous to dereference */ 2042 if (is_string_field(field) && 2043 (field->filter_type != FILTER_PTR_STRING)) { 2044 flags |= HIST_FIELD_FL_STRING; 2045 2046 hist_field->size = MAX_FILTER_STR_VAL; 2047 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2048 if (!hist_field->type) 2049 goto free; 2050 2051 if (field->filter_type == FILTER_STATIC_STRING) { 2052 hist_field->fn_num = HIST_FIELD_FN_STRING; 2053 hist_field->size = field->size; 2054 } else if (field->filter_type == FILTER_DYN_STRING) { 2055 hist_field->fn_num = HIST_FIELD_FN_DYNSTRING; 2056 } else if (field->filter_type == FILTER_RDYN_STRING) 2057 hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; 2058 else 2059 hist_field->fn_num = HIST_FIELD_FN_PSTRING; 2060 } else if (field->filter_type == FILTER_STACKTRACE) { 2061 flags |= HIST_FIELD_FL_STACKTRACE; 2062 2063 hist_field->size = MAX_FILTER_STR_VAL; 2064 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2065 if (!hist_field->type) 2066 goto free; 2067 2068 hist_field->fn_num = HIST_FIELD_FN_STACK; 2069 } else { 2070 hist_field->size = field->size; 2071 hist_field->is_signed = field->is_signed; 2072 hist_field->type = kstrdup_const(field->type, GFP_KERNEL); 2073 if (!hist_field->type) 2074 goto free; 2075 2076 hist_field->fn_num = select_value_fn(field->size, 2077 field->is_signed); 2078 if (hist_field->fn_num == HIST_FIELD_FN_NOP) { 2079 destroy_hist_field(hist_field, 0); 2080 return NULL; 2081 } 2082 } 2083 out: 2084 hist_field->field = field; 2085 hist_field->flags = flags; 2086 2087 if (var_name) { 2088 hist_field->var.name = kstrdup(var_name, GFP_KERNEL); 2089 if (!hist_field->var.name) 2090 goto free; 2091 } 2092 2093 return hist_field; 2094 free: 2095 destroy_hist_field(hist_field, 0); 2096 return NULL; 2097 } 2098 2099 static void destroy_hist_fields(struct hist_trigger_data *hist_data) 2100 { 2101 unsigned int i; 2102 2103 for (i = 0; i < HIST_FIELDS_MAX; i++) { 2104 if (hist_data->fields[i]) { 2105 destroy_hist_field(hist_data->fields[i], 0); 2106 hist_data->fields[i] = NULL; 2107 } 2108 } 2109 2110 for (i = 0; i < hist_data->n_var_refs; i++) { 2111 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF)); 2112 __destroy_hist_field(hist_data->var_refs[i]); 2113 hist_data->var_refs[i] = NULL; 2114 } 2115 } 2116 2117 static int init_var_ref(struct hist_field *ref_field, 2118 struct hist_field *var_field, 2119 char *system, char *event_name) 2120 { 2121 int err = 0; 2122 2123 ref_field->var.idx = var_field->var.idx; 2124 ref_field->var.hist_data = var_field->hist_data; 2125 ref_field->size = var_field->size; 2126 ref_field->is_signed = var_field->is_signed; 2127 ref_field->flags |= var_field->flags & 2128 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2129 2130 if (system) { 2131 ref_field->system = kstrdup(system, GFP_KERNEL); 2132 if (!ref_field->system) 2133 return -ENOMEM; 2134 } 2135 2136 if (event_name) { 2137 ref_field->event_name = kstrdup(event_name, GFP_KERNEL); 2138 if (!ref_field->event_name) { 2139 err = -ENOMEM; 2140 goto free; 2141 } 2142 } 2143 2144 if (var_field->var.name) { 2145 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL); 2146 if (!ref_field->name) { 2147 err = -ENOMEM; 2148 goto free; 2149 } 2150 } else if (var_field->name) { 2151 ref_field->name = kstrdup(var_field->name, GFP_KERNEL); 2152 if (!ref_field->name) { 2153 err = -ENOMEM; 2154 goto free; 2155 } 2156 } 2157 2158 ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL); 2159 if (!ref_field->type) { 2160 err = -ENOMEM; 2161 goto free; 2162 } 2163 out: 2164 return err; 2165 free: 2166 kfree(ref_field->system); 2167 ref_field->system = NULL; 2168 kfree(ref_field->event_name); 2169 ref_field->event_name = NULL; 2170 kfree(ref_field->name); 2171 ref_field->name = NULL; 2172 2173 goto out; 2174 } 2175 2176 static int find_var_ref_idx(struct hist_trigger_data *hist_data, 2177 struct hist_field *var_field) 2178 { 2179 struct hist_field *ref_field; 2180 int i; 2181 2182 for (i = 0; i < hist_data->n_var_refs; i++) { 2183 ref_field = hist_data->var_refs[i]; 2184 if (ref_field->var.idx == var_field->var.idx && 2185 ref_field->var.hist_data == var_field->hist_data) 2186 return i; 2187 } 2188 2189 return -ENOENT; 2190 } 2191 2192 /** 2193 * create_var_ref - Create a variable reference and attach it to trigger 2194 * @hist_data: The trigger that will be referencing the variable 2195 * @var_field: The VAR field to create a reference to 2196 * @system: The optional system string 2197 * @event_name: The optional event_name string 2198 * 2199 * Given a variable hist_field, create a VAR_REF hist_field that 2200 * represents a reference to it. 2201 * 2202 * This function also adds the reference to the trigger that 2203 * now references the variable. 2204 * 2205 * Return: The VAR_REF field if successful, NULL if not 2206 */ 2207 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, 2208 struct hist_field *var_field, 2209 char *system, char *event_name) 2210 { 2211 unsigned long flags = HIST_FIELD_FL_VAR_REF; 2212 struct hist_field *ref_field; 2213 int i; 2214 2215 /* Check if the variable already exists */ 2216 for (i = 0; i < hist_data->n_var_refs; i++) { 2217 ref_field = hist_data->var_refs[i]; 2218 if (ref_field->var.idx == var_field->var.idx && 2219 ref_field->var.hist_data == var_field->hist_data) { 2220 get_hist_field(ref_field); 2221 return ref_field; 2222 } 2223 } 2224 /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ 2225 if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) 2226 return NULL; 2227 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); 2228 if (ref_field) { 2229 if (init_var_ref(ref_field, var_field, system, event_name)) { 2230 destroy_hist_field(ref_field, 0); 2231 return NULL; 2232 } 2233 2234 hist_data->var_refs[hist_data->n_var_refs] = ref_field; 2235 ref_field->var_ref_idx = hist_data->n_var_refs++; 2236 } 2237 2238 return ref_field; 2239 } 2240 2241 static bool is_var_ref(char *var_name) 2242 { 2243 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$') 2244 return false; 2245 2246 return true; 2247 } 2248 2249 static char *field_name_from_var(struct hist_trigger_data *hist_data, 2250 char *var_name) 2251 { 2252 char *name, *field; 2253 unsigned int i; 2254 2255 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 2256 name = hist_data->attrs->var_defs.name[i]; 2257 2258 if (strcmp(var_name, name) == 0) { 2259 field = hist_data->attrs->var_defs.expr[i]; 2260 if (contains_operator(field, NULL) || is_var_ref(field)) 2261 continue; 2262 return field; 2263 } 2264 } 2265 2266 return NULL; 2267 } 2268 2269 static char *local_field_var_ref(struct hist_trigger_data *hist_data, 2270 char *system, char *event_name, 2271 char *var_name) 2272 { 2273 struct trace_event_call *call; 2274 2275 if (system && event_name) { 2276 call = hist_data->event_file->event_call; 2277 2278 if (strcmp(system, call->class->system) != 0) 2279 return NULL; 2280 2281 if (strcmp(event_name, trace_event_name(call)) != 0) 2282 return NULL; 2283 } 2284 2285 if (!!system != !!event_name) 2286 return NULL; 2287 2288 if (!is_var_ref(var_name)) 2289 return NULL; 2290 2291 var_name++; 2292 2293 return field_name_from_var(hist_data, var_name); 2294 } 2295 2296 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data, 2297 char *system, char *event_name, 2298 char *var_name) 2299 { 2300 struct hist_field *var_field = NULL, *ref_field = NULL; 2301 struct trace_array *tr = hist_data->event_file->tr; 2302 2303 if (!is_var_ref(var_name)) 2304 return NULL; 2305 2306 var_name++; 2307 2308 var_field = find_event_var(hist_data, system, event_name, var_name); 2309 if (var_field) 2310 ref_field = create_var_ref(hist_data, var_field, 2311 system, event_name); 2312 2313 if (!ref_field) 2314 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name)); 2315 2316 return ref_field; 2317 } 2318 2319 static struct ftrace_event_field * 2320 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, 2321 char *field_str, unsigned long *flags, unsigned long *buckets) 2322 { 2323 struct ftrace_event_field *field = NULL; 2324 char *field_name, *modifier, *str; 2325 struct trace_array *tr = file->tr; 2326 2327 modifier = str = kstrdup(field_str, GFP_KERNEL); 2328 if (!modifier) 2329 return ERR_PTR(-ENOMEM); 2330 2331 field_name = strsep(&modifier, "."); 2332 if (modifier) { 2333 if (strcmp(modifier, "hex") == 0) 2334 *flags |= HIST_FIELD_FL_HEX; 2335 else if (strcmp(modifier, "sym") == 0) 2336 *flags |= HIST_FIELD_FL_SYM; 2337 /* 2338 * 'sym-offset' occurrences in the trigger string are modified 2339 * to 'symXoffset' to simplify arithmetic expression parsing. 2340 */ 2341 else if (strcmp(modifier, "symXoffset") == 0) 2342 *flags |= HIST_FIELD_FL_SYM_OFFSET; 2343 else if ((strcmp(modifier, "execname") == 0) && 2344 (strcmp(field_name, "common_pid") == 0)) 2345 *flags |= HIST_FIELD_FL_EXECNAME; 2346 else if (strcmp(modifier, "syscall") == 0) 2347 *flags |= HIST_FIELD_FL_SYSCALL; 2348 else if (strcmp(modifier, "stacktrace") == 0) 2349 *flags |= HIST_FIELD_FL_STACKTRACE; 2350 else if (strcmp(modifier, "log2") == 0) 2351 *flags |= HIST_FIELD_FL_LOG2; 2352 else if (strcmp(modifier, "usecs") == 0) 2353 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; 2354 else if (strncmp(modifier, "bucket", 6) == 0) { 2355 int ret; 2356 2357 modifier += 6; 2358 2359 if (*modifier == 's') 2360 modifier++; 2361 if (*modifier != '=') 2362 goto error; 2363 modifier++; 2364 ret = kstrtoul(modifier, 0, buckets); 2365 if (ret || !(*buckets)) 2366 goto error; 2367 *flags |= HIST_FIELD_FL_BUCKET; 2368 } else if (strncmp(modifier, "percent", 7) == 0) { 2369 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2370 goto error; 2371 *flags |= HIST_FIELD_FL_PERCENT; 2372 } else if (strncmp(modifier, "graph", 5) == 0) { 2373 if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY)) 2374 goto error; 2375 *flags |= HIST_FIELD_FL_GRAPH; 2376 } else { 2377 error: 2378 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier)); 2379 field = ERR_PTR(-EINVAL); 2380 goto out; 2381 } 2382 } 2383 2384 if (strcmp(field_name, "common_timestamp") == 0) { 2385 *flags |= HIST_FIELD_FL_TIMESTAMP; 2386 hist_data->enable_timestamps = true; 2387 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS) 2388 hist_data->attrs->ts_in_usecs = true; 2389 } else if (strcmp(field_name, "common_stacktrace") == 0) { 2390 *flags |= HIST_FIELD_FL_STACKTRACE; 2391 } else if (strcmp(field_name, "common_cpu") == 0) { 2392 *flags |= HIST_FIELD_FL_CPU; 2393 } else if (strcmp(field_name, "common_comm") == 0) { 2394 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2395 } else if (strcmp(field_name, "hitcount") == 0) 2396 *flags |= HIST_FIELD_FL_HITCOUNT; 2397 else { 2398 field = trace_find_event_field(file->event_call, field_name); 2399 if (!field || !field->size) { 2400 /* 2401 * For backward compatibility, if field_name 2402 * was "cpu" or "stacktrace", then we treat this 2403 * the same as common_cpu and common_stacktrace 2404 * respectively. This also works for "CPU", and 2405 * "STACKTRACE". 2406 */ 2407 if (field && field->filter_type == FILTER_CPU) { 2408 *flags |= HIST_FIELD_FL_CPU; 2409 } else if (field && field->filter_type == FILTER_STACKTRACE) { 2410 *flags |= HIST_FIELD_FL_STACKTRACE; 2411 } else if (field && field->filter_type == FILTER_COMM) { 2412 *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; 2413 } else { 2414 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, 2415 errpos(field_name)); 2416 field = ERR_PTR(-EINVAL); 2417 goto out; 2418 } 2419 } 2420 } 2421 out: 2422 kfree(str); 2423 2424 return field; 2425 } 2426 2427 static struct hist_field *create_alias(struct hist_trigger_data *hist_data, 2428 struct hist_field *var_ref, 2429 char *var_name) 2430 { 2431 struct hist_field *alias = NULL; 2432 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR; 2433 2434 alias = create_hist_field(hist_data, NULL, flags, var_name); 2435 if (!alias) 2436 return NULL; 2437 2438 alias->fn_num = var_ref->fn_num; 2439 alias->operands[0] = var_ref; 2440 2441 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { 2442 destroy_hist_field(alias, 0); 2443 return NULL; 2444 } 2445 2446 alias->var_ref_idx = var_ref->var_ref_idx; 2447 2448 return alias; 2449 } 2450 2451 static struct hist_field *parse_const(struct hist_trigger_data *hist_data, 2452 char *str, char *var_name, 2453 unsigned long *flags) 2454 { 2455 struct trace_array *tr = hist_data->event_file->tr; 2456 struct hist_field *field = NULL; 2457 u64 constant; 2458 2459 if (kstrtoull(str, 0, &constant)) { 2460 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str)); 2461 return NULL; 2462 } 2463 2464 *flags |= HIST_FIELD_FL_CONST; 2465 field = create_hist_field(hist_data, NULL, *flags, var_name); 2466 if (!field) 2467 return NULL; 2468 2469 field->constant = constant; 2470 2471 return field; 2472 } 2473 2474 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data, 2475 struct trace_event_file *file, char *str, 2476 unsigned long *flags, char *var_name) 2477 { 2478 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str; 2479 struct ftrace_event_field *field = NULL; 2480 struct hist_field *hist_field = NULL; 2481 unsigned long buckets = 0; 2482 int ret = 0; 2483 2484 if (isdigit(str[0])) { 2485 hist_field = parse_const(hist_data, str, var_name, flags); 2486 if (!hist_field) { 2487 ret = -EINVAL; 2488 goto out; 2489 } 2490 return hist_field; 2491 } 2492 2493 s = strchr(str, '.'); 2494 if (s) { 2495 s = strchr(++s, '.'); 2496 if (s) { 2497 ref_system = strsep(&str, "."); 2498 if (!str) { 2499 ret = -EINVAL; 2500 goto out; 2501 } 2502 ref_event = strsep(&str, "."); 2503 if (!str) { 2504 ret = -EINVAL; 2505 goto out; 2506 } 2507 ref_var = str; 2508 } 2509 } 2510 2511 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var); 2512 if (!s) { 2513 hist_field = parse_var_ref(hist_data, ref_system, 2514 ref_event, ref_var); 2515 if (hist_field) { 2516 if (var_name) { 2517 hist_field = create_alias(hist_data, hist_field, var_name); 2518 if (!hist_field) { 2519 ret = -ENOMEM; 2520 goto out; 2521 } 2522 } 2523 return hist_field; 2524 } 2525 } else 2526 str = s; 2527 2528 field = parse_field(hist_data, file, str, flags, &buckets); 2529 if (IS_ERR(field)) { 2530 ret = PTR_ERR(field); 2531 goto out; 2532 } 2533 2534 hist_field = create_hist_field(hist_data, field, *flags, var_name); 2535 if (!hist_field) { 2536 ret = -ENOMEM; 2537 goto out; 2538 } 2539 hist_field->buckets = buckets; 2540 2541 return hist_field; 2542 out: 2543 return ERR_PTR(ret); 2544 } 2545 2546 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2547 struct trace_event_file *file, 2548 char *str, unsigned long flags, 2549 char *var_name, unsigned int *n_subexprs); 2550 2551 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, 2552 struct trace_event_file *file, 2553 char *str, unsigned long flags, 2554 char *var_name, unsigned int *n_subexprs) 2555 { 2556 struct hist_field *operand1, *expr = NULL; 2557 unsigned long operand_flags; 2558 int ret = 0; 2559 char *s; 2560 2561 /* Unary minus operator, increment n_subexprs */ 2562 ++*n_subexprs; 2563 2564 /* we support only -(xxx) i.e. explicit parens required */ 2565 2566 if (*n_subexprs > 3) { 2567 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2568 ret = -EINVAL; 2569 goto free; 2570 } 2571 2572 str++; /* skip leading '-' */ 2573 2574 s = strchr(str, '('); 2575 if (s) 2576 str++; 2577 else { 2578 ret = -EINVAL; 2579 goto free; 2580 } 2581 2582 s = strrchr(str, ')'); 2583 if (s) { 2584 /* unary minus not supported in sub-expressions */ 2585 if (*(s+1) != '\0') { 2586 hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR, 2587 errpos(str)); 2588 ret = -EINVAL; 2589 goto free; 2590 } 2591 *s = '\0'; 2592 } 2593 else { 2594 ret = -EINVAL; /* no closing ')' */ 2595 goto free; 2596 } 2597 2598 flags |= HIST_FIELD_FL_EXPR; 2599 expr = create_hist_field(hist_data, NULL, flags, var_name); 2600 if (!expr) { 2601 ret = -ENOMEM; 2602 goto free; 2603 } 2604 2605 operand_flags = 0; 2606 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2607 if (IS_ERR(operand1)) { 2608 ret = PTR_ERR(operand1); 2609 goto free; 2610 } 2611 if (operand1->flags & HIST_FIELD_FL_STRING) { 2612 /* String type can not be the operand of unary operator. */ 2613 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2614 destroy_hist_field(operand1, 0); 2615 ret = -EINVAL; 2616 goto free; 2617 } 2618 2619 expr->flags |= operand1->flags & 2620 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2621 expr->fn_num = HIST_FIELD_FN_UMINUS; 2622 expr->operands[0] = operand1; 2623 expr->size = operand1->size; 2624 expr->is_signed = operand1->is_signed; 2625 expr->operator = FIELD_OP_UNARY_MINUS; 2626 expr->name = expr_str(expr, 0); 2627 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2628 if (!expr->type) { 2629 ret = -ENOMEM; 2630 goto free; 2631 } 2632 2633 return expr; 2634 free: 2635 destroy_hist_field(expr, 0); 2636 return ERR_PTR(ret); 2637 } 2638 2639 /* 2640 * If the operands are var refs, return pointers the 2641 * variable(s) referenced in var1 and var2, else NULL. 2642 */ 2643 static int check_expr_operands(struct trace_array *tr, 2644 struct hist_field *operand1, 2645 struct hist_field *operand2, 2646 struct hist_field **var1, 2647 struct hist_field **var2) 2648 { 2649 unsigned long operand1_flags = operand1->flags; 2650 unsigned long operand2_flags = operand2->flags; 2651 2652 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) || 2653 (operand1_flags & HIST_FIELD_FL_ALIAS)) { 2654 struct hist_field *var; 2655 2656 var = find_var_field(operand1->var.hist_data, operand1->name); 2657 if (!var) 2658 return -EINVAL; 2659 operand1_flags = var->flags; 2660 *var1 = var; 2661 } 2662 2663 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) || 2664 (operand2_flags & HIST_FIELD_FL_ALIAS)) { 2665 struct hist_field *var; 2666 2667 var = find_var_field(operand2->var.hist_data, operand2->name); 2668 if (!var) 2669 return -EINVAL; 2670 operand2_flags = var->flags; 2671 *var2 = var; 2672 } 2673 2674 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) != 2675 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) { 2676 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0); 2677 return -EINVAL; 2678 } 2679 2680 return 0; 2681 } 2682 2683 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, 2684 struct trace_event_file *file, 2685 char *str, unsigned long flags, 2686 char *var_name, unsigned int *n_subexprs) 2687 { 2688 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL; 2689 struct hist_field *var1 = NULL, *var2 = NULL; 2690 unsigned long operand_flags, operand2_flags; 2691 int field_op, ret = -EINVAL; 2692 char *sep, *operand1_str; 2693 enum hist_field_fn op_fn; 2694 bool combine_consts; 2695 2696 if (*n_subexprs > 3) { 2697 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str)); 2698 return ERR_PTR(-EINVAL); 2699 } 2700 2701 field_op = contains_operator(str, &sep); 2702 2703 if (field_op == FIELD_OP_NONE) 2704 return parse_atom(hist_data, file, str, &flags, var_name); 2705 2706 if (field_op == FIELD_OP_UNARY_MINUS) 2707 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs); 2708 2709 /* Binary operator found, increment n_subexprs */ 2710 ++*n_subexprs; 2711 2712 /* Split the expression string at the root operator */ 2713 if (!sep) 2714 return ERR_PTR(-EINVAL); 2715 2716 *sep = '\0'; 2717 operand1_str = str; 2718 str = sep+1; 2719 2720 /* Binary operator requires both operands */ 2721 if (*operand1_str == '\0' || *str == '\0') 2722 return ERR_PTR(-EINVAL); 2723 2724 operand_flags = 0; 2725 2726 /* LHS of string is an expression e.g. a+b in a+b+c */ 2727 operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs); 2728 if (IS_ERR(operand1)) 2729 return ERR_CAST(operand1); 2730 2731 if (operand1->flags & HIST_FIELD_FL_STRING) { 2732 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str)); 2733 ret = -EINVAL; 2734 goto free_op1; 2735 } 2736 2737 /* RHS of string is another expression e.g. c in a+b+c */ 2738 operand_flags = 0; 2739 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs); 2740 if (IS_ERR(operand2)) { 2741 ret = PTR_ERR(operand2); 2742 goto free_op1; 2743 } 2744 if (operand2->flags & HIST_FIELD_FL_STRING) { 2745 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str)); 2746 ret = -EINVAL; 2747 goto free_operands; 2748 } 2749 2750 switch (field_op) { 2751 case FIELD_OP_MINUS: 2752 op_fn = HIST_FIELD_FN_MINUS; 2753 break; 2754 case FIELD_OP_PLUS: 2755 op_fn = HIST_FIELD_FN_PLUS; 2756 break; 2757 case FIELD_OP_DIV: 2758 op_fn = HIST_FIELD_FN_DIV; 2759 break; 2760 case FIELD_OP_MULT: 2761 op_fn = HIST_FIELD_FN_MULT; 2762 break; 2763 default: 2764 ret = -EINVAL; 2765 goto free_operands; 2766 } 2767 2768 ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2); 2769 if (ret) 2770 goto free_operands; 2771 2772 operand_flags = var1 ? var1->flags : operand1->flags; 2773 operand2_flags = var2 ? var2->flags : operand2->flags; 2774 2775 /* 2776 * If both operands are constant, the expression can be 2777 * collapsed to a single constant. 2778 */ 2779 combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST; 2780 2781 flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR; 2782 2783 flags |= operand1->flags & 2784 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); 2785 2786 expr = create_hist_field(hist_data, NULL, flags, var_name); 2787 if (!expr) { 2788 ret = -ENOMEM; 2789 goto free_operands; 2790 } 2791 2792 operand1->read_once = true; 2793 operand2->read_once = true; 2794 2795 /* The operands are now owned and free'd by 'expr' */ 2796 expr->operands[0] = operand1; 2797 expr->operands[1] = operand2; 2798 2799 if (field_op == FIELD_OP_DIV && 2800 operand2_flags & HIST_FIELD_FL_CONST) { 2801 u64 divisor = var2 ? var2->constant : operand2->constant; 2802 2803 if (!divisor) { 2804 hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str)); 2805 ret = -EDOM; 2806 goto free_expr; 2807 } 2808 2809 /* 2810 * Copy the divisor here so we don't have to look it up 2811 * later if this is a var ref 2812 */ 2813 operand2->constant = divisor; 2814 op_fn = hist_field_get_div_fn(operand2); 2815 } 2816 2817 expr->fn_num = op_fn; 2818 2819 if (combine_consts) { 2820 if (var1) 2821 expr->operands[0] = var1; 2822 if (var2) 2823 expr->operands[1] = var2; 2824 2825 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL); 2826 expr->fn_num = HIST_FIELD_FN_CONST; 2827 2828 expr->operands[0] = NULL; 2829 expr->operands[1] = NULL; 2830 2831 /* 2832 * var refs won't be destroyed immediately 2833 * See: destroy_hist_field() 2834 */ 2835 destroy_hist_field(operand2, 0); 2836 destroy_hist_field(operand1, 0); 2837 2838 expr->name = expr_str(expr, 0); 2839 } else { 2840 /* The operand sizes should be the same, so just pick one */ 2841 expr->size = operand1->size; 2842 expr->is_signed = operand1->is_signed; 2843 2844 expr->operator = field_op; 2845 expr->type = kstrdup_const(operand1->type, GFP_KERNEL); 2846 if (!expr->type) { 2847 ret = -ENOMEM; 2848 goto free_expr; 2849 } 2850 2851 expr->name = expr_str(expr, 0); 2852 } 2853 2854 return expr; 2855 2856 free_operands: 2857 destroy_hist_field(operand2, 0); 2858 free_op1: 2859 destroy_hist_field(operand1, 0); 2860 return ERR_PTR(ret); 2861 2862 free_expr: 2863 destroy_hist_field(expr, 0); 2864 return ERR_PTR(ret); 2865 } 2866 2867 static char *find_trigger_filter(struct hist_trigger_data *hist_data, 2868 struct trace_event_file *file) 2869 { 2870 struct event_trigger_data *test; 2871 2872 lockdep_assert_held(&event_mutex); 2873 2874 list_for_each_entry(test, &file->triggers, list) { 2875 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2876 if (test->private_data == hist_data) 2877 return test->filter_str; 2878 } 2879 } 2880 2881 return NULL; 2882 } 2883 2884 static struct event_command trigger_hist_cmd; 2885 static int event_hist_trigger_parse(struct event_command *cmd_ops, 2886 struct trace_event_file *file, 2887 char *glob, char *cmd, 2888 char *param_and_filter); 2889 2890 static bool compatible_keys(struct hist_trigger_data *target_hist_data, 2891 struct hist_trigger_data *hist_data, 2892 unsigned int n_keys) 2893 { 2894 struct hist_field *target_hist_field, *hist_field; 2895 unsigned int n, i, j; 2896 2897 if (hist_data->n_fields - hist_data->n_vals != n_keys) 2898 return false; 2899 2900 i = hist_data->n_vals; 2901 j = target_hist_data->n_vals; 2902 2903 for (n = 0; n < n_keys; n++) { 2904 hist_field = hist_data->fields[i + n]; 2905 target_hist_field = target_hist_data->fields[j + n]; 2906 2907 if (strcmp(hist_field->type, target_hist_field->type) != 0) 2908 return false; 2909 if (hist_field->size != target_hist_field->size) 2910 return false; 2911 if (hist_field->is_signed != target_hist_field->is_signed) 2912 return false; 2913 } 2914 2915 return true; 2916 } 2917 2918 static struct hist_trigger_data * 2919 find_compatible_hist(struct hist_trigger_data *target_hist_data, 2920 struct trace_event_file *file) 2921 { 2922 struct hist_trigger_data *hist_data; 2923 struct event_trigger_data *test; 2924 unsigned int n_keys; 2925 2926 lockdep_assert_held(&event_mutex); 2927 2928 n_keys = target_hist_data->n_fields - target_hist_data->n_vals; 2929 2930 list_for_each_entry(test, &file->triggers, list) { 2931 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 2932 hist_data = test->private_data; 2933 2934 if (compatible_keys(target_hist_data, hist_data, n_keys)) 2935 return hist_data; 2936 } 2937 } 2938 2939 return NULL; 2940 } 2941 2942 static struct trace_event_file *event_file(struct trace_array *tr, 2943 char *system, char *event_name) 2944 { 2945 struct trace_event_file *file; 2946 2947 file = __find_event_file(tr, system, event_name); 2948 if (!file) 2949 return ERR_PTR(-EINVAL); 2950 2951 return file; 2952 } 2953 2954 static struct hist_field * 2955 find_synthetic_field_var(struct hist_trigger_data *target_hist_data, 2956 char *system, char *event_name, char *field_name) 2957 { 2958 struct hist_field *event_var; 2959 char *synthetic_name; 2960 2961 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 2962 if (!synthetic_name) 2963 return ERR_PTR(-ENOMEM); 2964 2965 strcpy(synthetic_name, "synthetic_"); 2966 strcat(synthetic_name, field_name); 2967 2968 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name); 2969 2970 kfree(synthetic_name); 2971 2972 return event_var; 2973 } 2974 2975 /** 2976 * create_field_var_hist - Automatically create a histogram and var for a field 2977 * @target_hist_data: The target hist trigger 2978 * @subsys_name: Optional subsystem name 2979 * @event_name: Optional event name 2980 * @field_name: The name of the field (and the resulting variable) 2981 * 2982 * Hist trigger actions fetch data from variables, not directly from 2983 * events. However, for convenience, users are allowed to directly 2984 * specify an event field in an action, which will be automatically 2985 * converted into a variable on their behalf. 2986 * 2987 * If a user specifies a field on an event that isn't the event the 2988 * histogram currently being defined (the target event histogram), the 2989 * only way that can be accomplished is if a new hist trigger is 2990 * created and the field variable defined on that. 2991 * 2992 * This function creates a new histogram compatible with the target 2993 * event (meaning a histogram with the same key as the target 2994 * histogram), and creates a variable for the specified field, but 2995 * with 'synthetic_' prepended to the variable name in order to avoid 2996 * collision with normal field variables. 2997 * 2998 * Return: The variable created for the field. 2999 */ 3000 static struct hist_field * 3001 create_field_var_hist(struct hist_trigger_data *target_hist_data, 3002 char *subsys_name, char *event_name, char *field_name) 3003 { 3004 struct trace_array *tr = target_hist_data->event_file->tr; 3005 struct hist_trigger_data *hist_data; 3006 unsigned int i, n, first = true; 3007 struct field_var_hist *var_hist; 3008 struct trace_event_file *file; 3009 struct hist_field *key_field; 3010 struct hist_field *event_var; 3011 char *saved_filter; 3012 char *cmd; 3013 int ret; 3014 3015 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) { 3016 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3017 return ERR_PTR(-EINVAL); 3018 } 3019 3020 file = event_file(tr, subsys_name, event_name); 3021 3022 if (IS_ERR(file)) { 3023 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name)); 3024 ret = PTR_ERR(file); 3025 return ERR_PTR(ret); 3026 } 3027 3028 /* 3029 * Look for a histogram compatible with target. We'll use the 3030 * found histogram specification to create a new matching 3031 * histogram with our variable on it. target_hist_data is not 3032 * yet a registered histogram so we can't use that. 3033 */ 3034 hist_data = find_compatible_hist(target_hist_data, file); 3035 if (!hist_data) { 3036 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name)); 3037 return ERR_PTR(-EINVAL); 3038 } 3039 3040 /* See if a synthetic field variable has already been created */ 3041 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3042 event_name, field_name); 3043 if (!IS_ERR_OR_NULL(event_var)) 3044 return event_var; 3045 3046 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL); 3047 if (!var_hist) 3048 return ERR_PTR(-ENOMEM); 3049 3050 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL); 3051 if (!cmd) { 3052 kfree(var_hist); 3053 return ERR_PTR(-ENOMEM); 3054 } 3055 3056 /* Use the same keys as the compatible histogram */ 3057 strcat(cmd, "keys="); 3058 3059 for_each_hist_key_field(i, hist_data) { 3060 key_field = hist_data->fields[i]; 3061 if (!first) 3062 strcat(cmd, ","); 3063 strcat(cmd, key_field->field->name); 3064 first = false; 3065 } 3066 3067 /* Create the synthetic field variable specification */ 3068 strcat(cmd, ":synthetic_"); 3069 strcat(cmd, field_name); 3070 strcat(cmd, "="); 3071 strcat(cmd, field_name); 3072 3073 /* Use the same filter as the compatible histogram */ 3074 saved_filter = find_trigger_filter(hist_data, file); 3075 if (saved_filter) { 3076 strcat(cmd, " if "); 3077 strcat(cmd, saved_filter); 3078 } 3079 3080 var_hist->cmd = kstrdup(cmd, GFP_KERNEL); 3081 if (!var_hist->cmd) { 3082 kfree(cmd); 3083 kfree(var_hist); 3084 return ERR_PTR(-ENOMEM); 3085 } 3086 3087 /* Save the compatible histogram information */ 3088 var_hist->hist_data = hist_data; 3089 3090 /* Create the new histogram with our variable */ 3091 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 3092 "", "hist", cmd); 3093 if (ret) { 3094 kfree(cmd); 3095 kfree(var_hist->cmd); 3096 kfree(var_hist); 3097 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name)); 3098 return ERR_PTR(ret); 3099 } 3100 3101 kfree(cmd); 3102 3103 /* If we can't find the variable, something went wrong */ 3104 event_var = find_synthetic_field_var(target_hist_data, subsys_name, 3105 event_name, field_name); 3106 if (IS_ERR_OR_NULL(event_var)) { 3107 kfree(var_hist->cmd); 3108 kfree(var_hist); 3109 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name)); 3110 return ERR_PTR(-EINVAL); 3111 } 3112 3113 n = target_hist_data->n_field_var_hists; 3114 target_hist_data->field_var_hists[n] = var_hist; 3115 target_hist_data->n_field_var_hists++; 3116 3117 return event_var; 3118 } 3119 3120 static struct hist_field * 3121 find_target_event_var(struct hist_trigger_data *hist_data, 3122 char *subsys_name, char *event_name, char *var_name) 3123 { 3124 struct trace_event_file *file = hist_data->event_file; 3125 struct hist_field *hist_field = NULL; 3126 3127 if (subsys_name) { 3128 struct trace_event_call *call; 3129 3130 if (!event_name) 3131 return NULL; 3132 3133 call = file->event_call; 3134 3135 if (strcmp(subsys_name, call->class->system) != 0) 3136 return NULL; 3137 3138 if (strcmp(event_name, trace_event_name(call)) != 0) 3139 return NULL; 3140 } 3141 3142 hist_field = find_var_field(hist_data, var_name); 3143 3144 return hist_field; 3145 } 3146 3147 static inline void __update_field_vars(struct tracing_map_elt *elt, 3148 struct trace_buffer *buffer, 3149 struct ring_buffer_event *rbe, 3150 void *rec, 3151 struct field_var **field_vars, 3152 unsigned int n_field_vars, 3153 unsigned int field_var_str_start) 3154 { 3155 struct hist_elt_data *elt_data = elt->private_data; 3156 unsigned int i, j, var_idx; 3157 u64 var_val; 3158 3159 /* Make sure stacktrace can fit in the string variable length */ 3160 BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX); 3161 3162 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) { 3163 struct field_var *field_var = field_vars[i]; 3164 struct hist_field *var = field_var->var; 3165 struct hist_field *val = field_var->val; 3166 3167 var_val = hist_fn_call(val, elt, buffer, rbe, rec); 3168 var_idx = var->var.idx; 3169 3170 if (val->flags & (HIST_FIELD_FL_STRING | 3171 HIST_FIELD_FL_STACKTRACE)) { 3172 char *str = elt_data->field_var_str[j++]; 3173 char *val_str = (char *)(uintptr_t)var_val; 3174 unsigned int size; 3175 3176 if (val->flags & HIST_FIELD_FL_STRING) { 3177 size = min(val->size, STR_VAR_LEN_MAX); 3178 strscpy(str, val_str, size); 3179 } else { 3180 char *stack_start = str + sizeof(unsigned long); 3181 int e; 3182 3183 e = stack_trace_save((void *)stack_start, 3184 HIST_STACKTRACE_DEPTH, 3185 HIST_STACKTRACE_SKIP); 3186 if (e < HIST_STACKTRACE_DEPTH - 1) 3187 ((unsigned long *)stack_start)[e] = 0; 3188 *((unsigned long *)str) = e; 3189 } 3190 var_val = (u64)(uintptr_t)str; 3191 } 3192 tracing_map_set_var(elt, var_idx, var_val); 3193 } 3194 } 3195 3196 static void update_field_vars(struct hist_trigger_data *hist_data, 3197 struct tracing_map_elt *elt, 3198 struct trace_buffer *buffer, 3199 struct ring_buffer_event *rbe, 3200 void *rec) 3201 { 3202 __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars, 3203 hist_data->n_field_vars, 0); 3204 } 3205 3206 static void save_track_data_vars(struct hist_trigger_data *hist_data, 3207 struct tracing_map_elt *elt, 3208 struct trace_buffer *buffer, void *rec, 3209 struct ring_buffer_event *rbe, void *key, 3210 struct action_data *data, u64 *var_ref_vals) 3211 { 3212 __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars, 3213 hist_data->n_save_vars, hist_data->n_field_var_str); 3214 } 3215 3216 static struct hist_field *create_var(struct hist_trigger_data *hist_data, 3217 struct trace_event_file *file, 3218 char *name, int size, const char *type) 3219 { 3220 struct hist_field *var; 3221 int idx; 3222 3223 if (find_var(hist_data, file, name) && !hist_data->remove) { 3224 var = ERR_PTR(-EINVAL); 3225 goto out; 3226 } 3227 3228 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL); 3229 if (!var) { 3230 var = ERR_PTR(-ENOMEM); 3231 goto out; 3232 } 3233 3234 idx = tracing_map_add_var(hist_data->map); 3235 if (idx < 0) { 3236 kfree(var); 3237 var = ERR_PTR(-EINVAL); 3238 goto out; 3239 } 3240 3241 var->ref = 1; 3242 var->flags = HIST_FIELD_FL_VAR; 3243 var->var.idx = idx; 3244 var->var.hist_data = var->hist_data = hist_data; 3245 var->size = size; 3246 var->var.name = kstrdup(name, GFP_KERNEL); 3247 var->type = kstrdup_const(type, GFP_KERNEL); 3248 if (!var->var.name || !var->type) { 3249 kfree_const(var->type); 3250 kfree(var->var.name); 3251 kfree(var); 3252 var = ERR_PTR(-ENOMEM); 3253 } 3254 out: 3255 return var; 3256 } 3257 3258 static struct field_var *create_field_var(struct hist_trigger_data *hist_data, 3259 struct trace_event_file *file, 3260 char *field_name) 3261 { 3262 struct hist_field *val = NULL, *var = NULL; 3263 unsigned long flags = HIST_FIELD_FL_VAR; 3264 struct trace_array *tr = file->tr; 3265 struct field_var *field_var; 3266 int ret = 0; 3267 3268 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) { 3269 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name)); 3270 ret = -EINVAL; 3271 goto err; 3272 } 3273 3274 val = parse_atom(hist_data, file, field_name, &flags, NULL); 3275 if (IS_ERR(val)) { 3276 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name)); 3277 ret = PTR_ERR(val); 3278 goto err; 3279 } 3280 3281 var = create_var(hist_data, file, field_name, val->size, val->type); 3282 if (IS_ERR(var)) { 3283 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name)); 3284 destroy_hist_field(val, 0); 3285 ret = PTR_ERR(var); 3286 goto err; 3287 } 3288 3289 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL); 3290 if (!field_var) { 3291 destroy_hist_field(val, 0); 3292 kfree_const(var->type); 3293 kfree(var->var.name); 3294 kfree(var); 3295 ret = -ENOMEM; 3296 goto err; 3297 } 3298 3299 field_var->var = var; 3300 field_var->val = val; 3301 out: 3302 return field_var; 3303 err: 3304 field_var = ERR_PTR(ret); 3305 goto out; 3306 } 3307 3308 /** 3309 * create_target_field_var - Automatically create a variable for a field 3310 * @target_hist_data: The target hist trigger 3311 * @subsys_name: Optional subsystem name 3312 * @event_name: Optional event name 3313 * @var_name: The name of the field (and the resulting variable) 3314 * 3315 * Hist trigger actions fetch data from variables, not directly from 3316 * events. However, for convenience, users are allowed to directly 3317 * specify an event field in an action, which will be automatically 3318 * converted into a variable on their behalf. 3319 * 3320 * This function creates a field variable with the name var_name on 3321 * the hist trigger currently being defined on the target event. If 3322 * subsys_name and event_name are specified, this function simply 3323 * verifies that they do in fact match the target event subsystem and 3324 * event name. 3325 * 3326 * Return: The variable created for the field. 3327 */ 3328 static struct field_var * 3329 create_target_field_var(struct hist_trigger_data *target_hist_data, 3330 char *subsys_name, char *event_name, char *var_name) 3331 { 3332 struct trace_event_file *file = target_hist_data->event_file; 3333 3334 if (subsys_name) { 3335 struct trace_event_call *call; 3336 3337 if (!event_name) 3338 return NULL; 3339 3340 call = file->event_call; 3341 3342 if (strcmp(subsys_name, call->class->system) != 0) 3343 return NULL; 3344 3345 if (strcmp(event_name, trace_event_name(call)) != 0) 3346 return NULL; 3347 } 3348 3349 return create_field_var(target_hist_data, file, var_name); 3350 } 3351 3352 static bool check_track_val_max(u64 track_val, u64 var_val) 3353 { 3354 if (var_val <= track_val) 3355 return false; 3356 3357 return true; 3358 } 3359 3360 static bool check_track_val_changed(u64 track_val, u64 var_val) 3361 { 3362 if (var_val == track_val) 3363 return false; 3364 3365 return true; 3366 } 3367 3368 static u64 get_track_val(struct hist_trigger_data *hist_data, 3369 struct tracing_map_elt *elt, 3370 struct action_data *data) 3371 { 3372 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3373 u64 track_val; 3374 3375 track_val = tracing_map_read_var(elt, track_var_idx); 3376 3377 return track_val; 3378 } 3379 3380 static void save_track_val(struct hist_trigger_data *hist_data, 3381 struct tracing_map_elt *elt, 3382 struct action_data *data, u64 var_val) 3383 { 3384 unsigned int track_var_idx = data->track_data.track_var->var.idx; 3385 3386 tracing_map_set_var(elt, track_var_idx, var_val); 3387 } 3388 3389 static void save_track_data(struct hist_trigger_data *hist_data, 3390 struct tracing_map_elt *elt, 3391 struct trace_buffer *buffer, void *rec, 3392 struct ring_buffer_event *rbe, void *key, 3393 struct action_data *data, u64 *var_ref_vals) 3394 { 3395 if (data->track_data.save_data) 3396 data->track_data.save_data(hist_data, elt, buffer, rec, rbe, 3397 key, data, var_ref_vals); 3398 } 3399 3400 static bool check_track_val(struct tracing_map_elt *elt, 3401 struct action_data *data, 3402 u64 var_val) 3403 { 3404 struct hist_trigger_data *hist_data; 3405 u64 track_val; 3406 3407 hist_data = data->track_data.track_var->hist_data; 3408 track_val = get_track_val(hist_data, elt, data); 3409 3410 return data->track_data.check_val(track_val, var_val); 3411 } 3412 3413 #ifdef CONFIG_TRACER_SNAPSHOT 3414 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3415 { 3416 /* called with tr->max_lock held */ 3417 struct track_data *track_data = tr->cond_snapshot->cond_data; 3418 struct hist_elt_data *elt_data, *track_elt_data; 3419 struct snapshot_context *context = cond_data; 3420 struct action_data *action; 3421 u64 track_val; 3422 3423 if (!track_data) 3424 return false; 3425 3426 action = track_data->action_data; 3427 3428 track_val = get_track_val(track_data->hist_data, context->elt, 3429 track_data->action_data); 3430 3431 if (!action->track_data.check_val(track_data->track_val, track_val)) 3432 return false; 3433 3434 track_data->track_val = track_val; 3435 memcpy(track_data->key, context->key, track_data->key_len); 3436 3437 elt_data = context->elt->private_data; 3438 track_elt_data = track_data->elt.private_data; 3439 if (elt_data->comm) 3440 strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN); 3441 3442 track_data->updated = true; 3443 3444 return true; 3445 } 3446 3447 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3448 struct tracing_map_elt *elt, 3449 struct trace_buffer *buffer, void *rec, 3450 struct ring_buffer_event *rbe, void *key, 3451 struct action_data *data, 3452 u64 *var_ref_vals) 3453 { 3454 struct trace_event_file *file = hist_data->event_file; 3455 struct snapshot_context context; 3456 3457 context.elt = elt; 3458 context.key = key; 3459 3460 tracing_snapshot_cond(file->tr, &context); 3461 } 3462 3463 static void hist_trigger_print_key(struct seq_file *m, 3464 struct hist_trigger_data *hist_data, 3465 void *key, 3466 struct tracing_map_elt *elt); 3467 3468 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data) 3469 { 3470 unsigned int i; 3471 3472 if (!hist_data->n_actions) 3473 return NULL; 3474 3475 for (i = 0; i < hist_data->n_actions; i++) { 3476 struct action_data *data = hist_data->actions[i]; 3477 3478 if (data->action == ACTION_SNAPSHOT) 3479 return data; 3480 } 3481 3482 return NULL; 3483 } 3484 3485 static void track_data_snapshot_print(struct seq_file *m, 3486 struct hist_trigger_data *hist_data) 3487 { 3488 struct trace_event_file *file = hist_data->event_file; 3489 struct track_data *track_data; 3490 struct action_data *action; 3491 3492 track_data = tracing_cond_snapshot_data(file->tr); 3493 if (!track_data) 3494 return; 3495 3496 if (!track_data->updated) 3497 return; 3498 3499 action = snapshot_action(hist_data); 3500 if (!action) 3501 return; 3502 3503 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n"); 3504 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu", 3505 action->handler == HANDLER_ONMAX ? "onmax" : "onchange", 3506 action->track_data.var_str, track_data->track_val); 3507 3508 seq_puts(m, "\ttriggered by event with key: "); 3509 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt); 3510 seq_putc(m, '\n'); 3511 } 3512 #else 3513 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data) 3514 { 3515 return false; 3516 } 3517 static void save_track_data_snapshot(struct hist_trigger_data *hist_data, 3518 struct tracing_map_elt *elt, 3519 struct trace_buffer *buffer, void *rec, 3520 struct ring_buffer_event *rbe, void *key, 3521 struct action_data *data, 3522 u64 *var_ref_vals) {} 3523 static void track_data_snapshot_print(struct seq_file *m, 3524 struct hist_trigger_data *hist_data) {} 3525 #endif /* CONFIG_TRACER_SNAPSHOT */ 3526 3527 static void track_data_print(struct seq_file *m, 3528 struct hist_trigger_data *hist_data, 3529 struct tracing_map_elt *elt, 3530 struct action_data *data) 3531 { 3532 u64 track_val = get_track_val(hist_data, elt, data); 3533 unsigned int i, save_var_idx; 3534 3535 if (data->handler == HANDLER_ONMAX) 3536 seq_printf(m, "\n\tmax: %10llu", track_val); 3537 else if (data->handler == HANDLER_ONCHANGE) 3538 seq_printf(m, "\n\tchanged: %10llu", track_val); 3539 3540 if (data->action == ACTION_SNAPSHOT) 3541 return; 3542 3543 for (i = 0; i < hist_data->n_save_vars; i++) { 3544 struct hist_field *save_val = hist_data->save_vars[i]->val; 3545 struct hist_field *save_var = hist_data->save_vars[i]->var; 3546 u64 val; 3547 3548 save_var_idx = save_var->var.idx; 3549 3550 val = tracing_map_read_var(elt, save_var_idx); 3551 3552 if (save_val->flags & HIST_FIELD_FL_STRING) { 3553 seq_printf(m, " %s: %-32s", save_var->var.name, 3554 (char *)(uintptr_t)(val)); 3555 } else 3556 seq_printf(m, " %s: %10llu", save_var->var.name, val); 3557 } 3558 } 3559 3560 static void ontrack_action(struct hist_trigger_data *hist_data, 3561 struct tracing_map_elt *elt, 3562 struct trace_buffer *buffer, void *rec, 3563 struct ring_buffer_event *rbe, void *key, 3564 struct action_data *data, u64 *var_ref_vals) 3565 { 3566 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx]; 3567 3568 if (check_track_val(elt, data, var_val)) { 3569 save_track_val(hist_data, elt, data, var_val); 3570 save_track_data(hist_data, elt, buffer, rec, rbe, 3571 key, data, var_ref_vals); 3572 } 3573 } 3574 3575 static void action_data_destroy(struct action_data *data) 3576 { 3577 unsigned int i; 3578 3579 lockdep_assert_held(&event_mutex); 3580 3581 kfree(data->action_name); 3582 3583 for (i = 0; i < data->n_params; i++) 3584 kfree(data->params[i]); 3585 3586 if (data->synth_event) 3587 data->synth_event->ref--; 3588 3589 kfree(data->synth_event_name); 3590 3591 kfree(data); 3592 } 3593 3594 static void track_data_destroy(struct hist_trigger_data *hist_data, 3595 struct action_data *data) 3596 { 3597 struct trace_event_file *file = hist_data->event_file; 3598 3599 destroy_hist_field(data->track_data.track_var, 0); 3600 3601 if (data->action == ACTION_SNAPSHOT) { 3602 struct track_data *track_data; 3603 3604 track_data = tracing_cond_snapshot_data(file->tr); 3605 if (track_data && track_data->hist_data == hist_data) { 3606 tracing_snapshot_cond_disable(file->tr); 3607 track_data_free(track_data); 3608 } 3609 } 3610 3611 kfree(data->track_data.var_str); 3612 3613 action_data_destroy(data); 3614 } 3615 3616 static int action_create(struct hist_trigger_data *hist_data, 3617 struct action_data *data); 3618 3619 static int track_data_create(struct hist_trigger_data *hist_data, 3620 struct action_data *data) 3621 { 3622 struct hist_field *var_field, *ref_field, *track_var = NULL; 3623 struct trace_event_file *file = hist_data->event_file; 3624 struct trace_array *tr = file->tr; 3625 char *track_data_var_str; 3626 int ret = 0; 3627 3628 track_data_var_str = data->track_data.var_str; 3629 if (track_data_var_str[0] != '$') { 3630 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str)); 3631 return -EINVAL; 3632 } 3633 track_data_var_str++; 3634 3635 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str); 3636 if (!var_field) { 3637 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str)); 3638 return -EINVAL; 3639 } 3640 3641 ref_field = create_var_ref(hist_data, var_field, NULL, NULL); 3642 if (!ref_field) 3643 return -ENOMEM; 3644 3645 data->track_data.var_ref = ref_field; 3646 3647 if (data->handler == HANDLER_ONMAX) 3648 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64"); 3649 if (IS_ERR(track_var)) { 3650 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3651 ret = PTR_ERR(track_var); 3652 goto out; 3653 } 3654 3655 if (data->handler == HANDLER_ONCHANGE) 3656 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64"); 3657 if (IS_ERR(track_var)) { 3658 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0); 3659 ret = PTR_ERR(track_var); 3660 goto out; 3661 } 3662 data->track_data.track_var = track_var; 3663 3664 ret = action_create(hist_data, data); 3665 out: 3666 return ret; 3667 } 3668 3669 static int parse_action_params(struct trace_array *tr, char *params, 3670 struct action_data *data) 3671 { 3672 char *param, *saved_param; 3673 bool first_param = true; 3674 int ret = 0; 3675 3676 while (params) { 3677 if (data->n_params >= SYNTH_FIELDS_MAX) { 3678 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0); 3679 ret = -EINVAL; 3680 goto out; 3681 } 3682 3683 param = strsep(¶ms, ","); 3684 if (!param) { 3685 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0); 3686 ret = -EINVAL; 3687 goto out; 3688 } 3689 3690 param = strstrip(param); 3691 if (strlen(param) < 2) { 3692 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param)); 3693 ret = -EINVAL; 3694 goto out; 3695 } 3696 3697 saved_param = kstrdup(param, GFP_KERNEL); 3698 if (!saved_param) { 3699 ret = -ENOMEM; 3700 goto out; 3701 } 3702 3703 if (first_param && data->use_trace_keyword) { 3704 data->synth_event_name = saved_param; 3705 first_param = false; 3706 continue; 3707 } 3708 first_param = false; 3709 3710 data->params[data->n_params++] = saved_param; 3711 } 3712 out: 3713 return ret; 3714 } 3715 3716 static int action_parse(struct trace_array *tr, char *str, struct action_data *data, 3717 enum handler_id handler) 3718 { 3719 char *action_name; 3720 int ret = 0; 3721 3722 strsep(&str, "."); 3723 if (!str) { 3724 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3725 ret = -EINVAL; 3726 goto out; 3727 } 3728 3729 action_name = strsep(&str, "("); 3730 if (!action_name || !str) { 3731 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0); 3732 ret = -EINVAL; 3733 goto out; 3734 } 3735 3736 if (str_has_prefix(action_name, "save")) { 3737 char *params = strsep(&str, ")"); 3738 3739 if (!params) { 3740 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0); 3741 ret = -EINVAL; 3742 goto out; 3743 } 3744 3745 ret = parse_action_params(tr, params, data); 3746 if (ret) 3747 goto out; 3748 3749 if (handler == HANDLER_ONMAX) 3750 data->track_data.check_val = check_track_val_max; 3751 else if (handler == HANDLER_ONCHANGE) 3752 data->track_data.check_val = check_track_val_changed; 3753 else { 3754 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3755 ret = -EINVAL; 3756 goto out; 3757 } 3758 3759 data->track_data.save_data = save_track_data_vars; 3760 data->fn = ontrack_action; 3761 data->action = ACTION_SAVE; 3762 } else if (str_has_prefix(action_name, "snapshot")) { 3763 char *params = strsep(&str, ")"); 3764 3765 if (!str) { 3766 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params)); 3767 ret = -EINVAL; 3768 goto out; 3769 } 3770 3771 if (handler == HANDLER_ONMAX) 3772 data->track_data.check_val = check_track_val_max; 3773 else if (handler == HANDLER_ONCHANGE) 3774 data->track_data.check_val = check_track_val_changed; 3775 else { 3776 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name)); 3777 ret = -EINVAL; 3778 goto out; 3779 } 3780 3781 data->track_data.save_data = save_track_data_snapshot; 3782 data->fn = ontrack_action; 3783 data->action = ACTION_SNAPSHOT; 3784 } else { 3785 char *params = strsep(&str, ")"); 3786 3787 if (str_has_prefix(action_name, "trace")) 3788 data->use_trace_keyword = true; 3789 3790 if (params) { 3791 ret = parse_action_params(tr, params, data); 3792 if (ret) 3793 goto out; 3794 } 3795 3796 if (handler == HANDLER_ONMAX) 3797 data->track_data.check_val = check_track_val_max; 3798 else if (handler == HANDLER_ONCHANGE) 3799 data->track_data.check_val = check_track_val_changed; 3800 3801 if (handler != HANDLER_ONMATCH) { 3802 data->track_data.save_data = action_trace; 3803 data->fn = ontrack_action; 3804 } else 3805 data->fn = action_trace; 3806 3807 data->action = ACTION_TRACE; 3808 } 3809 3810 data->action_name = kstrdup(action_name, GFP_KERNEL); 3811 if (!data->action_name) { 3812 ret = -ENOMEM; 3813 goto out; 3814 } 3815 3816 data->handler = handler; 3817 out: 3818 return ret; 3819 } 3820 3821 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data, 3822 char *str, enum handler_id handler) 3823 { 3824 struct action_data *data; 3825 int ret = -EINVAL; 3826 char *var_str; 3827 3828 data = kzalloc(sizeof(*data), GFP_KERNEL); 3829 if (!data) 3830 return ERR_PTR(-ENOMEM); 3831 3832 var_str = strsep(&str, ")"); 3833 if (!var_str || !str) { 3834 ret = -EINVAL; 3835 goto free; 3836 } 3837 3838 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL); 3839 if (!data->track_data.var_str) { 3840 ret = -ENOMEM; 3841 goto free; 3842 } 3843 3844 ret = action_parse(hist_data->event_file->tr, str, data, handler); 3845 if (ret) 3846 goto free; 3847 out: 3848 return data; 3849 free: 3850 track_data_destroy(hist_data, data); 3851 data = ERR_PTR(ret); 3852 goto out; 3853 } 3854 3855 static void onmatch_destroy(struct action_data *data) 3856 { 3857 kfree(data->match_data.event); 3858 kfree(data->match_data.event_system); 3859 3860 action_data_destroy(data); 3861 } 3862 3863 static void destroy_field_var(struct field_var *field_var) 3864 { 3865 if (!field_var) 3866 return; 3867 3868 destroy_hist_field(field_var->var, 0); 3869 destroy_hist_field(field_var->val, 0); 3870 3871 kfree(field_var); 3872 } 3873 3874 static void destroy_field_vars(struct hist_trigger_data *hist_data) 3875 { 3876 unsigned int i; 3877 3878 for (i = 0; i < hist_data->n_field_vars; i++) 3879 destroy_field_var(hist_data->field_vars[i]); 3880 3881 for (i = 0; i < hist_data->n_save_vars; i++) 3882 destroy_field_var(hist_data->save_vars[i]); 3883 } 3884 3885 static void save_field_var(struct hist_trigger_data *hist_data, 3886 struct field_var *field_var) 3887 { 3888 hist_data->field_vars[hist_data->n_field_vars++] = field_var; 3889 3890 /* Stack traces are saved in the string storage too */ 3891 if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 3892 hist_data->n_field_var_str++; 3893 } 3894 3895 3896 static int check_synth_field(struct synth_event *event, 3897 struct hist_field *hist_field, 3898 unsigned int field_pos) 3899 { 3900 struct synth_field *field; 3901 3902 if (field_pos >= event->n_fields) 3903 return -EINVAL; 3904 3905 field = event->fields[field_pos]; 3906 3907 /* 3908 * A dynamic string synth field can accept static or 3909 * dynamic. A static string synth field can only accept a 3910 * same-sized static string, which is checked for later. 3911 */ 3912 if (strstr(hist_field->type, "char[") && field->is_string 3913 && field->is_dynamic) 3914 return 0; 3915 3916 if (strstr(hist_field->type, "long[") && field->is_stack) 3917 return 0; 3918 3919 if (strcmp(field->type, hist_field->type) != 0) { 3920 if (field->size != hist_field->size || 3921 (!field->is_string && field->is_signed != hist_field->is_signed)) 3922 return -EINVAL; 3923 } 3924 3925 return 0; 3926 } 3927 3928 static struct hist_field * 3929 trace_action_find_var(struct hist_trigger_data *hist_data, 3930 struct action_data *data, 3931 char *system, char *event, char *var) 3932 { 3933 struct trace_array *tr = hist_data->event_file->tr; 3934 struct hist_field *hist_field; 3935 3936 var++; /* skip '$' */ 3937 3938 hist_field = find_target_event_var(hist_data, system, event, var); 3939 if (!hist_field) { 3940 if (!system && data->handler == HANDLER_ONMATCH) { 3941 system = data->match_data.event_system; 3942 event = data->match_data.event; 3943 } 3944 3945 hist_field = find_event_var(hist_data, system, event, var); 3946 } 3947 3948 if (!hist_field) 3949 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var)); 3950 3951 return hist_field; 3952 } 3953 3954 static struct hist_field * 3955 trace_action_create_field_var(struct hist_trigger_data *hist_data, 3956 struct action_data *data, char *system, 3957 char *event, char *var) 3958 { 3959 struct hist_field *hist_field = NULL; 3960 struct field_var *field_var; 3961 3962 /* 3963 * First try to create a field var on the target event (the 3964 * currently being defined). This will create a variable for 3965 * unqualified fields on the target event, or if qualified, 3966 * target fields that have qualified names matching the target. 3967 */ 3968 field_var = create_target_field_var(hist_data, system, event, var); 3969 3970 if (field_var && !IS_ERR(field_var)) { 3971 save_field_var(hist_data, field_var); 3972 hist_field = field_var->var; 3973 } else { 3974 field_var = NULL; 3975 /* 3976 * If no explicit system.event is specified, default to 3977 * looking for fields on the onmatch(system.event.xxx) 3978 * event. 3979 */ 3980 if (!system && data->handler == HANDLER_ONMATCH) { 3981 system = data->match_data.event_system; 3982 event = data->match_data.event; 3983 } 3984 3985 if (!event) 3986 goto free; 3987 /* 3988 * At this point, we're looking at a field on another 3989 * event. Because we can't modify a hist trigger on 3990 * another event to add a variable for a field, we need 3991 * to create a new trigger on that event and create the 3992 * variable at the same time. 3993 */ 3994 hist_field = create_field_var_hist(hist_data, system, event, var); 3995 if (IS_ERR(hist_field)) 3996 goto free; 3997 } 3998 out: 3999 return hist_field; 4000 free: 4001 destroy_field_var(field_var); 4002 hist_field = NULL; 4003 goto out; 4004 } 4005 4006 static int trace_action_create(struct hist_trigger_data *hist_data, 4007 struct action_data *data) 4008 { 4009 struct trace_array *tr = hist_data->event_file->tr; 4010 char *event_name, *param, *system = NULL; 4011 struct hist_field *hist_field, *var_ref; 4012 unsigned int i; 4013 unsigned int field_pos = 0; 4014 struct synth_event *event; 4015 char *synth_event_name; 4016 int var_ref_idx, ret = 0; 4017 4018 lockdep_assert_held(&event_mutex); 4019 4020 /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */ 4021 if (data->n_params > SYNTH_FIELDS_MAX) 4022 return -EINVAL; 4023 4024 if (data->use_trace_keyword) 4025 synth_event_name = data->synth_event_name; 4026 else 4027 synth_event_name = data->action_name; 4028 4029 event = find_synth_event(synth_event_name); 4030 if (!event) { 4031 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name)); 4032 return -EINVAL; 4033 } 4034 4035 event->ref++; 4036 4037 for (i = 0; i < data->n_params; i++) { 4038 char *p; 4039 4040 p = param = kstrdup(data->params[i], GFP_KERNEL); 4041 if (!param) { 4042 ret = -ENOMEM; 4043 goto err; 4044 } 4045 4046 system = strsep(¶m, "."); 4047 if (!param) { 4048 param = (char *)system; 4049 system = event_name = NULL; 4050 } else { 4051 event_name = strsep(¶m, "."); 4052 if (!param) { 4053 kfree(p); 4054 ret = -EINVAL; 4055 goto err; 4056 } 4057 } 4058 4059 if (param[0] == '$') 4060 hist_field = trace_action_find_var(hist_data, data, 4061 system, event_name, 4062 param); 4063 else 4064 hist_field = trace_action_create_field_var(hist_data, 4065 data, 4066 system, 4067 event_name, 4068 param); 4069 4070 if (!hist_field) { 4071 kfree(p); 4072 ret = -EINVAL; 4073 goto err; 4074 } 4075 4076 if (check_synth_field(event, hist_field, field_pos) == 0) { 4077 var_ref = create_var_ref(hist_data, hist_field, 4078 system, event_name); 4079 if (!var_ref) { 4080 kfree(p); 4081 ret = -ENOMEM; 4082 goto err; 4083 } 4084 4085 var_ref_idx = find_var_ref_idx(hist_data, var_ref); 4086 if (WARN_ON(var_ref_idx < 0)) { 4087 kfree(p); 4088 ret = var_ref_idx; 4089 goto err; 4090 } 4091 4092 data->var_ref_idx[i] = var_ref_idx; 4093 4094 field_pos++; 4095 kfree(p); 4096 continue; 4097 } 4098 4099 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param)); 4100 kfree(p); 4101 ret = -EINVAL; 4102 goto err; 4103 } 4104 4105 if (field_pos != event->n_fields) { 4106 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name)); 4107 ret = -EINVAL; 4108 goto err; 4109 } 4110 4111 data->synth_event = event; 4112 out: 4113 return ret; 4114 err: 4115 event->ref--; 4116 4117 goto out; 4118 } 4119 4120 static int action_create(struct hist_trigger_data *hist_data, 4121 struct action_data *data) 4122 { 4123 struct trace_event_file *file = hist_data->event_file; 4124 struct trace_array *tr = file->tr; 4125 struct track_data *track_data; 4126 struct field_var *field_var; 4127 unsigned int i; 4128 char *param; 4129 int ret = 0; 4130 4131 if (data->action == ACTION_TRACE) 4132 return trace_action_create(hist_data, data); 4133 4134 if (data->action == ACTION_SNAPSHOT) { 4135 track_data = track_data_alloc(hist_data->key_size, data, hist_data); 4136 if (IS_ERR(track_data)) { 4137 ret = PTR_ERR(track_data); 4138 goto out; 4139 } 4140 4141 ret = tracing_snapshot_cond_enable(file->tr, track_data, 4142 cond_snapshot_update); 4143 if (ret) 4144 track_data_free(track_data); 4145 4146 goto out; 4147 } 4148 4149 if (data->action == ACTION_SAVE) { 4150 if (hist_data->n_save_vars) { 4151 ret = -EEXIST; 4152 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0); 4153 goto out; 4154 } 4155 4156 for (i = 0; i < data->n_params; i++) { 4157 param = kstrdup(data->params[i], GFP_KERNEL); 4158 if (!param) { 4159 ret = -ENOMEM; 4160 goto out; 4161 } 4162 4163 field_var = create_target_field_var(hist_data, NULL, NULL, param); 4164 if (IS_ERR(field_var)) { 4165 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL, 4166 errpos(param)); 4167 ret = PTR_ERR(field_var); 4168 kfree(param); 4169 goto out; 4170 } 4171 4172 hist_data->save_vars[hist_data->n_save_vars++] = field_var; 4173 if (field_var->val->flags & 4174 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4175 hist_data->n_save_var_str++; 4176 kfree(param); 4177 } 4178 } 4179 out: 4180 return ret; 4181 } 4182 4183 static int onmatch_create(struct hist_trigger_data *hist_data, 4184 struct action_data *data) 4185 { 4186 return action_create(hist_data, data); 4187 } 4188 4189 static struct action_data *onmatch_parse(struct trace_array *tr, char *str) 4190 { 4191 char *match_event, *match_event_system; 4192 struct action_data *data; 4193 int ret = -EINVAL; 4194 4195 data = kzalloc(sizeof(*data), GFP_KERNEL); 4196 if (!data) 4197 return ERR_PTR(-ENOMEM); 4198 4199 match_event = strsep(&str, ")"); 4200 if (!match_event || !str) { 4201 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event)); 4202 goto free; 4203 } 4204 4205 match_event_system = strsep(&match_event, "."); 4206 if (!match_event) { 4207 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system)); 4208 goto free; 4209 } 4210 4211 if (IS_ERR(event_file(tr, match_event_system, match_event))) { 4212 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event)); 4213 goto free; 4214 } 4215 4216 data->match_data.event = kstrdup(match_event, GFP_KERNEL); 4217 if (!data->match_data.event) { 4218 ret = -ENOMEM; 4219 goto free; 4220 } 4221 4222 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL); 4223 if (!data->match_data.event_system) { 4224 ret = -ENOMEM; 4225 goto free; 4226 } 4227 4228 ret = action_parse(tr, str, data, HANDLER_ONMATCH); 4229 if (ret) 4230 goto free; 4231 out: 4232 return data; 4233 free: 4234 onmatch_destroy(data); 4235 data = ERR_PTR(ret); 4236 goto out; 4237 } 4238 4239 static int create_hitcount_val(struct hist_trigger_data *hist_data) 4240 { 4241 hist_data->fields[HITCOUNT_IDX] = 4242 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL); 4243 if (!hist_data->fields[HITCOUNT_IDX]) 4244 return -ENOMEM; 4245 4246 hist_data->n_vals++; 4247 hist_data->n_fields++; 4248 4249 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX)) 4250 return -EINVAL; 4251 4252 return 0; 4253 } 4254 4255 static int __create_val_field(struct hist_trigger_data *hist_data, 4256 unsigned int val_idx, 4257 struct trace_event_file *file, 4258 char *var_name, char *field_str, 4259 unsigned long flags) 4260 { 4261 struct hist_field *hist_field; 4262 int ret = 0, n_subexprs = 0; 4263 4264 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs); 4265 if (IS_ERR(hist_field)) { 4266 ret = PTR_ERR(hist_field); 4267 goto out; 4268 } 4269 4270 /* values and variables should not have some modifiers */ 4271 if (hist_field->flags & HIST_FIELD_FL_VAR) { 4272 /* Variable */ 4273 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4274 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2)) 4275 goto err; 4276 } else { 4277 /* Value */ 4278 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | 4279 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | 4280 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | 4281 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) 4282 goto err; 4283 } 4284 4285 hist_data->fields[val_idx] = hist_field; 4286 4287 ++hist_data->n_vals; 4288 ++hist_data->n_fields; 4289 4290 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4291 ret = -EINVAL; 4292 out: 4293 return ret; 4294 err: 4295 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); 4296 return -EINVAL; 4297 } 4298 4299 static int create_val_field(struct hist_trigger_data *hist_data, 4300 unsigned int val_idx, 4301 struct trace_event_file *file, 4302 char *field_str) 4303 { 4304 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX)) 4305 return -EINVAL; 4306 4307 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0); 4308 } 4309 4310 static const char no_comm[] = "(no comm)"; 4311 4312 static u64 hist_field_execname(struct hist_field *hist_field, 4313 struct tracing_map_elt *elt, 4314 struct trace_buffer *buffer, 4315 struct ring_buffer_event *rbe, 4316 void *event) 4317 { 4318 struct hist_elt_data *elt_data; 4319 4320 if (WARN_ON_ONCE(!elt)) 4321 return (u64)(unsigned long)no_comm; 4322 4323 elt_data = elt->private_data; 4324 4325 if (WARN_ON_ONCE(!elt_data->comm)) 4326 return (u64)(unsigned long)no_comm; 4327 4328 return (u64)(unsigned long)(elt_data->comm); 4329 } 4330 4331 static u64 hist_field_stack(struct hist_field *hist_field, 4332 struct tracing_map_elt *elt, 4333 struct trace_buffer *buffer, 4334 struct ring_buffer_event *rbe, 4335 void *event) 4336 { 4337 u32 str_item = *(u32 *)(event + hist_field->field->offset); 4338 int str_loc = str_item & 0xffff; 4339 char *addr = (char *)(event + str_loc); 4340 4341 return (u64)(unsigned long)addr; 4342 } 4343 4344 static u64 hist_fn_call(struct hist_field *hist_field, 4345 struct tracing_map_elt *elt, 4346 struct trace_buffer *buffer, 4347 struct ring_buffer_event *rbe, 4348 void *event) 4349 { 4350 switch (hist_field->fn_num) { 4351 case HIST_FIELD_FN_VAR_REF: 4352 return hist_field_var_ref(hist_field, elt, buffer, rbe, event); 4353 case HIST_FIELD_FN_COUNTER: 4354 return hist_field_counter(hist_field, elt, buffer, rbe, event); 4355 case HIST_FIELD_FN_CONST: 4356 return hist_field_const(hist_field, elt, buffer, rbe, event); 4357 case HIST_FIELD_FN_LOG2: 4358 return hist_field_log2(hist_field, elt, buffer, rbe, event); 4359 case HIST_FIELD_FN_BUCKET: 4360 return hist_field_bucket(hist_field, elt, buffer, rbe, event); 4361 case HIST_FIELD_FN_TIMESTAMP: 4362 return hist_field_timestamp(hist_field, elt, buffer, rbe, event); 4363 case HIST_FIELD_FN_CPU: 4364 return hist_field_cpu(hist_field, elt, buffer, rbe, event); 4365 case HIST_FIELD_FN_COMM: 4366 return hist_field_comm(hist_field, elt, buffer, rbe, event); 4367 case HIST_FIELD_FN_STRING: 4368 return hist_field_string(hist_field, elt, buffer, rbe, event); 4369 case HIST_FIELD_FN_DYNSTRING: 4370 return hist_field_dynstring(hist_field, elt, buffer, rbe, event); 4371 case HIST_FIELD_FN_RELDYNSTRING: 4372 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event); 4373 case HIST_FIELD_FN_PSTRING: 4374 return hist_field_pstring(hist_field, elt, buffer, rbe, event); 4375 case HIST_FIELD_FN_S64: 4376 return hist_field_s64(hist_field, elt, buffer, rbe, event); 4377 case HIST_FIELD_FN_U64: 4378 return hist_field_u64(hist_field, elt, buffer, rbe, event); 4379 case HIST_FIELD_FN_S32: 4380 return hist_field_s32(hist_field, elt, buffer, rbe, event); 4381 case HIST_FIELD_FN_U32: 4382 return hist_field_u32(hist_field, elt, buffer, rbe, event); 4383 case HIST_FIELD_FN_S16: 4384 return hist_field_s16(hist_field, elt, buffer, rbe, event); 4385 case HIST_FIELD_FN_U16: 4386 return hist_field_u16(hist_field, elt, buffer, rbe, event); 4387 case HIST_FIELD_FN_S8: 4388 return hist_field_s8(hist_field, elt, buffer, rbe, event); 4389 case HIST_FIELD_FN_U8: 4390 return hist_field_u8(hist_field, elt, buffer, rbe, event); 4391 case HIST_FIELD_FN_UMINUS: 4392 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event); 4393 case HIST_FIELD_FN_MINUS: 4394 return hist_field_minus(hist_field, elt, buffer, rbe, event); 4395 case HIST_FIELD_FN_PLUS: 4396 return hist_field_plus(hist_field, elt, buffer, rbe, event); 4397 case HIST_FIELD_FN_DIV: 4398 return hist_field_div(hist_field, elt, buffer, rbe, event); 4399 case HIST_FIELD_FN_MULT: 4400 return hist_field_mult(hist_field, elt, buffer, rbe, event); 4401 case HIST_FIELD_FN_DIV_POWER2: 4402 return div_by_power_of_two(hist_field, elt, buffer, rbe, event); 4403 case HIST_FIELD_FN_DIV_NOT_POWER2: 4404 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event); 4405 case HIST_FIELD_FN_DIV_MULT_SHIFT: 4406 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event); 4407 case HIST_FIELD_FN_EXECNAME: 4408 return hist_field_execname(hist_field, elt, buffer, rbe, event); 4409 case HIST_FIELD_FN_STACK: 4410 return hist_field_stack(hist_field, elt, buffer, rbe, event); 4411 default: 4412 return 0; 4413 } 4414 } 4415 4416 /* Convert a var that points to common_pid.execname to a string */ 4417 static void update_var_execname(struct hist_field *hist_field) 4418 { 4419 hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR | 4420 HIST_FIELD_FL_EXECNAME; 4421 hist_field->size = MAX_FILTER_STR_VAL; 4422 hist_field->is_signed = 0; 4423 4424 kfree_const(hist_field->type); 4425 hist_field->type = "char[]"; 4426 4427 hist_field->fn_num = HIST_FIELD_FN_EXECNAME; 4428 } 4429 4430 static int create_var_field(struct hist_trigger_data *hist_data, 4431 unsigned int val_idx, 4432 struct trace_event_file *file, 4433 char *var_name, char *expr_str) 4434 { 4435 struct trace_array *tr = hist_data->event_file->tr; 4436 unsigned long flags = 0; 4437 int ret; 4438 4439 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX)) 4440 return -EINVAL; 4441 4442 if (find_var(hist_data, file, var_name) && !hist_data->remove) { 4443 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name)); 4444 return -EINVAL; 4445 } 4446 4447 flags |= HIST_FIELD_FL_VAR; 4448 hist_data->n_vars++; 4449 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX)) 4450 return -EINVAL; 4451 4452 ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags); 4453 4454 if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME) 4455 update_var_execname(hist_data->fields[val_idx]); 4456 4457 if (!ret && hist_data->fields[val_idx]->flags & 4458 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) 4459 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++; 4460 4461 return ret; 4462 } 4463 4464 static int create_val_fields(struct hist_trigger_data *hist_data, 4465 struct trace_event_file *file) 4466 { 4467 unsigned int i, j = 1, n_hitcount = 0; 4468 char *fields_str, *field_str; 4469 int ret; 4470 4471 ret = create_hitcount_val(hist_data); 4472 if (ret) 4473 goto out; 4474 4475 fields_str = hist_data->attrs->vals_str; 4476 if (!fields_str) 4477 goto out; 4478 4479 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX && 4480 j < TRACING_MAP_VALS_MAX; i++) { 4481 field_str = strsep(&fields_str, ","); 4482 if (!field_str) 4483 break; 4484 4485 if (strcmp(field_str, "hitcount") == 0) { 4486 if (!n_hitcount++) 4487 continue; 4488 } 4489 4490 ret = create_val_field(hist_data, j++, file, field_str); 4491 if (ret) 4492 goto out; 4493 } 4494 4495 if (fields_str && (strcmp(fields_str, "hitcount") != 0)) 4496 ret = -EINVAL; 4497 out: 4498 /* There is only raw hitcount but nohitcount suppresses it. */ 4499 if (j == 1 && hist_data->attrs->no_hitcount) { 4500 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0); 4501 ret = -ENOENT; 4502 } 4503 4504 return ret; 4505 } 4506 4507 static int create_key_field(struct hist_trigger_data *hist_data, 4508 unsigned int key_idx, 4509 unsigned int key_offset, 4510 struct trace_event_file *file, 4511 char *field_str) 4512 { 4513 struct trace_array *tr = hist_data->event_file->tr; 4514 struct hist_field *hist_field = NULL; 4515 unsigned long flags = 0; 4516 unsigned int key_size; 4517 int ret = 0, n_subexprs = 0; 4518 4519 if (WARN_ON(key_idx >= HIST_FIELDS_MAX)) 4520 return -EINVAL; 4521 4522 flags |= HIST_FIELD_FL_KEY; 4523 4524 if (strcmp(field_str, "stacktrace") == 0) { 4525 flags |= HIST_FIELD_FL_STACKTRACE; 4526 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH; 4527 hist_field = create_hist_field(hist_data, NULL, flags, NULL); 4528 } else { 4529 hist_field = parse_expr(hist_data, file, field_str, flags, 4530 NULL, &n_subexprs); 4531 if (IS_ERR(hist_field)) { 4532 ret = PTR_ERR(hist_field); 4533 goto out; 4534 } 4535 4536 if (field_has_hist_vars(hist_field, 0)) { 4537 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str)); 4538 destroy_hist_field(hist_field, 0); 4539 ret = -EINVAL; 4540 goto out; 4541 } 4542 4543 key_size = hist_field->size; 4544 } 4545 4546 hist_data->fields[key_idx] = hist_field; 4547 4548 key_size = ALIGN(key_size, sizeof(u64)); 4549 hist_data->fields[key_idx]->size = key_size; 4550 hist_data->fields[key_idx]->offset = key_offset; 4551 4552 hist_data->key_size += key_size; 4553 4554 if (hist_data->key_size > HIST_KEY_SIZE_MAX) { 4555 ret = -EINVAL; 4556 goto out; 4557 } 4558 4559 hist_data->n_keys++; 4560 hist_data->n_fields++; 4561 4562 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX)) 4563 return -EINVAL; 4564 4565 ret = key_size; 4566 out: 4567 return ret; 4568 } 4569 4570 static int create_key_fields(struct hist_trigger_data *hist_data, 4571 struct trace_event_file *file) 4572 { 4573 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals; 4574 char *fields_str, *field_str; 4575 int ret = -EINVAL; 4576 4577 fields_str = hist_data->attrs->keys_str; 4578 if (!fields_str) 4579 goto out; 4580 4581 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) { 4582 field_str = strsep(&fields_str, ","); 4583 if (!field_str) 4584 break; 4585 ret = create_key_field(hist_data, i, key_offset, 4586 file, field_str); 4587 if (ret < 0) 4588 goto out; 4589 key_offset += ret; 4590 } 4591 if (fields_str) { 4592 ret = -EINVAL; 4593 goto out; 4594 } 4595 ret = 0; 4596 out: 4597 return ret; 4598 } 4599 4600 static int create_var_fields(struct hist_trigger_data *hist_data, 4601 struct trace_event_file *file) 4602 { 4603 unsigned int i, j = hist_data->n_vals; 4604 int ret = 0; 4605 4606 unsigned int n_vars = hist_data->attrs->var_defs.n_vars; 4607 4608 for (i = 0; i < n_vars; i++) { 4609 char *var_name = hist_data->attrs->var_defs.name[i]; 4610 char *expr = hist_data->attrs->var_defs.expr[i]; 4611 4612 ret = create_var_field(hist_data, j++, file, var_name, expr); 4613 if (ret) 4614 goto out; 4615 } 4616 out: 4617 return ret; 4618 } 4619 4620 static void free_var_defs(struct hist_trigger_data *hist_data) 4621 { 4622 unsigned int i; 4623 4624 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) { 4625 kfree(hist_data->attrs->var_defs.name[i]); 4626 kfree(hist_data->attrs->var_defs.expr[i]); 4627 } 4628 4629 hist_data->attrs->var_defs.n_vars = 0; 4630 } 4631 4632 static int parse_var_defs(struct hist_trigger_data *hist_data) 4633 { 4634 struct trace_array *tr = hist_data->event_file->tr; 4635 char *s, *str, *var_name, *field_str; 4636 unsigned int i, j, n_vars = 0; 4637 int ret = 0; 4638 4639 for (i = 0; i < hist_data->attrs->n_assignments; i++) { 4640 str = hist_data->attrs->assignment_str[i]; 4641 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) { 4642 field_str = strsep(&str, ","); 4643 if (!field_str) 4644 break; 4645 4646 var_name = strsep(&field_str, "="); 4647 if (!var_name || !field_str) { 4648 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT, 4649 errpos(var_name)); 4650 ret = -EINVAL; 4651 goto free; 4652 } 4653 4654 if (n_vars == TRACING_MAP_VARS_MAX) { 4655 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name)); 4656 ret = -EINVAL; 4657 goto free; 4658 } 4659 4660 s = kstrdup(var_name, GFP_KERNEL); 4661 if (!s) { 4662 ret = -ENOMEM; 4663 goto free; 4664 } 4665 hist_data->attrs->var_defs.name[n_vars] = s; 4666 4667 s = kstrdup(field_str, GFP_KERNEL); 4668 if (!s) { 4669 kfree(hist_data->attrs->var_defs.name[n_vars]); 4670 hist_data->attrs->var_defs.name[n_vars] = NULL; 4671 ret = -ENOMEM; 4672 goto free; 4673 } 4674 hist_data->attrs->var_defs.expr[n_vars++] = s; 4675 4676 hist_data->attrs->var_defs.n_vars = n_vars; 4677 } 4678 } 4679 4680 return ret; 4681 free: 4682 free_var_defs(hist_data); 4683 4684 return ret; 4685 } 4686 4687 static int create_hist_fields(struct hist_trigger_data *hist_data, 4688 struct trace_event_file *file) 4689 { 4690 int ret; 4691 4692 ret = parse_var_defs(hist_data); 4693 if (ret) 4694 return ret; 4695 4696 ret = create_val_fields(hist_data, file); 4697 if (ret) 4698 goto out; 4699 4700 ret = create_var_fields(hist_data, file); 4701 if (ret) 4702 goto out; 4703 4704 ret = create_key_fields(hist_data, file); 4705 4706 out: 4707 free_var_defs(hist_data); 4708 4709 return ret; 4710 } 4711 4712 static int is_descending(struct trace_array *tr, const char *str) 4713 { 4714 if (!str) 4715 return 0; 4716 4717 if (strcmp(str, "descending") == 0) 4718 return 1; 4719 4720 if (strcmp(str, "ascending") == 0) 4721 return 0; 4722 4723 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str)); 4724 4725 return -EINVAL; 4726 } 4727 4728 static int create_sort_keys(struct hist_trigger_data *hist_data) 4729 { 4730 struct trace_array *tr = hist_data->event_file->tr; 4731 char *fields_str = hist_data->attrs->sort_key_str; 4732 struct tracing_map_sort_key *sort_key; 4733 int descending, ret = 0; 4734 unsigned int i, j, k; 4735 4736 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */ 4737 4738 if (!fields_str) 4739 goto out; 4740 4741 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) { 4742 struct hist_field *hist_field; 4743 char *field_str, *field_name; 4744 const char *test_name; 4745 4746 sort_key = &hist_data->sort_keys[i]; 4747 4748 field_str = strsep(&fields_str, ","); 4749 if (!field_str) 4750 break; 4751 4752 if (!*field_str) { 4753 ret = -EINVAL; 4754 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4755 break; 4756 } 4757 4758 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) { 4759 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort=")); 4760 ret = -EINVAL; 4761 break; 4762 } 4763 4764 field_name = strsep(&field_str, "."); 4765 if (!field_name || !*field_name) { 4766 ret = -EINVAL; 4767 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort=")); 4768 break; 4769 } 4770 4771 if (strcmp(field_name, "hitcount") == 0) { 4772 descending = is_descending(tr, field_str); 4773 if (descending < 0) { 4774 ret = descending; 4775 break; 4776 } 4777 sort_key->descending = descending; 4778 continue; 4779 } 4780 4781 for (j = 1, k = 1; j < hist_data->n_fields; j++) { 4782 unsigned int idx; 4783 4784 hist_field = hist_data->fields[j]; 4785 if (hist_field->flags & HIST_FIELD_FL_VAR) 4786 continue; 4787 4788 idx = k++; 4789 4790 test_name = hist_field_name(hist_field, 0); 4791 4792 if (strcmp(field_name, test_name) == 0) { 4793 sort_key->field_idx = idx; 4794 descending = is_descending(tr, field_str); 4795 if (descending < 0) { 4796 ret = descending; 4797 goto out; 4798 } 4799 sort_key->descending = descending; 4800 break; 4801 } 4802 } 4803 if (j == hist_data->n_fields) { 4804 ret = -EINVAL; 4805 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name)); 4806 break; 4807 } 4808 } 4809 4810 hist_data->n_sort_keys = i; 4811 out: 4812 return ret; 4813 } 4814 4815 static void destroy_actions(struct hist_trigger_data *hist_data) 4816 { 4817 unsigned int i; 4818 4819 for (i = 0; i < hist_data->n_actions; i++) { 4820 struct action_data *data = hist_data->actions[i]; 4821 4822 if (data->handler == HANDLER_ONMATCH) 4823 onmatch_destroy(data); 4824 else if (data->handler == HANDLER_ONMAX || 4825 data->handler == HANDLER_ONCHANGE) 4826 track_data_destroy(hist_data, data); 4827 else 4828 kfree(data); 4829 } 4830 } 4831 4832 static int parse_actions(struct hist_trigger_data *hist_data) 4833 { 4834 struct trace_array *tr = hist_data->event_file->tr; 4835 struct action_data *data; 4836 unsigned int i; 4837 int ret = 0; 4838 char *str; 4839 int len; 4840 4841 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4842 enum handler_id hid = 0; 4843 char *action_str; 4844 4845 str = hist_data->attrs->action_str[i]; 4846 4847 if ((len = str_has_prefix(str, "onmatch("))) 4848 hid = HANDLER_ONMATCH; 4849 else if ((len = str_has_prefix(str, "onmax("))) 4850 hid = HANDLER_ONMAX; 4851 else if ((len = str_has_prefix(str, "onchange("))) 4852 hid = HANDLER_ONCHANGE; 4853 4854 action_str = str + len; 4855 4856 switch (hid) { 4857 case HANDLER_ONMATCH: 4858 data = onmatch_parse(tr, action_str); 4859 break; 4860 case HANDLER_ONMAX: 4861 case HANDLER_ONCHANGE: 4862 data = track_data_parse(hist_data, action_str, hid); 4863 break; 4864 default: 4865 data = ERR_PTR(-EINVAL); 4866 break; 4867 } 4868 4869 if (IS_ERR(data)) { 4870 ret = PTR_ERR(data); 4871 break; 4872 } 4873 4874 hist_data->actions[hist_data->n_actions++] = data; 4875 } 4876 4877 return ret; 4878 } 4879 4880 static int create_actions(struct hist_trigger_data *hist_data) 4881 { 4882 struct action_data *data; 4883 unsigned int i; 4884 int ret = 0; 4885 4886 for (i = 0; i < hist_data->attrs->n_actions; i++) { 4887 data = hist_data->actions[i]; 4888 4889 if (data->handler == HANDLER_ONMATCH) { 4890 ret = onmatch_create(hist_data, data); 4891 if (ret) 4892 break; 4893 } else if (data->handler == HANDLER_ONMAX || 4894 data->handler == HANDLER_ONCHANGE) { 4895 ret = track_data_create(hist_data, data); 4896 if (ret) 4897 break; 4898 } else { 4899 ret = -EINVAL; 4900 break; 4901 } 4902 } 4903 4904 return ret; 4905 } 4906 4907 static void print_actions(struct seq_file *m, 4908 struct hist_trigger_data *hist_data, 4909 struct tracing_map_elt *elt) 4910 { 4911 unsigned int i; 4912 4913 for (i = 0; i < hist_data->n_actions; i++) { 4914 struct action_data *data = hist_data->actions[i]; 4915 4916 if (data->action == ACTION_SNAPSHOT) 4917 continue; 4918 4919 if (data->handler == HANDLER_ONMAX || 4920 data->handler == HANDLER_ONCHANGE) 4921 track_data_print(m, hist_data, elt, data); 4922 } 4923 } 4924 4925 static void print_action_spec(struct seq_file *m, 4926 struct hist_trigger_data *hist_data, 4927 struct action_data *data) 4928 { 4929 unsigned int i; 4930 4931 if (data->action == ACTION_SAVE) { 4932 for (i = 0; i < hist_data->n_save_vars; i++) { 4933 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name); 4934 if (i < hist_data->n_save_vars - 1) 4935 seq_puts(m, ","); 4936 } 4937 } else if (data->action == ACTION_TRACE) { 4938 if (data->use_trace_keyword) 4939 seq_printf(m, "%s", data->synth_event_name); 4940 for (i = 0; i < data->n_params; i++) { 4941 if (i || data->use_trace_keyword) 4942 seq_puts(m, ","); 4943 seq_printf(m, "%s", data->params[i]); 4944 } 4945 } 4946 } 4947 4948 static void print_track_data_spec(struct seq_file *m, 4949 struct hist_trigger_data *hist_data, 4950 struct action_data *data) 4951 { 4952 if (data->handler == HANDLER_ONMAX) 4953 seq_puts(m, ":onmax("); 4954 else if (data->handler == HANDLER_ONCHANGE) 4955 seq_puts(m, ":onchange("); 4956 seq_printf(m, "%s", data->track_data.var_str); 4957 seq_printf(m, ").%s(", data->action_name); 4958 4959 print_action_spec(m, hist_data, data); 4960 4961 seq_puts(m, ")"); 4962 } 4963 4964 static void print_onmatch_spec(struct seq_file *m, 4965 struct hist_trigger_data *hist_data, 4966 struct action_data *data) 4967 { 4968 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system, 4969 data->match_data.event); 4970 4971 seq_printf(m, "%s(", data->action_name); 4972 4973 print_action_spec(m, hist_data, data); 4974 4975 seq_puts(m, ")"); 4976 } 4977 4978 static bool actions_match(struct hist_trigger_data *hist_data, 4979 struct hist_trigger_data *hist_data_test) 4980 { 4981 unsigned int i, j; 4982 4983 if (hist_data->n_actions != hist_data_test->n_actions) 4984 return false; 4985 4986 for (i = 0; i < hist_data->n_actions; i++) { 4987 struct action_data *data = hist_data->actions[i]; 4988 struct action_data *data_test = hist_data_test->actions[i]; 4989 char *action_name, *action_name_test; 4990 4991 if (data->handler != data_test->handler) 4992 return false; 4993 if (data->action != data_test->action) 4994 return false; 4995 4996 if (data->n_params != data_test->n_params) 4997 return false; 4998 4999 for (j = 0; j < data->n_params; j++) { 5000 if (strcmp(data->params[j], data_test->params[j]) != 0) 5001 return false; 5002 } 5003 5004 if (data->use_trace_keyword) 5005 action_name = data->synth_event_name; 5006 else 5007 action_name = data->action_name; 5008 5009 if (data_test->use_trace_keyword) 5010 action_name_test = data_test->synth_event_name; 5011 else 5012 action_name_test = data_test->action_name; 5013 5014 if (strcmp(action_name, action_name_test) != 0) 5015 return false; 5016 5017 if (data->handler == HANDLER_ONMATCH) { 5018 if (strcmp(data->match_data.event_system, 5019 data_test->match_data.event_system) != 0) 5020 return false; 5021 if (strcmp(data->match_data.event, 5022 data_test->match_data.event) != 0) 5023 return false; 5024 } else if (data->handler == HANDLER_ONMAX || 5025 data->handler == HANDLER_ONCHANGE) { 5026 if (strcmp(data->track_data.var_str, 5027 data_test->track_data.var_str) != 0) 5028 return false; 5029 } 5030 } 5031 5032 return true; 5033 } 5034 5035 5036 static void print_actions_spec(struct seq_file *m, 5037 struct hist_trigger_data *hist_data) 5038 { 5039 unsigned int i; 5040 5041 for (i = 0; i < hist_data->n_actions; i++) { 5042 struct action_data *data = hist_data->actions[i]; 5043 5044 if (data->handler == HANDLER_ONMATCH) 5045 print_onmatch_spec(m, hist_data, data); 5046 else if (data->handler == HANDLER_ONMAX || 5047 data->handler == HANDLER_ONCHANGE) 5048 print_track_data_spec(m, hist_data, data); 5049 } 5050 } 5051 5052 static void destroy_field_var_hists(struct hist_trigger_data *hist_data) 5053 { 5054 unsigned int i; 5055 5056 for (i = 0; i < hist_data->n_field_var_hists; i++) { 5057 kfree(hist_data->field_var_hists[i]->cmd); 5058 kfree(hist_data->field_var_hists[i]); 5059 } 5060 } 5061 5062 static void destroy_hist_data(struct hist_trigger_data *hist_data) 5063 { 5064 if (!hist_data) 5065 return; 5066 5067 destroy_hist_trigger_attrs(hist_data->attrs); 5068 destroy_hist_fields(hist_data); 5069 tracing_map_destroy(hist_data->map); 5070 5071 destroy_actions(hist_data); 5072 destroy_field_vars(hist_data); 5073 destroy_field_var_hists(hist_data); 5074 5075 kfree(hist_data); 5076 } 5077 5078 static int create_tracing_map_fields(struct hist_trigger_data *hist_data) 5079 { 5080 struct tracing_map *map = hist_data->map; 5081 struct ftrace_event_field *field; 5082 struct hist_field *hist_field; 5083 int i, idx = 0; 5084 5085 for_each_hist_field(i, hist_data) { 5086 hist_field = hist_data->fields[i]; 5087 if (hist_field->flags & HIST_FIELD_FL_KEY) { 5088 tracing_map_cmp_fn_t cmp_fn; 5089 5090 field = hist_field->field; 5091 5092 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE) 5093 cmp_fn = tracing_map_cmp_none; 5094 else if (!field || hist_field->flags & HIST_FIELD_FL_CPU) 5095 cmp_fn = tracing_map_cmp_num(hist_field->size, 5096 hist_field->is_signed); 5097 else if (is_string_field(field)) 5098 cmp_fn = tracing_map_cmp_string; 5099 else 5100 cmp_fn = tracing_map_cmp_num(field->size, 5101 field->is_signed); 5102 idx = tracing_map_add_key_field(map, 5103 hist_field->offset, 5104 cmp_fn); 5105 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR)) 5106 idx = tracing_map_add_sum_field(map); 5107 5108 if (idx < 0) 5109 return idx; 5110 5111 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5112 idx = tracing_map_add_var(map); 5113 if (idx < 0) 5114 return idx; 5115 hist_field->var.idx = idx; 5116 hist_field->var.hist_data = hist_data; 5117 } 5118 } 5119 5120 return 0; 5121 } 5122 5123 static struct hist_trigger_data * 5124 create_hist_data(unsigned int map_bits, 5125 struct hist_trigger_attrs *attrs, 5126 struct trace_event_file *file, 5127 bool remove) 5128 { 5129 const struct tracing_map_ops *map_ops = NULL; 5130 struct hist_trigger_data *hist_data; 5131 int ret = 0; 5132 5133 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL); 5134 if (!hist_data) 5135 return ERR_PTR(-ENOMEM); 5136 5137 hist_data->attrs = attrs; 5138 hist_data->remove = remove; 5139 hist_data->event_file = file; 5140 5141 ret = parse_actions(hist_data); 5142 if (ret) 5143 goto free; 5144 5145 ret = create_hist_fields(hist_data, file); 5146 if (ret) 5147 goto free; 5148 5149 ret = create_sort_keys(hist_data); 5150 if (ret) 5151 goto free; 5152 5153 map_ops = &hist_trigger_elt_data_ops; 5154 5155 hist_data->map = tracing_map_create(map_bits, hist_data->key_size, 5156 map_ops, hist_data); 5157 if (IS_ERR(hist_data->map)) { 5158 ret = PTR_ERR(hist_data->map); 5159 hist_data->map = NULL; 5160 goto free; 5161 } 5162 5163 ret = create_tracing_map_fields(hist_data); 5164 if (ret) 5165 goto free; 5166 out: 5167 return hist_data; 5168 free: 5169 hist_data->attrs = NULL; 5170 5171 destroy_hist_data(hist_data); 5172 5173 hist_data = ERR_PTR(ret); 5174 5175 goto out; 5176 } 5177 5178 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, 5179 struct tracing_map_elt *elt, 5180 struct trace_buffer *buffer, void *rec, 5181 struct ring_buffer_event *rbe, 5182 u64 *var_ref_vals) 5183 { 5184 struct hist_elt_data *elt_data; 5185 struct hist_field *hist_field; 5186 unsigned int i, var_idx; 5187 u64 hist_val; 5188 5189 elt_data = elt->private_data; 5190 elt_data->var_ref_vals = var_ref_vals; 5191 5192 for_each_hist_val_field(i, hist_data) { 5193 hist_field = hist_data->fields[i]; 5194 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5195 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5196 var_idx = hist_field->var.idx; 5197 5198 if (hist_field->flags & 5199 (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) { 5200 unsigned int str_start, var_str_idx, idx; 5201 char *str, *val_str; 5202 unsigned int size; 5203 5204 str_start = hist_data->n_field_var_str + 5205 hist_data->n_save_var_str; 5206 var_str_idx = hist_field->var_str_idx; 5207 idx = str_start + var_str_idx; 5208 5209 str = elt_data->field_var_str[idx]; 5210 val_str = (char *)(uintptr_t)hist_val; 5211 5212 if (hist_field->flags & HIST_FIELD_FL_STRING) { 5213 size = min(hist_field->size, STR_VAR_LEN_MAX); 5214 strscpy(str, val_str, size); 5215 } else { 5216 char *stack_start = str + sizeof(unsigned long); 5217 int e; 5218 5219 e = stack_trace_save((void *)stack_start, 5220 HIST_STACKTRACE_DEPTH, 5221 HIST_STACKTRACE_SKIP); 5222 if (e < HIST_STACKTRACE_DEPTH - 1) 5223 ((unsigned long *)stack_start)[e] = 0; 5224 *((unsigned long *)str) = e; 5225 } 5226 hist_val = (u64)(uintptr_t)str; 5227 } 5228 tracing_map_set_var(elt, var_idx, hist_val); 5229 continue; 5230 } 5231 tracing_map_update_sum(elt, i, hist_val); 5232 } 5233 5234 for_each_hist_key_field(i, hist_data) { 5235 hist_field = hist_data->fields[i]; 5236 if (hist_field->flags & HIST_FIELD_FL_VAR) { 5237 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); 5238 var_idx = hist_field->var.idx; 5239 tracing_map_set_var(elt, var_idx, hist_val); 5240 } 5241 } 5242 5243 update_field_vars(hist_data, elt, buffer, rbe, rec); 5244 } 5245 5246 static inline void add_to_key(char *compound_key, void *key, 5247 struct hist_field *key_field, void *rec) 5248 { 5249 size_t size = key_field->size; 5250 5251 if (key_field->flags & HIST_FIELD_FL_STRING) { 5252 5253 if (key_field->flags & HIST_FIELD_FL_COMM) { 5254 size = strlen((char *)key); 5255 } else { 5256 struct ftrace_event_field *field; 5257 5258 field = key_field->field; 5259 if (field->filter_type == FILTER_DYN_STRING || 5260 field->filter_type == FILTER_RDYN_STRING) 5261 size = *(u32 *)(rec + field->offset) >> 16; 5262 else if (field->filter_type == FILTER_STATIC_STRING) 5263 size = field->size; 5264 } 5265 5266 /* ensure NULL-termination */ 5267 if (size > key_field->size - 1) 5268 size = key_field->size - 1; 5269 } 5270 memcpy(compound_key + key_field->offset, key, size); 5271 } 5272 5273 static void 5274 hist_trigger_actions(struct hist_trigger_data *hist_data, 5275 struct tracing_map_elt *elt, 5276 struct trace_buffer *buffer, void *rec, 5277 struct ring_buffer_event *rbe, void *key, 5278 u64 *var_ref_vals) 5279 { 5280 struct action_data *data; 5281 unsigned int i; 5282 5283 for (i = 0; i < hist_data->n_actions; i++) { 5284 data = hist_data->actions[i]; 5285 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals); 5286 } 5287 } 5288 5289 /* 5290 * The hist_pad structure is used to save information to create 5291 * a histogram from the histogram trigger. It's too big to store 5292 * on the stack, so when the histogram trigger is initialized 5293 * a percpu array of 4 hist_pad structures is allocated. 5294 * This will cover every context from normal, softirq, irq and NMI 5295 * in the very unlikely event that a trigger happens at each of 5296 * these contexts and interrupts a currently active trigger. 5297 */ 5298 struct hist_pad { 5299 unsigned long entries[HIST_STACKTRACE_DEPTH]; 5300 u64 var_ref_vals[TRACING_MAP_VARS_MAX]; 5301 char compound_key[HIST_KEY_SIZE_MAX]; 5302 }; 5303 5304 static struct hist_pad __percpu *hist_pads; 5305 static DEFINE_PER_CPU(int, hist_pad_cnt); 5306 static refcount_t hist_pad_ref; 5307 5308 /* One hist_pad for every context (normal, softirq, irq, NMI) */ 5309 #define MAX_HIST_CNT 4 5310 5311 static int alloc_hist_pad(void) 5312 { 5313 lockdep_assert_held(&event_mutex); 5314 5315 if (refcount_read(&hist_pad_ref)) { 5316 refcount_inc(&hist_pad_ref); 5317 return 0; 5318 } 5319 5320 hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT, 5321 __alignof__(struct hist_pad)); 5322 if (!hist_pads) 5323 return -ENOMEM; 5324 5325 refcount_set(&hist_pad_ref, 1); 5326 return 0; 5327 } 5328 5329 static void free_hist_pad(void) 5330 { 5331 lockdep_assert_held(&event_mutex); 5332 5333 if (!refcount_dec_and_test(&hist_pad_ref)) 5334 return; 5335 5336 free_percpu(hist_pads); 5337 hist_pads = NULL; 5338 } 5339 5340 static struct hist_pad *get_hist_pad(void) 5341 { 5342 struct hist_pad *hist_pad; 5343 int cnt; 5344 5345 if (WARN_ON_ONCE(!hist_pads)) 5346 return NULL; 5347 5348 preempt_disable(); 5349 5350 hist_pad = per_cpu_ptr(hist_pads, smp_processor_id()); 5351 5352 if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) { 5353 preempt_enable(); 5354 return NULL; 5355 } 5356 5357 cnt = this_cpu_inc_return(hist_pad_cnt) - 1; 5358 5359 return &hist_pad[cnt]; 5360 } 5361 5362 static void put_hist_pad(void) 5363 { 5364 this_cpu_dec(hist_pad_cnt); 5365 preempt_enable(); 5366 } 5367 5368 static void event_hist_trigger(struct event_trigger_data *data, 5369 struct trace_buffer *buffer, void *rec, 5370 struct ring_buffer_event *rbe) 5371 { 5372 struct hist_trigger_data *hist_data = data->private_data; 5373 bool use_compound_key = (hist_data->n_keys > 1); 5374 struct tracing_map_elt *elt = NULL; 5375 struct hist_field *key_field; 5376 struct hist_pad *hist_pad; 5377 u64 field_contents; 5378 void *key = NULL; 5379 unsigned int i; 5380 5381 if (unlikely(!rbe)) 5382 return; 5383 5384 hist_pad = get_hist_pad(); 5385 if (!hist_pad) 5386 return; 5387 5388 memset(hist_pad->compound_key, 0, hist_data->key_size); 5389 5390 for_each_hist_key_field(i, hist_data) { 5391 key_field = hist_data->fields[i]; 5392 5393 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5394 unsigned long *entries = hist_pad->entries; 5395 5396 memset(entries, 0, HIST_STACKTRACE_SIZE); 5397 if (key_field->field) { 5398 unsigned long *stack, n_entries; 5399 5400 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5401 stack = (unsigned long *)(long)field_contents; 5402 n_entries = *stack; 5403 memcpy(entries, ++stack, n_entries * sizeof(unsigned long)); 5404 } else { 5405 stack_trace_save(entries, HIST_STACKTRACE_DEPTH, 5406 HIST_STACKTRACE_SKIP); 5407 } 5408 key = entries; 5409 } else { 5410 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); 5411 if (key_field->flags & HIST_FIELD_FL_STRING) { 5412 key = (void *)(unsigned long)field_contents; 5413 use_compound_key = true; 5414 } else 5415 key = (void *)&field_contents; 5416 } 5417 5418 if (use_compound_key) 5419 add_to_key(hist_pad->compound_key, key, key_field, rec); 5420 } 5421 5422 if (use_compound_key) 5423 key = hist_pad->compound_key; 5424 5425 if (hist_data->n_var_refs && 5426 !resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false)) 5427 goto out; 5428 5429 elt = tracing_map_insert(hist_data->map, key); 5430 if (!elt) 5431 goto out; 5432 5433 hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals); 5434 5435 if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) { 5436 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, 5437 key, hist_pad->var_ref_vals); 5438 } 5439 5440 hist_poll_wakeup(); 5441 5442 out: 5443 put_hist_pad(); 5444 } 5445 5446 static void hist_trigger_stacktrace_print(struct seq_file *m, 5447 unsigned long *stacktrace_entries, 5448 unsigned int max_entries) 5449 { 5450 unsigned int spaces = 8; 5451 unsigned int i; 5452 5453 for (i = 0; i < max_entries; i++) { 5454 if (!stacktrace_entries[i]) 5455 return; 5456 5457 seq_printf(m, "%*c", 1 + spaces, ' '); 5458 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]); 5459 } 5460 } 5461 5462 static void hist_trigger_print_key(struct seq_file *m, 5463 struct hist_trigger_data *hist_data, 5464 void *key, 5465 struct tracing_map_elt *elt) 5466 { 5467 struct hist_field *key_field; 5468 bool multiline = false; 5469 const char *field_name; 5470 unsigned int i; 5471 u64 uval; 5472 5473 seq_puts(m, "{ "); 5474 5475 for_each_hist_key_field(i, hist_data) { 5476 key_field = hist_data->fields[i]; 5477 5478 if (i > hist_data->n_vals) 5479 seq_puts(m, ", "); 5480 5481 field_name = hist_field_name(key_field, 0); 5482 5483 if (key_field->flags & HIST_FIELD_FL_HEX) { 5484 uval = *(u64 *)(key + key_field->offset); 5485 seq_printf(m, "%s: %llx", field_name, uval); 5486 } else if (key_field->flags & HIST_FIELD_FL_SYM) { 5487 uval = *(u64 *)(key + key_field->offset); 5488 seq_printf(m, "%s: [%llx] %-45ps", field_name, 5489 uval, (void *)(uintptr_t)uval); 5490 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) { 5491 uval = *(u64 *)(key + key_field->offset); 5492 seq_printf(m, "%s: [%llx] %-55pS", field_name, 5493 uval, (void *)(uintptr_t)uval); 5494 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) { 5495 struct hist_elt_data *elt_data = elt->private_data; 5496 char *comm; 5497 5498 if (WARN_ON_ONCE(!elt_data)) 5499 return; 5500 5501 comm = elt_data->comm; 5502 5503 uval = *(u64 *)(key + key_field->offset); 5504 seq_printf(m, "%s: %-16s[%10llu]", field_name, 5505 comm, uval); 5506 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) { 5507 const char *syscall_name; 5508 5509 uval = *(u64 *)(key + key_field->offset); 5510 syscall_name = get_syscall_name(uval); 5511 if (!syscall_name) 5512 syscall_name = "unknown_syscall"; 5513 5514 seq_printf(m, "%s: %-30s[%3llu]", field_name, 5515 syscall_name, uval); 5516 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) { 5517 if (key_field->field) 5518 seq_printf(m, "%s.stacktrace", key_field->field->name); 5519 else 5520 seq_puts(m, "common_stacktrace:\n"); 5521 hist_trigger_stacktrace_print(m, 5522 key + key_field->offset, 5523 HIST_STACKTRACE_DEPTH); 5524 multiline = true; 5525 } else if (key_field->flags & HIST_FIELD_FL_LOG2) { 5526 seq_printf(m, "%s: ~ 2^%-2llu", field_name, 5527 *(u64 *)(key + key_field->offset)); 5528 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) { 5529 unsigned long buckets = key_field->buckets; 5530 uval = *(u64 *)(key + key_field->offset); 5531 seq_printf(m, "%s: ~ %llu-%llu", field_name, 5532 uval, uval + buckets -1); 5533 } else if (key_field->flags & HIST_FIELD_FL_STRING) { 5534 seq_printf(m, "%s: %-50s", field_name, 5535 (char *)(key + key_field->offset)); 5536 } else { 5537 uval = *(u64 *)(key + key_field->offset); 5538 seq_printf(m, "%s: %10llu", field_name, uval); 5539 } 5540 } 5541 5542 if (!multiline) 5543 seq_puts(m, " "); 5544 5545 seq_puts(m, "}"); 5546 } 5547 5548 /* Get the 100 times of the percentage of @val in @total */ 5549 static inline unsigned int __get_percentage(u64 val, u64 total) 5550 { 5551 if (!total) 5552 goto div0; 5553 5554 if (val < (U64_MAX / 10000)) 5555 return (unsigned int)div64_ul(val * 10000, total); 5556 5557 total = div64_u64(total, 10000); 5558 if (!total) 5559 goto div0; 5560 5561 return (unsigned int)div64_ul(val, total); 5562 div0: 5563 return val ? UINT_MAX : 0; 5564 } 5565 5566 #define BAR_CHAR '#' 5567 5568 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max) 5569 { 5570 unsigned int len = __get_percentage(val, max); 5571 int i; 5572 5573 if (len == UINT_MAX) { 5574 snprintf(buf, size, "[ERROR]"); 5575 return buf; 5576 } 5577 5578 len = len * size / 10000; 5579 for (i = 0; i < len && i < size; i++) 5580 buf[i] = BAR_CHAR; 5581 while (i < size) 5582 buf[i++] = ' '; 5583 buf[size] = '\0'; 5584 5585 return buf; 5586 } 5587 5588 struct hist_val_stat { 5589 u64 max; 5590 u64 total; 5591 }; 5592 5593 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx, 5594 const char *field_name, unsigned long flags, 5595 struct hist_val_stat *stats, 5596 struct tracing_map_elt *elt) 5597 { 5598 u64 val = tracing_map_read_sum(elt, idx); 5599 unsigned int pc; 5600 char bar[21]; 5601 5602 if (flags & HIST_FIELD_FL_PERCENT) { 5603 pc = __get_percentage(val, stats[idx].total); 5604 if (pc == UINT_MAX) 5605 seq_printf(m, " %s (%%):[ERROR]", field_name); 5606 else 5607 seq_printf(m, " %s (%%): %3u.%02u", field_name, 5608 pc / 100, pc % 100); 5609 } else if (flags & HIST_FIELD_FL_GRAPH) { 5610 seq_printf(m, " %s: %20s", field_name, 5611 __fill_bar_str(bar, 20, val, stats[idx].max)); 5612 } else if (flags & HIST_FIELD_FL_HEX) { 5613 seq_printf(m, " %s: %10llx", field_name, val); 5614 } else { 5615 seq_printf(m, " %s: %10llu", field_name, val); 5616 } 5617 } 5618 5619 static void hist_trigger_entry_print(struct seq_file *m, 5620 struct hist_trigger_data *hist_data, 5621 struct hist_val_stat *stats, 5622 void *key, 5623 struct tracing_map_elt *elt) 5624 { 5625 const char *field_name; 5626 unsigned int i = HITCOUNT_IDX; 5627 unsigned long flags; 5628 5629 hist_trigger_print_key(m, hist_data, key, elt); 5630 5631 /* At first, show the raw hitcount if !nohitcount */ 5632 if (!hist_data->attrs->no_hitcount) 5633 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt); 5634 5635 for (i = 1; i < hist_data->n_vals; i++) { 5636 field_name = hist_field_name(hist_data->fields[i], 0); 5637 flags = hist_data->fields[i]->flags; 5638 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR) 5639 continue; 5640 5641 seq_puts(m, " "); 5642 hist_trigger_print_val(m, i, field_name, flags, stats, elt); 5643 } 5644 5645 print_actions(m, hist_data, elt); 5646 5647 seq_puts(m, "\n"); 5648 } 5649 5650 static int print_entries(struct seq_file *m, 5651 struct hist_trigger_data *hist_data) 5652 { 5653 struct tracing_map_sort_entry **sort_entries = NULL; 5654 struct tracing_map *map = hist_data->map; 5655 int i, j, n_entries; 5656 struct hist_val_stat *stats = NULL; 5657 u64 val; 5658 5659 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys, 5660 hist_data->n_sort_keys, 5661 &sort_entries); 5662 if (n_entries < 0) 5663 return n_entries; 5664 5665 /* Calculate the max and the total for each field if needed. */ 5666 for (j = 0; j < hist_data->n_vals; j++) { 5667 if (!(hist_data->fields[j]->flags & 5668 (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH))) 5669 continue; 5670 if (!stats) { 5671 stats = kcalloc(hist_data->n_vals, sizeof(*stats), 5672 GFP_KERNEL); 5673 if (!stats) { 5674 n_entries = -ENOMEM; 5675 goto out; 5676 } 5677 } 5678 for (i = 0; i < n_entries; i++) { 5679 val = tracing_map_read_sum(sort_entries[i]->elt, j); 5680 stats[j].total += val; 5681 if (stats[j].max < val) 5682 stats[j].max = val; 5683 } 5684 } 5685 5686 for (i = 0; i < n_entries; i++) 5687 hist_trigger_entry_print(m, hist_data, stats, 5688 sort_entries[i]->key, 5689 sort_entries[i]->elt); 5690 5691 kfree(stats); 5692 out: 5693 tracing_map_destroy_sort_entries(sort_entries, n_entries); 5694 5695 return n_entries; 5696 } 5697 5698 static void hist_trigger_show(struct seq_file *m, 5699 struct event_trigger_data *data, int n) 5700 { 5701 struct hist_trigger_data *hist_data; 5702 int n_entries; 5703 5704 if (n > 0) 5705 seq_puts(m, "\n\n"); 5706 5707 seq_puts(m, "# event histogram\n#\n# trigger info: "); 5708 data->cmd_ops->print(m, data); 5709 seq_puts(m, "#\n\n"); 5710 5711 hist_data = data->private_data; 5712 n_entries = print_entries(m, hist_data); 5713 if (n_entries < 0) 5714 n_entries = 0; 5715 5716 track_data_snapshot_print(m, hist_data); 5717 5718 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n", 5719 (u64)atomic64_read(&hist_data->map->hits), 5720 n_entries, (u64)atomic64_read(&hist_data->map->drops)); 5721 } 5722 5723 struct hist_file_data { 5724 struct file *file; 5725 u64 last_read; 5726 u64 last_act; 5727 }; 5728 5729 static u64 get_hist_hit_count(struct trace_event_file *event_file) 5730 { 5731 struct hist_trigger_data *hist_data; 5732 struct event_trigger_data *data; 5733 u64 ret = 0; 5734 5735 list_for_each_entry(data, &event_file->triggers, list) { 5736 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) { 5737 hist_data = data->private_data; 5738 ret += atomic64_read(&hist_data->map->hits); 5739 } 5740 } 5741 return ret; 5742 } 5743 5744 static int hist_show(struct seq_file *m, void *v) 5745 { 5746 struct hist_file_data *hist_file = m->private; 5747 struct event_trigger_data *data; 5748 struct trace_event_file *event_file; 5749 int n = 0; 5750 5751 guard(mutex)(&event_mutex); 5752 5753 event_file = event_file_file(hist_file->file); 5754 if (unlikely(!event_file)) 5755 return -ENODEV; 5756 5757 list_for_each_entry(data, &event_file->triggers, list) { 5758 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 5759 hist_trigger_show(m, data, n++); 5760 } 5761 hist_file->last_read = get_hist_hit_count(event_file); 5762 /* 5763 * Update last_act too so that poll()/POLLPRI can wait for the next 5764 * event after any syscall on hist file. 5765 */ 5766 hist_file->last_act = hist_file->last_read; 5767 5768 return 0; 5769 } 5770 5771 static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wait) 5772 { 5773 struct trace_event_file *event_file; 5774 struct seq_file *m = file->private_data; 5775 struct hist_file_data *hist_file = m->private; 5776 __poll_t ret = 0; 5777 u64 cnt; 5778 5779 guard(mutex)(&event_mutex); 5780 5781 event_file = event_file_data(file); 5782 if (!event_file) 5783 return EPOLLERR; 5784 5785 hist_poll_wait(file, wait); 5786 5787 cnt = get_hist_hit_count(event_file); 5788 if (hist_file->last_read != cnt) 5789 ret |= EPOLLIN | EPOLLRDNORM; 5790 if (hist_file->last_act != cnt) { 5791 hist_file->last_act = cnt; 5792 ret |= EPOLLPRI; 5793 } 5794 5795 return ret; 5796 } 5797 5798 static int event_hist_release(struct inode *inode, struct file *file) 5799 { 5800 struct seq_file *m = file->private_data; 5801 struct hist_file_data *hist_file = m->private; 5802 5803 kfree(hist_file); 5804 return tracing_single_release_file_tr(inode, file); 5805 } 5806 5807 static int event_hist_open(struct inode *inode, struct file *file) 5808 { 5809 struct trace_event_file *event_file; 5810 struct hist_file_data *hist_file; 5811 int ret; 5812 5813 ret = tracing_open_file_tr(inode, file); 5814 if (ret) 5815 return ret; 5816 5817 guard(mutex)(&event_mutex); 5818 5819 event_file = event_file_data(file); 5820 if (!event_file) { 5821 ret = -ENODEV; 5822 goto err; 5823 } 5824 5825 hist_file = kzalloc(sizeof(*hist_file), GFP_KERNEL); 5826 if (!hist_file) { 5827 ret = -ENOMEM; 5828 goto err; 5829 } 5830 5831 hist_file->file = file; 5832 hist_file->last_act = get_hist_hit_count(event_file); 5833 5834 /* Clear private_data to avoid warning in single_open() */ 5835 file->private_data = NULL; 5836 ret = single_open(file, hist_show, hist_file); 5837 if (ret) { 5838 kfree(hist_file); 5839 goto err; 5840 } 5841 5842 return 0; 5843 err: 5844 tracing_release_file_tr(inode, file); 5845 return ret; 5846 } 5847 5848 const struct file_operations event_hist_fops = { 5849 .open = event_hist_open, 5850 .read = seq_read, 5851 .llseek = seq_lseek, 5852 .release = event_hist_release, 5853 .poll = event_hist_poll, 5854 }; 5855 5856 #ifdef CONFIG_HIST_TRIGGERS_DEBUG 5857 static void hist_field_debug_show_flags(struct seq_file *m, 5858 unsigned long flags) 5859 { 5860 seq_puts(m, " flags:\n"); 5861 5862 if (flags & HIST_FIELD_FL_KEY) 5863 seq_puts(m, " HIST_FIELD_FL_KEY\n"); 5864 else if (flags & HIST_FIELD_FL_HITCOUNT) 5865 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n"); 5866 else if (flags & HIST_FIELD_FL_VAR) 5867 seq_puts(m, " HIST_FIELD_FL_VAR\n"); 5868 else if (flags & HIST_FIELD_FL_VAR_REF) 5869 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n"); 5870 else 5871 seq_puts(m, " VAL: normal u64 value\n"); 5872 5873 if (flags & HIST_FIELD_FL_ALIAS) 5874 seq_puts(m, " HIST_FIELD_FL_ALIAS\n"); 5875 else if (flags & HIST_FIELD_FL_CONST) 5876 seq_puts(m, " HIST_FIELD_FL_CONST\n"); 5877 } 5878 5879 static int hist_field_debug_show(struct seq_file *m, 5880 struct hist_field *field, unsigned long flags) 5881 { 5882 if ((field->flags & flags) != flags) { 5883 seq_printf(m, "ERROR: bad flags - %lx\n", flags); 5884 return -EINVAL; 5885 } 5886 5887 hist_field_debug_show_flags(m, field->flags); 5888 if (field->field) 5889 seq_printf(m, " ftrace_event_field name: %s\n", 5890 field->field->name); 5891 5892 if (field->flags & HIST_FIELD_FL_VAR) { 5893 seq_printf(m, " var.name: %s\n", field->var.name); 5894 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5895 field->var.idx); 5896 } 5897 5898 if (field->flags & HIST_FIELD_FL_CONST) 5899 seq_printf(m, " constant: %llu\n", field->constant); 5900 5901 if (field->flags & HIST_FIELD_FL_ALIAS) 5902 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5903 field->var_ref_idx); 5904 5905 if (field->flags & HIST_FIELD_FL_VAR_REF) { 5906 seq_printf(m, " name: %s\n", field->name); 5907 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5908 field->var.idx); 5909 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data); 5910 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n", 5911 field->var_ref_idx); 5912 if (field->system) 5913 seq_printf(m, " system: %s\n", field->system); 5914 if (field->event_name) 5915 seq_printf(m, " event_name: %s\n", field->event_name); 5916 } 5917 5918 seq_printf(m, " type: %s\n", field->type); 5919 seq_printf(m, " size: %u\n", field->size); 5920 seq_printf(m, " is_signed: %u\n", field->is_signed); 5921 5922 return 0; 5923 } 5924 5925 static int field_var_debug_show(struct seq_file *m, 5926 struct field_var *field_var, unsigned int i, 5927 bool save_vars) 5928 { 5929 const char *vars_name = save_vars ? "save_vars" : "field_vars"; 5930 struct hist_field *field; 5931 int ret = 0; 5932 5933 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i); 5934 5935 field = field_var->var; 5936 5937 seq_printf(m, "\n %s[%d].var:\n", vars_name, i); 5938 5939 hist_field_debug_show_flags(m, field->flags); 5940 seq_printf(m, " var.name: %s\n", field->var.name); 5941 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n", 5942 field->var.idx); 5943 5944 field = field_var->val; 5945 5946 seq_printf(m, "\n %s[%d].val:\n", vars_name, i); 5947 if (field->field) 5948 seq_printf(m, " ftrace_event_field name: %s\n", 5949 field->field->name); 5950 else { 5951 ret = -EINVAL; 5952 goto out; 5953 } 5954 5955 seq_printf(m, " type: %s\n", field->type); 5956 seq_printf(m, " size: %u\n", field->size); 5957 seq_printf(m, " is_signed: %u\n", field->is_signed); 5958 out: 5959 return ret; 5960 } 5961 5962 static int hist_action_debug_show(struct seq_file *m, 5963 struct action_data *data, int i) 5964 { 5965 int ret = 0; 5966 5967 if (data->handler == HANDLER_ONMAX || 5968 data->handler == HANDLER_ONCHANGE) { 5969 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i); 5970 ret = hist_field_debug_show(m, data->track_data.var_ref, 5971 HIST_FIELD_FL_VAR_REF); 5972 if (ret) 5973 goto out; 5974 5975 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i); 5976 ret = hist_field_debug_show(m, data->track_data.track_var, 5977 HIST_FIELD_FL_VAR); 5978 if (ret) 5979 goto out; 5980 } 5981 5982 if (data->handler == HANDLER_ONMATCH) { 5983 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n", 5984 i, data->match_data.event_system); 5985 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n", 5986 i, data->match_data.event); 5987 } 5988 out: 5989 return ret; 5990 } 5991 5992 static int hist_actions_debug_show(struct seq_file *m, 5993 struct hist_trigger_data *hist_data) 5994 { 5995 int i, ret = 0; 5996 5997 if (hist_data->n_actions) 5998 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n"); 5999 6000 for (i = 0; i < hist_data->n_actions; i++) { 6001 struct action_data *action = hist_data->actions[i]; 6002 6003 ret = hist_action_debug_show(m, action, i); 6004 if (ret) 6005 goto out; 6006 } 6007 6008 if (hist_data->n_save_vars) 6009 seq_puts(m, "\n save action variables (save() params):\n"); 6010 6011 for (i = 0; i < hist_data->n_save_vars; i++) { 6012 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true); 6013 if (ret) 6014 goto out; 6015 } 6016 out: 6017 return ret; 6018 } 6019 6020 static void hist_trigger_debug_show(struct seq_file *m, 6021 struct event_trigger_data *data, int n) 6022 { 6023 struct hist_trigger_data *hist_data; 6024 int i, ret; 6025 6026 if (n > 0) 6027 seq_puts(m, "\n\n"); 6028 6029 seq_puts(m, "# event histogram\n#\n# trigger info: "); 6030 data->cmd_ops->print(m, data); 6031 seq_puts(m, "#\n\n"); 6032 6033 hist_data = data->private_data; 6034 6035 seq_printf(m, "hist_data: %p\n\n", hist_data); 6036 seq_printf(m, " n_vals: %u\n", hist_data->n_vals); 6037 seq_printf(m, " n_keys: %u\n", hist_data->n_keys); 6038 seq_printf(m, " n_fields: %u\n", hist_data->n_fields); 6039 6040 seq_puts(m, "\n val fields:\n\n"); 6041 6042 seq_puts(m, " hist_data->fields[0]:\n"); 6043 ret = hist_field_debug_show(m, hist_data->fields[0], 6044 HIST_FIELD_FL_HITCOUNT); 6045 if (ret) 6046 return; 6047 6048 for (i = 1; i < hist_data->n_vals; i++) { 6049 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6050 ret = hist_field_debug_show(m, hist_data->fields[i], 0); 6051 if (ret) 6052 return; 6053 } 6054 6055 seq_puts(m, "\n key fields:\n"); 6056 6057 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) { 6058 seq_printf(m, "\n hist_data->fields[%d]:\n", i); 6059 ret = hist_field_debug_show(m, hist_data->fields[i], 6060 HIST_FIELD_FL_KEY); 6061 if (ret) 6062 return; 6063 } 6064 6065 if (hist_data->n_var_refs) 6066 seq_puts(m, "\n variable reference fields:\n"); 6067 6068 for (i = 0; i < hist_data->n_var_refs; i++) { 6069 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i); 6070 ret = hist_field_debug_show(m, hist_data->var_refs[i], 6071 HIST_FIELD_FL_VAR_REF); 6072 if (ret) 6073 return; 6074 } 6075 6076 if (hist_data->n_field_vars) 6077 seq_puts(m, "\n field variables:\n"); 6078 6079 for (i = 0; i < hist_data->n_field_vars; i++) { 6080 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false); 6081 if (ret) 6082 return; 6083 } 6084 6085 ret = hist_actions_debug_show(m, hist_data); 6086 if (ret) 6087 return; 6088 } 6089 6090 static int hist_debug_show(struct seq_file *m, void *v) 6091 { 6092 struct event_trigger_data *data; 6093 struct trace_event_file *event_file; 6094 int n = 0; 6095 6096 guard(mutex)(&event_mutex); 6097 6098 event_file = event_file_file(m->private); 6099 if (unlikely(!event_file)) 6100 return -ENODEV; 6101 6102 list_for_each_entry(data, &event_file->triggers, list) { 6103 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) 6104 hist_trigger_debug_show(m, data, n++); 6105 } 6106 return 0; 6107 } 6108 6109 static int event_hist_debug_open(struct inode *inode, struct file *file) 6110 { 6111 int ret; 6112 6113 ret = tracing_open_file_tr(inode, file); 6114 if (ret) 6115 return ret; 6116 6117 /* Clear private_data to avoid warning in single_open() */ 6118 file->private_data = NULL; 6119 ret = single_open(file, hist_debug_show, file); 6120 if (ret) 6121 tracing_release_file_tr(inode, file); 6122 return ret; 6123 } 6124 6125 const struct file_operations event_hist_debug_fops = { 6126 .open = event_hist_debug_open, 6127 .read = seq_read, 6128 .llseek = seq_lseek, 6129 .release = tracing_single_release_file_tr, 6130 }; 6131 #endif 6132 6133 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field) 6134 { 6135 const char *field_name = hist_field_name(hist_field, 0); 6136 6137 if (hist_field->var.name) 6138 seq_printf(m, "%s=", hist_field->var.name); 6139 6140 if (hist_field->flags & HIST_FIELD_FL_CPU) 6141 seq_puts(m, "common_cpu"); 6142 if (hist_field->flags & HIST_FIELD_FL_COMM) 6143 seq_puts(m, "common_comm"); 6144 else if (hist_field->flags & HIST_FIELD_FL_CONST) 6145 seq_printf(m, "%llu", hist_field->constant); 6146 else if (field_name) { 6147 if (hist_field->flags & HIST_FIELD_FL_VAR_REF || 6148 hist_field->flags & HIST_FIELD_FL_ALIAS) 6149 seq_putc(m, '$'); 6150 seq_printf(m, "%s", field_name); 6151 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP) 6152 seq_puts(m, "common_timestamp"); 6153 6154 if (hist_field->flags) { 6155 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) && 6156 !(hist_field->flags & HIST_FIELD_FL_EXPR) && 6157 !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) { 6158 const char *flags = get_hist_field_flags(hist_field); 6159 6160 if (flags) 6161 seq_printf(m, ".%s", flags); 6162 } 6163 } 6164 if (hist_field->buckets) 6165 seq_printf(m, "=%ld", hist_field->buckets); 6166 } 6167 6168 static int event_hist_trigger_print(struct seq_file *m, 6169 struct event_trigger_data *data) 6170 { 6171 struct hist_trigger_data *hist_data = data->private_data; 6172 struct hist_field *field; 6173 bool have_var = false; 6174 bool show_val = false; 6175 unsigned int i; 6176 6177 seq_puts(m, HIST_PREFIX); 6178 6179 if (data->name) 6180 seq_printf(m, "%s:", data->name); 6181 6182 seq_puts(m, "keys="); 6183 6184 for_each_hist_key_field(i, hist_data) { 6185 field = hist_data->fields[i]; 6186 6187 if (i > hist_data->n_vals) 6188 seq_puts(m, ","); 6189 6190 if (field->flags & HIST_FIELD_FL_STACKTRACE) { 6191 if (field->field) 6192 seq_printf(m, "%s.stacktrace", field->field->name); 6193 else 6194 seq_puts(m, "common_stacktrace"); 6195 } else 6196 hist_field_print(m, field); 6197 } 6198 6199 seq_puts(m, ":vals="); 6200 6201 for_each_hist_val_field(i, hist_data) { 6202 field = hist_data->fields[i]; 6203 if (field->flags & HIST_FIELD_FL_VAR) { 6204 have_var = true; 6205 continue; 6206 } 6207 6208 if (i == HITCOUNT_IDX) { 6209 if (hist_data->attrs->no_hitcount) 6210 continue; 6211 seq_puts(m, "hitcount"); 6212 } else { 6213 if (show_val) 6214 seq_puts(m, ","); 6215 hist_field_print(m, field); 6216 } 6217 show_val = true; 6218 } 6219 6220 if (have_var) { 6221 unsigned int n = 0; 6222 6223 seq_puts(m, ":"); 6224 6225 for_each_hist_val_field(i, hist_data) { 6226 field = hist_data->fields[i]; 6227 6228 if (field->flags & HIST_FIELD_FL_VAR) { 6229 if (n++) 6230 seq_puts(m, ","); 6231 hist_field_print(m, field); 6232 } 6233 } 6234 } 6235 6236 seq_puts(m, ":sort="); 6237 6238 for (i = 0; i < hist_data->n_sort_keys; i++) { 6239 struct tracing_map_sort_key *sort_key; 6240 unsigned int idx, first_key_idx; 6241 6242 /* skip VAR vals */ 6243 first_key_idx = hist_data->n_vals - hist_data->n_vars; 6244 6245 sort_key = &hist_data->sort_keys[i]; 6246 idx = sort_key->field_idx; 6247 6248 if (WARN_ON(idx >= HIST_FIELDS_MAX)) 6249 return -EINVAL; 6250 6251 if (i > 0) 6252 seq_puts(m, ","); 6253 6254 if (idx == HITCOUNT_IDX) 6255 seq_puts(m, "hitcount"); 6256 else { 6257 if (idx >= first_key_idx) 6258 idx += hist_data->n_vars; 6259 hist_field_print(m, hist_data->fields[idx]); 6260 } 6261 6262 if (sort_key->descending) 6263 seq_puts(m, ".descending"); 6264 } 6265 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits)); 6266 if (hist_data->enable_timestamps) 6267 seq_printf(m, ":clock=%s", hist_data->attrs->clock); 6268 if (hist_data->attrs->no_hitcount) 6269 seq_puts(m, ":nohitcount"); 6270 6271 print_actions_spec(m, hist_data); 6272 6273 if (data->filter_str) 6274 seq_printf(m, " if %s", data->filter_str); 6275 6276 if (data->paused) 6277 seq_puts(m, " [paused]"); 6278 else 6279 seq_puts(m, " [active]"); 6280 6281 seq_putc(m, '\n'); 6282 6283 return 0; 6284 } 6285 6286 static int event_hist_trigger_init(struct event_trigger_data *data) 6287 { 6288 struct hist_trigger_data *hist_data = data->private_data; 6289 6290 if (alloc_hist_pad() < 0) 6291 return -ENOMEM; 6292 6293 if (!data->ref && hist_data->attrs->name) 6294 save_named_trigger(hist_data->attrs->name, data); 6295 6296 data->ref++; 6297 6298 return 0; 6299 } 6300 6301 static void unregister_field_var_hists(struct hist_trigger_data *hist_data) 6302 { 6303 struct trace_event_file *file; 6304 unsigned int i; 6305 char *cmd; 6306 int ret; 6307 6308 for (i = 0; i < hist_data->n_field_var_hists; i++) { 6309 file = hist_data->field_var_hists[i]->hist_data->event_file; 6310 cmd = hist_data->field_var_hists[i]->cmd; 6311 ret = event_hist_trigger_parse(&trigger_hist_cmd, file, 6312 "!hist", "hist", cmd); 6313 WARN_ON_ONCE(ret < 0); 6314 } 6315 } 6316 6317 static void event_hist_trigger_free(struct event_trigger_data *data) 6318 { 6319 struct hist_trigger_data *hist_data = data->private_data; 6320 6321 if (WARN_ON_ONCE(data->ref <= 0)) 6322 return; 6323 6324 data->ref--; 6325 if (!data->ref) { 6326 if (data->name) 6327 del_named_trigger(data); 6328 6329 trigger_data_free(data); 6330 6331 remove_hist_vars(hist_data); 6332 6333 unregister_field_var_hists(hist_data); 6334 6335 destroy_hist_data(hist_data); 6336 } 6337 free_hist_pad(); 6338 } 6339 6340 static int event_hist_trigger_named_init(struct event_trigger_data *data) 6341 { 6342 int ret; 6343 6344 data->ref++; 6345 6346 save_named_trigger(data->named_data->name, data); 6347 6348 ret = event_hist_trigger_init(data->named_data); 6349 if (ret < 0) { 6350 kfree(data->cmd_ops); 6351 data->cmd_ops = &trigger_hist_cmd; 6352 } 6353 6354 return ret; 6355 } 6356 6357 static void event_hist_trigger_named_free(struct event_trigger_data *data) 6358 { 6359 if (WARN_ON_ONCE(data->ref <= 0)) 6360 return; 6361 6362 event_hist_trigger_free(data->named_data); 6363 6364 data->ref--; 6365 if (!data->ref) { 6366 struct event_command *cmd_ops = data->cmd_ops; 6367 6368 del_named_trigger(data); 6369 trigger_data_free(data); 6370 kfree(cmd_ops); 6371 } 6372 } 6373 6374 static void hist_clear(struct event_trigger_data *data) 6375 { 6376 struct hist_trigger_data *hist_data = data->private_data; 6377 6378 if (data->name) 6379 pause_named_trigger(data); 6380 6381 tracepoint_synchronize_unregister(); 6382 6383 tracing_map_clear(hist_data->map); 6384 6385 if (data->name) 6386 unpause_named_trigger(data); 6387 } 6388 6389 static bool compatible_field(struct ftrace_event_field *field, 6390 struct ftrace_event_field *test_field) 6391 { 6392 if (field == test_field) 6393 return true; 6394 if (field == NULL || test_field == NULL) 6395 return false; 6396 if (strcmp(field->name, test_field->name) != 0) 6397 return false; 6398 if (strcmp(field->type, test_field->type) != 0) 6399 return false; 6400 if (field->size != test_field->size) 6401 return false; 6402 if (field->is_signed != test_field->is_signed) 6403 return false; 6404 6405 return true; 6406 } 6407 6408 static bool hist_trigger_match(struct event_trigger_data *data, 6409 struct event_trigger_data *data_test, 6410 struct event_trigger_data *named_data, 6411 bool ignore_filter) 6412 { 6413 struct tracing_map_sort_key *sort_key, *sort_key_test; 6414 struct hist_trigger_data *hist_data, *hist_data_test; 6415 struct hist_field *key_field, *key_field_test; 6416 unsigned int i; 6417 6418 if (named_data && (named_data != data_test) && 6419 (named_data != data_test->named_data)) 6420 return false; 6421 6422 if (!named_data && is_named_trigger(data_test)) 6423 return false; 6424 6425 hist_data = data->private_data; 6426 hist_data_test = data_test->private_data; 6427 6428 if (hist_data->n_vals != hist_data_test->n_vals || 6429 hist_data->n_fields != hist_data_test->n_fields || 6430 hist_data->n_sort_keys != hist_data_test->n_sort_keys) 6431 return false; 6432 6433 if (!ignore_filter) { 6434 if ((data->filter_str && !data_test->filter_str) || 6435 (!data->filter_str && data_test->filter_str)) 6436 return false; 6437 } 6438 6439 for_each_hist_field(i, hist_data) { 6440 key_field = hist_data->fields[i]; 6441 key_field_test = hist_data_test->fields[i]; 6442 6443 if (key_field->flags != key_field_test->flags) 6444 return false; 6445 if (!compatible_field(key_field->field, key_field_test->field)) 6446 return false; 6447 if (key_field->offset != key_field_test->offset) 6448 return false; 6449 if (key_field->size != key_field_test->size) 6450 return false; 6451 if (key_field->is_signed != key_field_test->is_signed) 6452 return false; 6453 if (!!key_field->var.name != !!key_field_test->var.name) 6454 return false; 6455 if (key_field->var.name && 6456 strcmp(key_field->var.name, key_field_test->var.name) != 0) 6457 return false; 6458 } 6459 6460 for (i = 0; i < hist_data->n_sort_keys; i++) { 6461 sort_key = &hist_data->sort_keys[i]; 6462 sort_key_test = &hist_data_test->sort_keys[i]; 6463 6464 if (sort_key->field_idx != sort_key_test->field_idx || 6465 sort_key->descending != sort_key_test->descending) 6466 return false; 6467 } 6468 6469 if (!ignore_filter && data->filter_str && 6470 (strcmp(data->filter_str, data_test->filter_str) != 0)) 6471 return false; 6472 6473 if (!actions_match(hist_data, hist_data_test)) 6474 return false; 6475 6476 return true; 6477 } 6478 6479 static bool existing_hist_update_only(char *glob, 6480 struct event_trigger_data *data, 6481 struct trace_event_file *file) 6482 { 6483 struct hist_trigger_data *hist_data = data->private_data; 6484 struct event_trigger_data *test, *named_data = NULL; 6485 bool updated = false; 6486 6487 if (!hist_data->attrs->pause && !hist_data->attrs->cont && 6488 !hist_data->attrs->clear) 6489 goto out; 6490 6491 if (hist_data->attrs->name) { 6492 named_data = find_named_trigger(hist_data->attrs->name); 6493 if (named_data) { 6494 if (!hist_trigger_match(data, named_data, named_data, 6495 true)) 6496 goto out; 6497 } 6498 } 6499 6500 if (hist_data->attrs->name && !named_data) 6501 goto out; 6502 6503 list_for_each_entry(test, &file->triggers, list) { 6504 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6505 if (!hist_trigger_match(data, test, named_data, false)) 6506 continue; 6507 if (hist_data->attrs->pause) 6508 test->paused = true; 6509 else if (hist_data->attrs->cont) 6510 test->paused = false; 6511 else if (hist_data->attrs->clear) 6512 hist_clear(test); 6513 updated = true; 6514 goto out; 6515 } 6516 } 6517 out: 6518 return updated; 6519 } 6520 6521 static int hist_register_trigger(char *glob, 6522 struct event_trigger_data *data, 6523 struct trace_event_file *file) 6524 { 6525 struct hist_trigger_data *hist_data = data->private_data; 6526 struct event_trigger_data *test, *named_data = NULL; 6527 struct trace_array *tr = file->tr; 6528 int ret = 0; 6529 6530 if (hist_data->attrs->name) { 6531 named_data = find_named_trigger(hist_data->attrs->name); 6532 if (named_data) { 6533 if (!hist_trigger_match(data, named_data, named_data, 6534 true)) { 6535 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name)); 6536 ret = -EINVAL; 6537 goto out; 6538 } 6539 } 6540 } 6541 6542 if (hist_data->attrs->name && !named_data) 6543 goto new; 6544 6545 lockdep_assert_held(&event_mutex); 6546 6547 list_for_each_entry(test, &file->triggers, list) { 6548 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6549 if (hist_trigger_match(data, test, named_data, false)) { 6550 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0); 6551 ret = -EEXIST; 6552 goto out; 6553 } 6554 } 6555 } 6556 new: 6557 if (hist_data->attrs->cont || hist_data->attrs->clear) { 6558 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0); 6559 ret = -ENOENT; 6560 goto out; 6561 } 6562 6563 if (hist_data->attrs->pause) 6564 data->paused = true; 6565 6566 if (named_data) { 6567 struct event_command *cmd_ops; 6568 6569 data->private_data = named_data->private_data; 6570 set_named_trigger_data(data, named_data); 6571 /* Copy the command ops and update some of the functions */ 6572 cmd_ops = kmalloc(sizeof(*cmd_ops), GFP_KERNEL); 6573 if (!cmd_ops) { 6574 ret = -ENOMEM; 6575 goto out; 6576 } 6577 *cmd_ops = *data->cmd_ops; 6578 cmd_ops->init = event_hist_trigger_named_init; 6579 cmd_ops->free = event_hist_trigger_named_free; 6580 data->cmd_ops = cmd_ops; 6581 } 6582 6583 if (data->cmd_ops->init) { 6584 ret = data->cmd_ops->init(data); 6585 if (ret < 0) 6586 goto out; 6587 } 6588 6589 if (hist_data->enable_timestamps) { 6590 char *clock = hist_data->attrs->clock; 6591 6592 ret = tracing_set_clock(file->tr, hist_data->attrs->clock); 6593 if (ret) { 6594 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock)); 6595 goto out; 6596 } 6597 6598 tracing_set_filter_buffering(file->tr, true); 6599 } 6600 6601 if (named_data) 6602 destroy_hist_data(hist_data); 6603 out: 6604 return ret; 6605 } 6606 6607 static int hist_trigger_enable(struct event_trigger_data *data, 6608 struct trace_event_file *file) 6609 { 6610 int ret = 0; 6611 6612 list_add_tail_rcu(&data->list, &file->triggers); 6613 6614 update_cond_flag(file); 6615 6616 if (trace_event_trigger_enable_disable(file, 1) < 0) { 6617 list_del_rcu(&data->list); 6618 update_cond_flag(file); 6619 ret--; 6620 } 6621 6622 return ret; 6623 } 6624 6625 static bool have_hist_trigger_match(struct event_trigger_data *data, 6626 struct trace_event_file *file) 6627 { 6628 struct hist_trigger_data *hist_data = data->private_data; 6629 struct event_trigger_data *test, *named_data = NULL; 6630 bool match = false; 6631 6632 lockdep_assert_held(&event_mutex); 6633 6634 if (hist_data->attrs->name) 6635 named_data = find_named_trigger(hist_data->attrs->name); 6636 6637 list_for_each_entry(test, &file->triggers, list) { 6638 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6639 if (hist_trigger_match(data, test, named_data, false)) { 6640 match = true; 6641 break; 6642 } 6643 } 6644 } 6645 6646 return match; 6647 } 6648 6649 static bool hist_trigger_check_refs(struct event_trigger_data *data, 6650 struct trace_event_file *file) 6651 { 6652 struct hist_trigger_data *hist_data = data->private_data; 6653 struct event_trigger_data *test, *named_data = NULL; 6654 6655 lockdep_assert_held(&event_mutex); 6656 6657 if (hist_data->attrs->name) 6658 named_data = find_named_trigger(hist_data->attrs->name); 6659 6660 list_for_each_entry(test, &file->triggers, list) { 6661 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6662 if (!hist_trigger_match(data, test, named_data, false)) 6663 continue; 6664 hist_data = test->private_data; 6665 if (check_var_refs(hist_data)) 6666 return true; 6667 break; 6668 } 6669 } 6670 6671 return false; 6672 } 6673 6674 static void hist_unregister_trigger(char *glob, 6675 struct event_trigger_data *data, 6676 struct trace_event_file *file) 6677 { 6678 struct event_trigger_data *test = NULL, *iter, *named_data = NULL; 6679 struct hist_trigger_data *hist_data = data->private_data; 6680 6681 lockdep_assert_held(&event_mutex); 6682 6683 if (hist_data->attrs->name) 6684 named_data = find_named_trigger(hist_data->attrs->name); 6685 6686 list_for_each_entry(iter, &file->triggers, list) { 6687 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6688 if (!hist_trigger_match(data, iter, named_data, false)) 6689 continue; 6690 test = iter; 6691 list_del_rcu(&test->list); 6692 trace_event_trigger_enable_disable(file, 0); 6693 update_cond_flag(file); 6694 break; 6695 } 6696 } 6697 6698 if (test && test->cmd_ops->free) 6699 test->cmd_ops->free(test); 6700 6701 if (hist_data->enable_timestamps) { 6702 if (!hist_data->remove || test) 6703 tracing_set_filter_buffering(file->tr, false); 6704 } 6705 } 6706 6707 static bool hist_file_check_refs(struct trace_event_file *file) 6708 { 6709 struct hist_trigger_data *hist_data; 6710 struct event_trigger_data *test; 6711 6712 lockdep_assert_held(&event_mutex); 6713 6714 list_for_each_entry(test, &file->triggers, list) { 6715 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6716 hist_data = test->private_data; 6717 if (check_var_refs(hist_data)) 6718 return true; 6719 } 6720 } 6721 6722 return false; 6723 } 6724 6725 static void hist_unreg_all(struct trace_event_file *file) 6726 { 6727 struct event_trigger_data *test, *n; 6728 struct hist_trigger_data *hist_data; 6729 struct synth_event *se; 6730 const char *se_name; 6731 6732 lockdep_assert_held(&event_mutex); 6733 6734 if (hist_file_check_refs(file)) 6735 return; 6736 6737 list_for_each_entry_safe(test, n, &file->triggers, list) { 6738 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6739 hist_data = test->private_data; 6740 list_del_rcu(&test->list); 6741 trace_event_trigger_enable_disable(file, 0); 6742 6743 se_name = trace_event_name(file->event_call); 6744 se = find_synth_event(se_name); 6745 if (se) 6746 se->ref--; 6747 6748 update_cond_flag(file); 6749 if (hist_data->enable_timestamps) 6750 tracing_set_filter_buffering(file->tr, false); 6751 if (test->cmd_ops->free) 6752 test->cmd_ops->free(test); 6753 } 6754 } 6755 } 6756 6757 static int event_hist_trigger_parse(struct event_command *cmd_ops, 6758 struct trace_event_file *file, 6759 char *glob, char *cmd, 6760 char *param_and_filter) 6761 { 6762 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT; 6763 struct event_trigger_data *trigger_data; 6764 struct hist_trigger_attrs *attrs; 6765 struct hist_trigger_data *hist_data; 6766 char *param, *filter, *p, *start; 6767 struct synth_event *se; 6768 const char *se_name; 6769 bool remove; 6770 int ret = 0; 6771 6772 lockdep_assert_held(&event_mutex); 6773 6774 if (WARN_ON(!glob)) 6775 return -EINVAL; 6776 6777 if (glob[0]) { 6778 hist_err_clear(); 6779 last_cmd_set(file, param_and_filter); 6780 } 6781 6782 remove = event_trigger_check_remove(glob); 6783 6784 if (event_trigger_empty_param(param_and_filter)) 6785 return -EINVAL; 6786 6787 /* 6788 * separate the trigger from the filter (k:v [if filter]) 6789 * allowing for whitespace in the trigger 6790 */ 6791 p = param = param_and_filter; 6792 do { 6793 p = strstr(p, "if"); 6794 if (!p) 6795 break; 6796 if (p == param_and_filter) 6797 return -EINVAL; 6798 if (*(p - 1) != ' ' && *(p - 1) != '\t') { 6799 p++; 6800 continue; 6801 } 6802 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1) 6803 return -EINVAL; 6804 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') { 6805 p++; 6806 continue; 6807 } 6808 break; 6809 } while (1); 6810 6811 if (!p) 6812 filter = NULL; 6813 else { 6814 *(p - 1) = '\0'; 6815 filter = strstrip(p); 6816 param = strstrip(param); 6817 } 6818 6819 /* 6820 * To simplify arithmetic expression parsing, replace occurrences of 6821 * '.sym-offset' modifier with '.symXoffset' 6822 */ 6823 start = strstr(param, ".sym-offset"); 6824 while (start) { 6825 *(start + 4) = 'X'; 6826 start = strstr(start + 11, ".sym-offset"); 6827 } 6828 6829 attrs = parse_hist_trigger_attrs(file->tr, param); 6830 if (IS_ERR(attrs)) 6831 return PTR_ERR(attrs); 6832 6833 if (attrs->map_bits) 6834 hist_trigger_bits = attrs->map_bits; 6835 6836 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove); 6837 if (IS_ERR(hist_data)) { 6838 destroy_hist_trigger_attrs(attrs); 6839 return PTR_ERR(hist_data); 6840 } 6841 6842 trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data); 6843 if (!trigger_data) { 6844 ret = -ENOMEM; 6845 goto out_free; 6846 } 6847 6848 ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data); 6849 if (ret < 0) 6850 goto out_free; 6851 6852 if (remove) { 6853 if (!have_hist_trigger_match(trigger_data, file)) 6854 goto out_free; 6855 6856 if (hist_trigger_check_refs(trigger_data, file)) { 6857 ret = -EBUSY; 6858 goto out_free; 6859 } 6860 6861 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6862 se_name = trace_event_name(file->event_call); 6863 se = find_synth_event(se_name); 6864 if (se) 6865 se->ref--; 6866 ret = 0; 6867 goto out_free; 6868 } 6869 6870 if (existing_hist_update_only(glob, trigger_data, file)) 6871 goto out_free; 6872 6873 if (!get_named_trigger_data(trigger_data)) { 6874 6875 ret = create_actions(hist_data); 6876 if (ret) 6877 goto out_free; 6878 6879 if (has_hist_vars(hist_data) || hist_data->n_var_refs) { 6880 ret = save_hist_vars(hist_data); 6881 if (ret) 6882 goto out_free; 6883 } 6884 6885 ret = tracing_map_init(hist_data->map); 6886 if (ret) 6887 goto out_free; 6888 } 6889 6890 ret = event_trigger_register(cmd_ops, file, glob, trigger_data); 6891 if (ret < 0) 6892 goto out_free; 6893 6894 ret = hist_trigger_enable(trigger_data, file); 6895 if (ret) 6896 goto out_unreg; 6897 6898 se_name = trace_event_name(file->event_call); 6899 se = find_synth_event(se_name); 6900 if (se) 6901 se->ref++; 6902 out: 6903 if (ret == 0 && glob[0]) 6904 hist_err_clear(); 6905 6906 return ret; 6907 out_unreg: 6908 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); 6909 out_free: 6910 event_trigger_reset_filter(cmd_ops, trigger_data); 6911 6912 remove_hist_vars(hist_data); 6913 6914 kfree(trigger_data); 6915 6916 destroy_hist_data(hist_data); 6917 goto out; 6918 } 6919 6920 static struct event_command trigger_hist_cmd = { 6921 .name = "hist", 6922 .trigger_type = ETT_EVENT_HIST, 6923 .flags = EVENT_CMD_FL_NEEDS_REC, 6924 .parse = event_hist_trigger_parse, 6925 .reg = hist_register_trigger, 6926 .unreg = hist_unregister_trigger, 6927 .unreg_all = hist_unreg_all, 6928 .set_filter = set_trigger_filter, 6929 .trigger = event_hist_trigger, 6930 .print = event_hist_trigger_print, 6931 .init = event_hist_trigger_init, 6932 .free = event_hist_trigger_free, 6933 }; 6934 6935 __init int register_trigger_hist_cmd(void) 6936 { 6937 int ret; 6938 6939 ret = register_event_command(&trigger_hist_cmd); 6940 WARN_ON(ret < 0); 6941 6942 return ret; 6943 } 6944 6945 static void 6946 hist_enable_trigger(struct event_trigger_data *data, 6947 struct trace_buffer *buffer, void *rec, 6948 struct ring_buffer_event *event) 6949 { 6950 struct enable_trigger_data *enable_data = data->private_data; 6951 struct event_trigger_data *test; 6952 6953 list_for_each_entry_rcu(test, &enable_data->file->triggers, list, 6954 lockdep_is_held(&event_mutex)) { 6955 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) { 6956 if (enable_data->enable) 6957 test->paused = false; 6958 else 6959 test->paused = true; 6960 } 6961 } 6962 } 6963 6964 static void hist_enable_unreg_all(struct trace_event_file *file) 6965 { 6966 struct event_trigger_data *test, *n; 6967 6968 list_for_each_entry_safe(test, n, &file->triggers, list) { 6969 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) { 6970 list_del_rcu(&test->list); 6971 update_cond_flag(file); 6972 trace_event_trigger_enable_disable(file, 0); 6973 if (test->cmd_ops->free) 6974 test->cmd_ops->free(test); 6975 } 6976 } 6977 } 6978 6979 static struct event_command trigger_hist_enable_cmd = { 6980 .name = ENABLE_HIST_STR, 6981 .trigger_type = ETT_HIST_ENABLE, 6982 .parse = event_enable_trigger_parse, 6983 .reg = event_enable_register_trigger, 6984 .unreg = event_enable_unregister_trigger, 6985 .unreg_all = hist_enable_unreg_all, 6986 .set_filter = set_trigger_filter, 6987 .trigger = hist_enable_trigger, 6988 .count_func = event_trigger_count, 6989 .print = event_enable_trigger_print, 6990 .init = event_trigger_init, 6991 .free = event_enable_trigger_free, 6992 }; 6993 6994 static struct event_command trigger_hist_disable_cmd = { 6995 .name = DISABLE_HIST_STR, 6996 .trigger_type = ETT_HIST_ENABLE, 6997 .parse = event_enable_trigger_parse, 6998 .reg = event_enable_register_trigger, 6999 .unreg = event_enable_unregister_trigger, 7000 .unreg_all = hist_enable_unreg_all, 7001 .set_filter = set_trigger_filter, 7002 .trigger = hist_enable_trigger, 7003 .count_func = event_trigger_count, 7004 .print = event_enable_trigger_print, 7005 .init = event_trigger_init, 7006 .free = event_enable_trigger_free, 7007 }; 7008 7009 static __init void unregister_trigger_hist_enable_disable_cmds(void) 7010 { 7011 unregister_event_command(&trigger_hist_enable_cmd); 7012 unregister_event_command(&trigger_hist_disable_cmd); 7013 } 7014 7015 __init int register_trigger_hist_enable_disable_cmds(void) 7016 { 7017 int ret; 7018 7019 ret = register_event_command(&trigger_hist_enable_cmd); 7020 if (WARN_ON(ret < 0)) 7021 return ret; 7022 ret = register_event_command(&trigger_hist_disable_cmd); 7023 if (WARN_ON(ret < 0)) 7024 unregister_trigger_hist_enable_disable_cmds(); 7025 7026 return ret; 7027 } 7028