1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * event tracer
4 *
5 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
6 *
7 * - Added format output of fields of the trace point.
8 * This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
9 *
10 */
11
12 #define pr_fmt(fmt) fmt
13
14 #include <linux/workqueue.h>
15 #include <linux/security.h>
16 #include <linux/spinlock.h>
17 #include <linux/kthread.h>
18 #include <linux/tracefs.h>
19 #include <linux/uaccess.h>
20 #include <linux/module.h>
21 #include <linux/ctype.h>
22 #include <linux/sort.h>
23 #include <linux/slab.h>
24 #include <linux/delay.h>
25
26 #include <trace/events/sched.h>
27 #include <trace/syscall.h>
28
29 #include <asm/setup.h>
30
31 #include "trace_output.h"
32
33 #undef TRACE_SYSTEM
34 #define TRACE_SYSTEM "TRACE_SYSTEM"
35
36 DEFINE_MUTEX(event_mutex);
37
38 LIST_HEAD(ftrace_events);
39 static LIST_HEAD(ftrace_generic_fields);
40 static LIST_HEAD(ftrace_common_fields);
41 static bool eventdir_initialized;
42
43 static LIST_HEAD(module_strings);
44
45 struct module_string {
46 struct list_head next;
47 struct module *module;
48 char *str;
49 };
50
51 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
52
53 static struct kmem_cache *field_cachep;
54 static struct kmem_cache *file_cachep;
55
system_refcount(struct event_subsystem * system)56 static inline int system_refcount(struct event_subsystem *system)
57 {
58 return system->ref_count;
59 }
60
system_refcount_inc(struct event_subsystem * system)61 static int system_refcount_inc(struct event_subsystem *system)
62 {
63 return system->ref_count++;
64 }
65
system_refcount_dec(struct event_subsystem * system)66 static int system_refcount_dec(struct event_subsystem *system)
67 {
68 return --system->ref_count;
69 }
70
71 /* Double loops, do not use break, only goto's work */
72 #define do_for_each_event_file(tr, file) \
73 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
74 list_for_each_entry(file, &tr->events, list)
75
76 #define do_for_each_event_file_safe(tr, file) \
77 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
78 struct trace_event_file *___n; \
79 list_for_each_entry_safe(file, ___n, &tr->events, list)
80
81 #define while_for_each_event_file() \
82 }
83
84 static struct ftrace_event_field *
__find_event_field(struct list_head * head,const char * name)85 __find_event_field(struct list_head *head, const char *name)
86 {
87 struct ftrace_event_field *field;
88
89 list_for_each_entry(field, head, link) {
90 if (!strcmp(field->name, name))
91 return field;
92 }
93
94 return NULL;
95 }
96
97 struct ftrace_event_field *
trace_find_event_field(struct trace_event_call * call,char * name)98 trace_find_event_field(struct trace_event_call *call, char *name)
99 {
100 struct ftrace_event_field *field;
101 struct list_head *head;
102
103 head = trace_get_fields(call);
104 field = __find_event_field(head, name);
105 if (field)
106 return field;
107
108 field = __find_event_field(&ftrace_generic_fields, name);
109 if (field)
110 return field;
111
112 return __find_event_field(&ftrace_common_fields, name);
113 }
114
__trace_define_field(struct list_head * head,const char * type,const char * name,int offset,int size,int is_signed,int filter_type,int len,int need_test)115 static int __trace_define_field(struct list_head *head, const char *type,
116 const char *name, int offset, int size,
117 int is_signed, int filter_type, int len,
118 int need_test)
119 {
120 struct ftrace_event_field *field;
121
122 field = kmem_cache_alloc(field_cachep, GFP_TRACE);
123 if (!field)
124 return -ENOMEM;
125
126 field->name = name;
127 field->type = type;
128
129 if (filter_type == FILTER_OTHER)
130 field->filter_type = filter_assign_type(type);
131 else
132 field->filter_type = filter_type;
133
134 field->offset = offset;
135 field->size = size;
136 field->is_signed = is_signed;
137 field->needs_test = need_test;
138 field->len = len;
139
140 list_add(&field->link, head);
141
142 return 0;
143 }
144
trace_define_field(struct trace_event_call * call,const char * type,const char * name,int offset,int size,int is_signed,int filter_type)145 int trace_define_field(struct trace_event_call *call, const char *type,
146 const char *name, int offset, int size, int is_signed,
147 int filter_type)
148 {
149 struct list_head *head;
150
151 if (WARN_ON(!call->class))
152 return 0;
153
154 head = trace_get_fields(call);
155 return __trace_define_field(head, type, name, offset, size,
156 is_signed, filter_type, 0, 0);
157 }
158 EXPORT_SYMBOL_GPL(trace_define_field);
159
trace_define_field_ext(struct trace_event_call * call,const char * type,const char * name,int offset,int size,int is_signed,int filter_type,int len,int need_test)160 static int trace_define_field_ext(struct trace_event_call *call, const char *type,
161 const char *name, int offset, int size, int is_signed,
162 int filter_type, int len, int need_test)
163 {
164 struct list_head *head;
165
166 if (WARN_ON(!call->class))
167 return 0;
168
169 head = trace_get_fields(call);
170 return __trace_define_field(head, type, name, offset, size,
171 is_signed, filter_type, len, need_test);
172 }
173
174 #define __generic_field(type, item, filter_type) \
175 ret = __trace_define_field(&ftrace_generic_fields, #type, \
176 #item, 0, 0, is_signed_type(type), \
177 filter_type, 0, 0); \
178 if (ret) \
179 return ret;
180
181 #define __common_field(type, item) \
182 ret = __trace_define_field(&ftrace_common_fields, #type, \
183 "common_" #item, \
184 offsetof(typeof(ent), item), \
185 sizeof(ent.item), \
186 is_signed_type(type), FILTER_OTHER, \
187 0, 0); \
188 if (ret) \
189 return ret;
190
trace_define_generic_fields(void)191 static int trace_define_generic_fields(void)
192 {
193 int ret;
194
195 __generic_field(int, CPU, FILTER_CPU);
196 __generic_field(int, cpu, FILTER_CPU);
197 __generic_field(int, common_cpu, FILTER_CPU);
198 __generic_field(char *, COMM, FILTER_COMM);
199 __generic_field(char *, comm, FILTER_COMM);
200 __generic_field(char *, stacktrace, FILTER_STACKTRACE);
201 __generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
202
203 return ret;
204 }
205
trace_define_common_fields(void)206 static int trace_define_common_fields(void)
207 {
208 int ret;
209 struct trace_entry ent;
210
211 __common_field(unsigned short, type);
212 __common_field(unsigned char, flags);
213 /* Holds both preempt_count and migrate_disable */
214 __common_field(unsigned char, preempt_count);
215 __common_field(int, pid);
216
217 return ret;
218 }
219
trace_destroy_fields(struct trace_event_call * call)220 static void trace_destroy_fields(struct trace_event_call *call)
221 {
222 struct ftrace_event_field *field, *next;
223 struct list_head *head;
224
225 head = trace_get_fields(call);
226 list_for_each_entry_safe(field, next, head, link) {
227 list_del(&field->link);
228 kmem_cache_free(field_cachep, field);
229 }
230 }
231
232 /*
233 * run-time version of trace_event_get_offsets_<call>() that returns the last
234 * accessible offset of trace fields excluding __dynamic_array bytes
235 */
trace_event_get_offsets(struct trace_event_call * call)236 int trace_event_get_offsets(struct trace_event_call *call)
237 {
238 struct ftrace_event_field *tail;
239 struct list_head *head;
240
241 head = trace_get_fields(call);
242 /*
243 * head->next points to the last field with the largest offset,
244 * since it was added last by trace_define_field()
245 */
246 tail = list_first_entry(head, struct ftrace_event_field, link);
247 return tail->offset + tail->size;
248 }
249
250
find_event_field(const char * fmt,struct trace_event_call * call)251 static struct trace_event_fields *find_event_field(const char *fmt,
252 struct trace_event_call *call)
253 {
254 struct trace_event_fields *field = call->class->fields_array;
255 const char *p = fmt;
256 int len;
257
258 if (!(len = str_has_prefix(fmt, "REC->")))
259 return NULL;
260 fmt += len;
261 for (p = fmt; *p; p++) {
262 if (!isalnum(*p) && *p != '_')
263 break;
264 }
265 len = p - fmt;
266
267 for (; field->type; field++) {
268 if (strncmp(field->name, fmt, len) || field->name[len])
269 continue;
270
271 return field;
272 }
273 return NULL;
274 }
275
276 /*
277 * Check if the referenced field is an array and return true,
278 * as arrays are OK to dereference.
279 */
test_field(const char * fmt,struct trace_event_call * call)280 static bool test_field(const char *fmt, struct trace_event_call *call)
281 {
282 struct trace_event_fields *field;
283
284 field = find_event_field(fmt, call);
285 if (!field)
286 return false;
287
288 /* This is an array and is OK to dereference. */
289 return strchr(field->type, '[') != NULL;
290 }
291
292 /* Look for a string within an argument */
find_print_string(const char * arg,const char * str,const char * end)293 static bool find_print_string(const char *arg, const char *str, const char *end)
294 {
295 const char *r;
296
297 r = strstr(arg, str);
298 return r && r < end;
299 }
300
301 /* Return true if the argument pointer is safe */
process_pointer(const char * fmt,int len,struct trace_event_call * call)302 static bool process_pointer(const char *fmt, int len, struct trace_event_call *call)
303 {
304 const char *r, *e, *a;
305
306 e = fmt + len;
307
308 /* Find the REC-> in the argument */
309 r = strstr(fmt, "REC->");
310 if (r && r < e) {
311 /*
312 * Addresses of events on the buffer, or an array on the buffer is
313 * OK to dereference. There's ways to fool this, but
314 * this is to catch common mistakes, not malicious code.
315 */
316 a = strchr(fmt, '&');
317 if ((a && (a < r)) || test_field(r, call))
318 return true;
319 } else if (find_print_string(fmt, "__get_dynamic_array(", e)) {
320 return true;
321 } else if (find_print_string(fmt, "__get_rel_dynamic_array(", e)) {
322 return true;
323 } else if (find_print_string(fmt, "__get_dynamic_array_len(", e)) {
324 return true;
325 } else if (find_print_string(fmt, "__get_rel_dynamic_array_len(", e)) {
326 return true;
327 } else if (find_print_string(fmt, "__get_sockaddr(", e)) {
328 return true;
329 } else if (find_print_string(fmt, "__get_rel_sockaddr(", e)) {
330 return true;
331 }
332 return false;
333 }
334
335 /* Return true if the string is safe */
process_string(const char * fmt,int len,struct trace_event_call * call)336 static bool process_string(const char *fmt, int len, struct trace_event_call *call)
337 {
338 struct trace_event_fields *field;
339 const char *r, *e, *s;
340
341 e = fmt + len;
342
343 /*
344 * There are several helper functions that return strings.
345 * If the argument contains a function, then assume its field is valid.
346 * It is considered that the argument has a function if it has:
347 * alphanumeric or '_' before a parenthesis.
348 */
349 s = fmt;
350 do {
351 r = strstr(s, "(");
352 if (!r || r >= e)
353 break;
354 for (int i = 1; r - i >= s; i++) {
355 char ch = *(r - i);
356 if (isspace(ch))
357 continue;
358 if (isalnum(ch) || ch == '_')
359 return true;
360 /* Anything else, this isn't a function */
361 break;
362 }
363 /* A function could be wrapped in parethesis, try the next one */
364 s = r + 1;
365 } while (s < e);
366
367 /*
368 * Check for arrays. If the argument has: foo[REC->val]
369 * then it is very likely that foo is an array of strings
370 * that are safe to use.
371 */
372 r = strstr(s, "[");
373 if (r && r < e) {
374 r = strstr(r, "REC->");
375 if (r && r < e)
376 return true;
377 }
378
379 /*
380 * If there's any strings in the argument consider this arg OK as it
381 * could be: REC->field ? "foo" : "bar" and we don't want to get into
382 * verifying that logic here.
383 */
384 if (find_print_string(fmt, "\"", e))
385 return true;
386
387 /* Dereferenced strings are also valid like any other pointer */
388 if (process_pointer(fmt, len, call))
389 return true;
390
391 /* Make sure the field is found */
392 field = find_event_field(fmt, call);
393 if (!field)
394 return false;
395
396 /* Test this field's string before printing the event */
397 call->flags |= TRACE_EVENT_FL_TEST_STR;
398 field->needs_test = 1;
399
400 return true;
401 }
402
403 /*
404 * Examine the print fmt of the event looking for unsafe dereference
405 * pointers using %p* that could be recorded in the trace event and
406 * much later referenced after the pointer was freed. Dereferencing
407 * pointers are OK, if it is dereferenced into the event itself.
408 */
test_event_printk(struct trace_event_call * call)409 static void test_event_printk(struct trace_event_call *call)
410 {
411 u64 dereference_flags = 0;
412 u64 string_flags = 0;
413 bool first = true;
414 const char *fmt;
415 int parens = 0;
416 char in_quote = 0;
417 int start_arg = 0;
418 int arg = 0;
419 int i, e;
420
421 fmt = call->print_fmt;
422
423 if (!fmt)
424 return;
425
426 for (i = 0; fmt[i]; i++) {
427 switch (fmt[i]) {
428 case '\\':
429 i++;
430 if (!fmt[i])
431 return;
432 continue;
433 case '"':
434 case '\'':
435 /*
436 * The print fmt starts with a string that
437 * is processed first to find %p* usage,
438 * then after the first string, the print fmt
439 * contains arguments that are used to check
440 * if the dereferenced %p* usage is safe.
441 */
442 if (first) {
443 if (fmt[i] == '\'')
444 continue;
445 if (in_quote) {
446 arg = 0;
447 first = false;
448 /*
449 * If there was no %p* uses
450 * the fmt is OK.
451 */
452 if (!dereference_flags)
453 return;
454 }
455 }
456 if (in_quote) {
457 if (in_quote == fmt[i])
458 in_quote = 0;
459 } else {
460 in_quote = fmt[i];
461 }
462 continue;
463 case '%':
464 if (!first || !in_quote)
465 continue;
466 i++;
467 if (!fmt[i])
468 return;
469 switch (fmt[i]) {
470 case '%':
471 continue;
472 case 'p':
473 /* Find dereferencing fields */
474 switch (fmt[i + 1]) {
475 case 'B': case 'R': case 'r':
476 case 'b': case 'M': case 'm':
477 case 'I': case 'i': case 'E':
478 case 'U': case 'V': case 'N':
479 case 'a': case 'd': case 'D':
480 case 'g': case 't': case 'C':
481 case 'O': case 'f':
482 if (WARN_ONCE(arg == 63,
483 "Too many args for event: %s",
484 trace_event_name(call)))
485 return;
486 dereference_flags |= 1ULL << arg;
487 }
488 break;
489 default:
490 {
491 bool star = false;
492 int j;
493
494 /* Increment arg if %*s exists. */
495 for (j = 0; fmt[i + j]; j++) {
496 if (isdigit(fmt[i + j]) ||
497 fmt[i + j] == '.')
498 continue;
499 if (fmt[i + j] == '*') {
500 star = true;
501 continue;
502 }
503 if ((fmt[i + j] == 's')) {
504 if (star)
505 arg++;
506 if (WARN_ONCE(arg == 63,
507 "Too many args for event: %s",
508 trace_event_name(call)))
509 return;
510 dereference_flags |= 1ULL << arg;
511 string_flags |= 1ULL << arg;
512 }
513 break;
514 }
515 break;
516 } /* default */
517
518 } /* switch */
519 arg++;
520 continue;
521 case '(':
522 if (in_quote)
523 continue;
524 parens++;
525 continue;
526 case ')':
527 if (in_quote)
528 continue;
529 parens--;
530 if (WARN_ONCE(parens < 0,
531 "Paren mismatch for event: %s\narg='%s'\n%*s",
532 trace_event_name(call),
533 fmt + start_arg,
534 (i - start_arg) + 5, "^"))
535 return;
536 continue;
537 case ',':
538 if (in_quote || parens)
539 continue;
540 e = i;
541 i++;
542 while (isspace(fmt[i]))
543 i++;
544
545 /*
546 * If start_arg is zero, then this is the start of the
547 * first argument. The processing of the argument happens
548 * when the end of the argument is found, as it needs to
549 * handle paranthesis and such.
550 */
551 if (!start_arg) {
552 start_arg = i;
553 /* Balance out the i++ in the for loop */
554 i--;
555 continue;
556 }
557
558 if (dereference_flags & (1ULL << arg)) {
559 if (string_flags & (1ULL << arg)) {
560 if (process_string(fmt + start_arg, e - start_arg, call))
561 dereference_flags &= ~(1ULL << arg);
562 } else if (process_pointer(fmt + start_arg, e - start_arg, call))
563 dereference_flags &= ~(1ULL << arg);
564 }
565
566 start_arg = i;
567 arg++;
568 /* Balance out the i++ in the for loop */
569 i--;
570 }
571 }
572
573 if (dereference_flags & (1ULL << arg)) {
574 if (string_flags & (1ULL << arg)) {
575 if (process_string(fmt + start_arg, i - start_arg, call))
576 dereference_flags &= ~(1ULL << arg);
577 } else if (process_pointer(fmt + start_arg, i - start_arg, call))
578 dereference_flags &= ~(1ULL << arg);
579 }
580
581 /*
582 * If you triggered the below warning, the trace event reported
583 * uses an unsafe dereference pointer %p*. As the data stored
584 * at the trace event time may no longer exist when the trace
585 * event is printed, dereferencing to the original source is
586 * unsafe. The source of the dereference must be copied into the
587 * event itself, and the dereference must access the copy instead.
588 */
589 if (WARN_ON_ONCE(dereference_flags)) {
590 arg = 1;
591 while (!(dereference_flags & 1)) {
592 dereference_flags >>= 1;
593 arg++;
594 }
595 pr_warn("event %s has unsafe dereference of argument %d\n",
596 trace_event_name(call), arg);
597 pr_warn("print_fmt: %s\n", fmt);
598 }
599 }
600
trace_event_raw_init(struct trace_event_call * call)601 int trace_event_raw_init(struct trace_event_call *call)
602 {
603 int id;
604
605 id = register_trace_event(&call->event);
606 if (!id)
607 return -ENODEV;
608
609 test_event_printk(call);
610
611 return 0;
612 }
613 EXPORT_SYMBOL_GPL(trace_event_raw_init);
614
trace_event_ignore_this_pid(struct trace_event_file * trace_file)615 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
616 {
617 struct trace_array *tr = trace_file->tr;
618 struct trace_array_cpu *data;
619 struct trace_pid_list *no_pid_list;
620 struct trace_pid_list *pid_list;
621
622 pid_list = rcu_dereference_raw(tr->filtered_pids);
623 no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
624
625 if (!pid_list && !no_pid_list)
626 return false;
627
628 data = this_cpu_ptr(tr->array_buffer.data);
629
630 return data->ignore_pid;
631 }
632 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
633
trace_event_buffer_reserve(struct trace_event_buffer * fbuffer,struct trace_event_file * trace_file,unsigned long len)634 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
635 struct trace_event_file *trace_file,
636 unsigned long len)
637 {
638 struct trace_event_call *event_call = trace_file->event_call;
639
640 if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
641 trace_event_ignore_this_pid(trace_file))
642 return NULL;
643
644 /*
645 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
646 * preemption (adding one to the preempt_count). Since we are
647 * interested in the preempt_count at the time the tracepoint was
648 * hit, we need to subtract one to offset the increment.
649 */
650 fbuffer->trace_ctx = tracing_gen_ctx_dec();
651 fbuffer->trace_file = trace_file;
652
653 fbuffer->event =
654 trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
655 event_call->event.type, len,
656 fbuffer->trace_ctx);
657 if (!fbuffer->event)
658 return NULL;
659
660 fbuffer->regs = NULL;
661 fbuffer->entry = ring_buffer_event_data(fbuffer->event);
662 return fbuffer->entry;
663 }
664 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
665
trace_event_reg(struct trace_event_call * call,enum trace_reg type,void * data)666 int trace_event_reg(struct trace_event_call *call,
667 enum trace_reg type, void *data)
668 {
669 struct trace_event_file *file = data;
670
671 WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
672 switch (type) {
673 case TRACE_REG_REGISTER:
674 return tracepoint_probe_register(call->tp,
675 call->class->probe,
676 file);
677 case TRACE_REG_UNREGISTER:
678 tracepoint_probe_unregister(call->tp,
679 call->class->probe,
680 file);
681 return 0;
682
683 #ifdef CONFIG_PERF_EVENTS
684 case TRACE_REG_PERF_REGISTER:
685 return tracepoint_probe_register(call->tp,
686 call->class->perf_probe,
687 call);
688 case TRACE_REG_PERF_UNREGISTER:
689 tracepoint_probe_unregister(call->tp,
690 call->class->perf_probe,
691 call);
692 return 0;
693 case TRACE_REG_PERF_OPEN:
694 case TRACE_REG_PERF_CLOSE:
695 case TRACE_REG_PERF_ADD:
696 case TRACE_REG_PERF_DEL:
697 return 0;
698 #endif
699 }
700 return 0;
701 }
702 EXPORT_SYMBOL_GPL(trace_event_reg);
703
trace_event_enable_cmd_record(bool enable)704 void trace_event_enable_cmd_record(bool enable)
705 {
706 struct trace_event_file *file;
707 struct trace_array *tr;
708
709 lockdep_assert_held(&event_mutex);
710
711 do_for_each_event_file(tr, file) {
712
713 if (!(file->flags & EVENT_FILE_FL_ENABLED))
714 continue;
715
716 if (enable) {
717 tracing_start_cmdline_record();
718 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
719 } else {
720 tracing_stop_cmdline_record();
721 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
722 }
723 } while_for_each_event_file();
724 }
725
trace_event_enable_tgid_record(bool enable)726 void trace_event_enable_tgid_record(bool enable)
727 {
728 struct trace_event_file *file;
729 struct trace_array *tr;
730
731 lockdep_assert_held(&event_mutex);
732
733 do_for_each_event_file(tr, file) {
734 if (!(file->flags & EVENT_FILE_FL_ENABLED))
735 continue;
736
737 if (enable) {
738 tracing_start_tgid_record();
739 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
740 } else {
741 tracing_stop_tgid_record();
742 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
743 &file->flags);
744 }
745 } while_for_each_event_file();
746 }
747
__ftrace_event_enable_disable(struct trace_event_file * file,int enable,int soft_disable)748 static int __ftrace_event_enable_disable(struct trace_event_file *file,
749 int enable, int soft_disable)
750 {
751 struct trace_event_call *call = file->event_call;
752 struct trace_array *tr = file->tr;
753 int ret = 0;
754 int disable;
755
756 switch (enable) {
757 case 0:
758 /*
759 * When soft_disable is set and enable is cleared, the sm_ref
760 * reference counter is decremented. If it reaches 0, we want
761 * to clear the SOFT_DISABLED flag but leave the event in the
762 * state that it was. That is, if the event was enabled and
763 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
764 * is set we do not want the event to be enabled before we
765 * clear the bit.
766 *
767 * When soft_disable is not set but the SOFT_MODE flag is,
768 * we do nothing. Do not disable the tracepoint, otherwise
769 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
770 */
771 if (soft_disable) {
772 if (atomic_dec_return(&file->sm_ref) > 0)
773 break;
774 disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
775 clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
776 /* Disable use of trace_buffered_event */
777 trace_buffered_event_disable();
778 } else
779 disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
780
781 if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
782 clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
783 if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
784 tracing_stop_cmdline_record();
785 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
786 }
787
788 if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
789 tracing_stop_tgid_record();
790 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
791 }
792
793 call->class->reg(call, TRACE_REG_UNREGISTER, file);
794 }
795 /* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
796 if (file->flags & EVENT_FILE_FL_SOFT_MODE)
797 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
798 else
799 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
800 break;
801 case 1:
802 /*
803 * When soft_disable is set and enable is set, we want to
804 * register the tracepoint for the event, but leave the event
805 * as is. That means, if the event was already enabled, we do
806 * nothing (but set SOFT_MODE). If the event is disabled, we
807 * set SOFT_DISABLED before enabling the event tracepoint, so
808 * it still seems to be disabled.
809 */
810 if (!soft_disable)
811 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
812 else {
813 if (atomic_inc_return(&file->sm_ref) > 1)
814 break;
815 set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
816 /* Enable use of trace_buffered_event */
817 trace_buffered_event_enable();
818 }
819
820 if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
821 bool cmd = false, tgid = false;
822
823 /* Keep the event disabled, when going to SOFT_MODE. */
824 if (soft_disable)
825 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
826
827 if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
828 cmd = true;
829 tracing_start_cmdline_record();
830 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
831 }
832
833 if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
834 tgid = true;
835 tracing_start_tgid_record();
836 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
837 }
838
839 ret = call->class->reg(call, TRACE_REG_REGISTER, file);
840 if (ret) {
841 if (cmd)
842 tracing_stop_cmdline_record();
843 if (tgid)
844 tracing_stop_tgid_record();
845 pr_info("event trace: Could not enable event "
846 "%s\n", trace_event_name(call));
847 break;
848 }
849 set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
850
851 /* WAS_ENABLED gets set but never cleared. */
852 set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
853 }
854 break;
855 }
856
857 return ret;
858 }
859
trace_event_enable_disable(struct trace_event_file * file,int enable,int soft_disable)860 int trace_event_enable_disable(struct trace_event_file *file,
861 int enable, int soft_disable)
862 {
863 return __ftrace_event_enable_disable(file, enable, soft_disable);
864 }
865
ftrace_event_enable_disable(struct trace_event_file * file,int enable)866 static int ftrace_event_enable_disable(struct trace_event_file *file,
867 int enable)
868 {
869 return __ftrace_event_enable_disable(file, enable, 0);
870 }
871
872 #ifdef CONFIG_MODULES
873 struct event_mod_load {
874 struct list_head list;
875 char *module;
876 char *match;
877 char *system;
878 char *event;
879 };
880
free_event_mod(struct event_mod_load * event_mod)881 static void free_event_mod(struct event_mod_load *event_mod)
882 {
883 list_del(&event_mod->list);
884 kfree(event_mod->module);
885 kfree(event_mod->match);
886 kfree(event_mod->system);
887 kfree(event_mod->event);
888 kfree(event_mod);
889 }
890
clear_mod_events(struct trace_array * tr)891 static void clear_mod_events(struct trace_array *tr)
892 {
893 struct event_mod_load *event_mod, *n;
894
895 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
896 free_event_mod(event_mod);
897 }
898 }
899
remove_cache_mod(struct trace_array * tr,const char * mod,const char * match,const char * system,const char * event)900 static int remove_cache_mod(struct trace_array *tr, const char *mod,
901 const char *match, const char *system, const char *event)
902 {
903 struct event_mod_load *event_mod, *n;
904 int ret = -EINVAL;
905
906 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
907 if (strcmp(event_mod->module, mod) != 0)
908 continue;
909
910 if (match && strcmp(event_mod->match, match) != 0)
911 continue;
912
913 if (system &&
914 (!event_mod->system || strcmp(event_mod->system, system) != 0))
915 continue;
916
917 if (event &&
918 (!event_mod->event || strcmp(event_mod->event, event) != 0))
919 continue;
920
921 free_event_mod(event_mod);
922 ret = 0;
923 }
924
925 return ret;
926 }
927
cache_mod(struct trace_array * tr,const char * mod,int set,const char * match,const char * system,const char * event)928 static int cache_mod(struct trace_array *tr, const char *mod, int set,
929 const char *match, const char *system, const char *event)
930 {
931 struct event_mod_load *event_mod;
932
933 /* If the module exists, then this just failed to find an event */
934 if (module_exists(mod))
935 return -EINVAL;
936
937 /* See if this is to remove a cached filter */
938 if (!set)
939 return remove_cache_mod(tr, mod, match, system, event);
940
941 event_mod = kzalloc(sizeof(*event_mod), GFP_KERNEL);
942 if (!event_mod)
943 return -ENOMEM;
944
945 INIT_LIST_HEAD(&event_mod->list);
946 event_mod->module = kstrdup(mod, GFP_KERNEL);
947 if (!event_mod->module)
948 goto out_free;
949
950 if (match) {
951 event_mod->match = kstrdup(match, GFP_KERNEL);
952 if (!event_mod->match)
953 goto out_free;
954 }
955
956 if (system) {
957 event_mod->system = kstrdup(system, GFP_KERNEL);
958 if (!event_mod->system)
959 goto out_free;
960 }
961
962 if (event) {
963 event_mod->event = kstrdup(event, GFP_KERNEL);
964 if (!event_mod->event)
965 goto out_free;
966 }
967
968 list_add(&event_mod->list, &tr->mod_events);
969
970 return 0;
971
972 out_free:
973 free_event_mod(event_mod);
974
975 return -ENOMEM;
976 }
977 #else /* CONFIG_MODULES */
clear_mod_events(struct trace_array * tr)978 static inline void clear_mod_events(struct trace_array *tr) { }
cache_mod(struct trace_array * tr,const char * mod,int set,const char * match,const char * system,const char * event)979 static int cache_mod(struct trace_array *tr, const char *mod, int set,
980 const char *match, const char *system, const char *event)
981 {
982 return -EINVAL;
983 }
984 #endif
985
ftrace_clear_events(struct trace_array * tr)986 static void ftrace_clear_events(struct trace_array *tr)
987 {
988 struct trace_event_file *file;
989
990 mutex_lock(&event_mutex);
991 list_for_each_entry(file, &tr->events, list) {
992 ftrace_event_enable_disable(file, 0);
993 }
994 clear_mod_events(tr);
995 mutex_unlock(&event_mutex);
996 }
997
998 static void
event_filter_pid_sched_process_exit(void * data,struct task_struct * task)999 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
1000 {
1001 struct trace_pid_list *pid_list;
1002 struct trace_array *tr = data;
1003
1004 pid_list = rcu_dereference_raw(tr->filtered_pids);
1005 trace_filter_add_remove_task(pid_list, NULL, task);
1006
1007 pid_list = rcu_dereference_raw(tr->filtered_no_pids);
1008 trace_filter_add_remove_task(pid_list, NULL, task);
1009 }
1010
1011 static void
event_filter_pid_sched_process_fork(void * data,struct task_struct * self,struct task_struct * task)1012 event_filter_pid_sched_process_fork(void *data,
1013 struct task_struct *self,
1014 struct task_struct *task)
1015 {
1016 struct trace_pid_list *pid_list;
1017 struct trace_array *tr = data;
1018
1019 pid_list = rcu_dereference_sched(tr->filtered_pids);
1020 trace_filter_add_remove_task(pid_list, self, task);
1021
1022 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1023 trace_filter_add_remove_task(pid_list, self, task);
1024 }
1025
trace_event_follow_fork(struct trace_array * tr,bool enable)1026 void trace_event_follow_fork(struct trace_array *tr, bool enable)
1027 {
1028 if (enable) {
1029 register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
1030 tr, INT_MIN);
1031 register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
1032 tr, INT_MAX);
1033 } else {
1034 unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
1035 tr);
1036 unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
1037 tr);
1038 }
1039 }
1040
1041 static void
event_filter_pid_sched_switch_probe_pre(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)1042 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
1043 struct task_struct *prev,
1044 struct task_struct *next,
1045 unsigned int prev_state)
1046 {
1047 struct trace_array *tr = data;
1048 struct trace_pid_list *no_pid_list;
1049 struct trace_pid_list *pid_list;
1050 bool ret;
1051
1052 pid_list = rcu_dereference_sched(tr->filtered_pids);
1053 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1054
1055 /*
1056 * Sched switch is funny, as we only want to ignore it
1057 * in the notrace case if both prev and next should be ignored.
1058 */
1059 ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
1060 trace_ignore_this_task(NULL, no_pid_list, next);
1061
1062 this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
1063 (trace_ignore_this_task(pid_list, NULL, prev) &&
1064 trace_ignore_this_task(pid_list, NULL, next)));
1065 }
1066
1067 static void
event_filter_pid_sched_switch_probe_post(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)1068 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
1069 struct task_struct *prev,
1070 struct task_struct *next,
1071 unsigned int prev_state)
1072 {
1073 struct trace_array *tr = data;
1074 struct trace_pid_list *no_pid_list;
1075 struct trace_pid_list *pid_list;
1076
1077 pid_list = rcu_dereference_sched(tr->filtered_pids);
1078 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1079
1080 this_cpu_write(tr->array_buffer.data->ignore_pid,
1081 trace_ignore_this_task(pid_list, no_pid_list, next));
1082 }
1083
1084 static void
event_filter_pid_sched_wakeup_probe_pre(void * data,struct task_struct * task)1085 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
1086 {
1087 struct trace_array *tr = data;
1088 struct trace_pid_list *no_pid_list;
1089 struct trace_pid_list *pid_list;
1090
1091 /* Nothing to do if we are already tracing */
1092 if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
1093 return;
1094
1095 pid_list = rcu_dereference_sched(tr->filtered_pids);
1096 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1097
1098 this_cpu_write(tr->array_buffer.data->ignore_pid,
1099 trace_ignore_this_task(pid_list, no_pid_list, task));
1100 }
1101
1102 static void
event_filter_pid_sched_wakeup_probe_post(void * data,struct task_struct * task)1103 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
1104 {
1105 struct trace_array *tr = data;
1106 struct trace_pid_list *no_pid_list;
1107 struct trace_pid_list *pid_list;
1108
1109 /* Nothing to do if we are not tracing */
1110 if (this_cpu_read(tr->array_buffer.data->ignore_pid))
1111 return;
1112
1113 pid_list = rcu_dereference_sched(tr->filtered_pids);
1114 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1115
1116 /* Set tracing if current is enabled */
1117 this_cpu_write(tr->array_buffer.data->ignore_pid,
1118 trace_ignore_this_task(pid_list, no_pid_list, current));
1119 }
1120
unregister_pid_events(struct trace_array * tr)1121 static void unregister_pid_events(struct trace_array *tr)
1122 {
1123 unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
1124 unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
1125
1126 unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
1127 unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
1128
1129 unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
1130 unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
1131
1132 unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
1133 unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
1134 }
1135
__ftrace_clear_event_pids(struct trace_array * tr,int type)1136 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
1137 {
1138 struct trace_pid_list *pid_list;
1139 struct trace_pid_list *no_pid_list;
1140 struct trace_event_file *file;
1141 int cpu;
1142
1143 pid_list = rcu_dereference_protected(tr->filtered_pids,
1144 lockdep_is_held(&event_mutex));
1145 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1146 lockdep_is_held(&event_mutex));
1147
1148 /* Make sure there's something to do */
1149 if (!pid_type_enabled(type, pid_list, no_pid_list))
1150 return;
1151
1152 if (!still_need_pid_events(type, pid_list, no_pid_list)) {
1153 unregister_pid_events(tr);
1154
1155 list_for_each_entry(file, &tr->events, list) {
1156 clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1157 }
1158
1159 for_each_possible_cpu(cpu)
1160 per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
1161 }
1162
1163 if (type & TRACE_PIDS)
1164 rcu_assign_pointer(tr->filtered_pids, NULL);
1165
1166 if (type & TRACE_NO_PIDS)
1167 rcu_assign_pointer(tr->filtered_no_pids, NULL);
1168
1169 /* Wait till all users are no longer using pid filtering */
1170 tracepoint_synchronize_unregister();
1171
1172 if ((type & TRACE_PIDS) && pid_list)
1173 trace_pid_list_free(pid_list);
1174
1175 if ((type & TRACE_NO_PIDS) && no_pid_list)
1176 trace_pid_list_free(no_pid_list);
1177 }
1178
ftrace_clear_event_pids(struct trace_array * tr,int type)1179 static void ftrace_clear_event_pids(struct trace_array *tr, int type)
1180 {
1181 mutex_lock(&event_mutex);
1182 __ftrace_clear_event_pids(tr, type);
1183 mutex_unlock(&event_mutex);
1184 }
1185
__put_system(struct event_subsystem * system)1186 static void __put_system(struct event_subsystem *system)
1187 {
1188 struct event_filter *filter = system->filter;
1189
1190 WARN_ON_ONCE(system_refcount(system) == 0);
1191 if (system_refcount_dec(system))
1192 return;
1193
1194 list_del(&system->list);
1195
1196 if (filter) {
1197 kfree(filter->filter_string);
1198 kfree(filter);
1199 }
1200 kfree_const(system->name);
1201 kfree(system);
1202 }
1203
__get_system(struct event_subsystem * system)1204 static void __get_system(struct event_subsystem *system)
1205 {
1206 WARN_ON_ONCE(system_refcount(system) == 0);
1207 system_refcount_inc(system);
1208 }
1209
__get_system_dir(struct trace_subsystem_dir * dir)1210 static void __get_system_dir(struct trace_subsystem_dir *dir)
1211 {
1212 WARN_ON_ONCE(dir->ref_count == 0);
1213 dir->ref_count++;
1214 __get_system(dir->subsystem);
1215 }
1216
__put_system_dir(struct trace_subsystem_dir * dir)1217 static void __put_system_dir(struct trace_subsystem_dir *dir)
1218 {
1219 WARN_ON_ONCE(dir->ref_count == 0);
1220 /* If the subsystem is about to be freed, the dir must be too */
1221 WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
1222
1223 __put_system(dir->subsystem);
1224 if (!--dir->ref_count)
1225 kfree(dir);
1226 }
1227
put_system(struct trace_subsystem_dir * dir)1228 static void put_system(struct trace_subsystem_dir *dir)
1229 {
1230 mutex_lock(&event_mutex);
1231 __put_system_dir(dir);
1232 mutex_unlock(&event_mutex);
1233 }
1234
remove_subsystem(struct trace_subsystem_dir * dir)1235 static void remove_subsystem(struct trace_subsystem_dir *dir)
1236 {
1237 if (!dir)
1238 return;
1239
1240 if (!--dir->nr_events) {
1241 eventfs_remove_dir(dir->ei);
1242 list_del(&dir->list);
1243 __put_system_dir(dir);
1244 }
1245 }
1246
event_file_get(struct trace_event_file * file)1247 void event_file_get(struct trace_event_file *file)
1248 {
1249 refcount_inc(&file->ref);
1250 }
1251
event_file_put(struct trace_event_file * file)1252 void event_file_put(struct trace_event_file *file)
1253 {
1254 if (WARN_ON_ONCE(!refcount_read(&file->ref))) {
1255 if (file->flags & EVENT_FILE_FL_FREED)
1256 kmem_cache_free(file_cachep, file);
1257 return;
1258 }
1259
1260 if (refcount_dec_and_test(&file->ref)) {
1261 /* Count should only go to zero when it is freed */
1262 if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1263 return;
1264 kmem_cache_free(file_cachep, file);
1265 }
1266 }
1267
remove_event_file_dir(struct trace_event_file * file)1268 static void remove_event_file_dir(struct trace_event_file *file)
1269 {
1270 eventfs_remove_dir(file->ei);
1271 list_del(&file->list);
1272 remove_subsystem(file->system);
1273 free_event_filter(file->filter);
1274 file->flags |= EVENT_FILE_FL_FREED;
1275 event_file_put(file);
1276 }
1277
1278 /*
1279 * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1280 */
1281 static int
__ftrace_set_clr_event_nolock(struct trace_array * tr,const char * match,const char * sub,const char * event,int set,const char * mod)1282 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1283 const char *sub, const char *event, int set,
1284 const char *mod)
1285 {
1286 struct trace_event_file *file;
1287 struct trace_event_call *call;
1288 char *module __free(kfree) = NULL;
1289 const char *name;
1290 int ret = -EINVAL;
1291 int eret = 0;
1292
1293 if (mod) {
1294 char *p;
1295
1296 module = kstrdup(mod, GFP_KERNEL);
1297 if (!module)
1298 return -ENOMEM;
1299
1300 /* Replace all '-' with '_' as that's what modules do */
1301 for (p = strchr(module, '-'); p; p = strchr(p + 1, '-'))
1302 *p = '_';
1303 }
1304
1305 list_for_each_entry(file, &tr->events, list) {
1306
1307 call = file->event_call;
1308
1309 /* If a module is specified, skip events that are not that module */
1310 if (module && (!call->module || strcmp(module_name(call->module), module)))
1311 continue;
1312
1313 name = trace_event_name(call);
1314
1315 if (!name || !call->class || !call->class->reg)
1316 continue;
1317
1318 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1319 continue;
1320
1321 if (match &&
1322 strcmp(match, name) != 0 &&
1323 strcmp(match, call->class->system) != 0)
1324 continue;
1325
1326 if (sub && strcmp(sub, call->class->system) != 0)
1327 continue;
1328
1329 if (event && strcmp(event, name) != 0)
1330 continue;
1331
1332 ret = ftrace_event_enable_disable(file, set);
1333
1334 /*
1335 * Save the first error and return that. Some events
1336 * may still have been enabled, but let the user
1337 * know that something went wrong.
1338 */
1339 if (ret && !eret)
1340 eret = ret;
1341
1342 ret = eret;
1343 }
1344
1345 /*
1346 * If this is a module setting and nothing was found,
1347 * check if the module was loaded. If it wasn't cache it.
1348 */
1349 if (module && ret == -EINVAL && !eret)
1350 ret = cache_mod(tr, module, set, match, sub, event);
1351
1352 return ret;
1353 }
1354
__ftrace_set_clr_event(struct trace_array * tr,const char * match,const char * sub,const char * event,int set,const char * mod)1355 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1356 const char *sub, const char *event, int set,
1357 const char *mod)
1358 {
1359 int ret;
1360
1361 mutex_lock(&event_mutex);
1362 ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set, mod);
1363 mutex_unlock(&event_mutex);
1364
1365 return ret;
1366 }
1367
ftrace_set_clr_event(struct trace_array * tr,char * buf,int set)1368 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1369 {
1370 char *event = NULL, *sub = NULL, *match, *mod;
1371 int ret;
1372
1373 if (!tr)
1374 return -ENOENT;
1375
1376 /* Modules events can be appened with :mod:<module> */
1377 mod = strstr(buf, ":mod:");
1378 if (mod) {
1379 *mod = '\0';
1380 /* move to the module name */
1381 mod += 5;
1382 }
1383
1384 /*
1385 * The buf format can be <subsystem>:<event-name>
1386 * *:<event-name> means any event by that name.
1387 * :<event-name> is the same.
1388 *
1389 * <subsystem>:* means all events in that subsystem
1390 * <subsystem>: means the same.
1391 *
1392 * <name> (no ':') means all events in a subsystem with
1393 * the name <name> or any event that matches <name>
1394 */
1395
1396 match = strsep(&buf, ":");
1397 if (buf) {
1398 sub = match;
1399 event = buf;
1400 match = NULL;
1401
1402 if (!strlen(sub) || strcmp(sub, "*") == 0)
1403 sub = NULL;
1404 if (!strlen(event) || strcmp(event, "*") == 0)
1405 event = NULL;
1406 } else if (mod) {
1407 /* Allow wildcard for no length or star */
1408 if (!strlen(match) || strcmp(match, "*") == 0)
1409 match = NULL;
1410 }
1411
1412 ret = __ftrace_set_clr_event(tr, match, sub, event, set, mod);
1413
1414 /* Put back the colon to allow this to be called again */
1415 if (buf)
1416 *(buf - 1) = ':';
1417
1418 return ret;
1419 }
1420
1421 /**
1422 * trace_set_clr_event - enable or disable an event
1423 * @system: system name to match (NULL for any system)
1424 * @event: event name to match (NULL for all events, within system)
1425 * @set: 1 to enable, 0 to disable
1426 *
1427 * This is a way for other parts of the kernel to enable or disable
1428 * event recording.
1429 *
1430 * Returns 0 on success, -EINVAL if the parameters do not match any
1431 * registered events.
1432 */
trace_set_clr_event(const char * system,const char * event,int set)1433 int trace_set_clr_event(const char *system, const char *event, int set)
1434 {
1435 struct trace_array *tr = top_trace_array();
1436
1437 if (!tr)
1438 return -ENODEV;
1439
1440 return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1441 }
1442 EXPORT_SYMBOL_GPL(trace_set_clr_event);
1443
1444 /**
1445 * trace_array_set_clr_event - enable or disable an event for a trace array.
1446 * @tr: concerned trace array.
1447 * @system: system name to match (NULL for any system)
1448 * @event: event name to match (NULL for all events, within system)
1449 * @enable: true to enable, false to disable
1450 *
1451 * This is a way for other parts of the kernel to enable or disable
1452 * event recording.
1453 *
1454 * Returns 0 on success, -EINVAL if the parameters do not match any
1455 * registered events.
1456 */
trace_array_set_clr_event(struct trace_array * tr,const char * system,const char * event,bool enable)1457 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1458 const char *event, bool enable)
1459 {
1460 int set;
1461
1462 if (!tr)
1463 return -ENOENT;
1464
1465 set = (enable == true) ? 1 : 0;
1466 return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1467 }
1468 EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1469
1470 /* 128 should be much more than enough */
1471 #define EVENT_BUF_SIZE 127
1472
1473 static ssize_t
ftrace_event_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)1474 ftrace_event_write(struct file *file, const char __user *ubuf,
1475 size_t cnt, loff_t *ppos)
1476 {
1477 struct trace_parser parser;
1478 struct seq_file *m = file->private_data;
1479 struct trace_array *tr = m->private;
1480 ssize_t read, ret;
1481
1482 if (!cnt)
1483 return 0;
1484
1485 ret = tracing_update_buffers(tr);
1486 if (ret < 0)
1487 return ret;
1488
1489 if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1490 return -ENOMEM;
1491
1492 read = trace_get_user(&parser, ubuf, cnt, ppos);
1493
1494 if (read >= 0 && trace_parser_loaded((&parser))) {
1495 int set = 1;
1496
1497 if (*parser.buffer == '!')
1498 set = 0;
1499
1500 ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1501 if (ret)
1502 goto out_put;
1503 }
1504
1505 ret = read;
1506
1507 out_put:
1508 trace_parser_put(&parser);
1509
1510 return ret;
1511 }
1512
1513 static void *
t_next(struct seq_file * m,void * v,loff_t * pos)1514 t_next(struct seq_file *m, void *v, loff_t *pos)
1515 {
1516 struct trace_event_file *file = v;
1517 struct trace_event_call *call;
1518 struct trace_array *tr = m->private;
1519
1520 (*pos)++;
1521
1522 list_for_each_entry_continue(file, &tr->events, list) {
1523 call = file->event_call;
1524 /*
1525 * The ftrace subsystem is for showing formats only.
1526 * They can not be enabled or disabled via the event files.
1527 */
1528 if (call->class && call->class->reg &&
1529 !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1530 return file;
1531 }
1532
1533 return NULL;
1534 }
1535
t_start(struct seq_file * m,loff_t * pos)1536 static void *t_start(struct seq_file *m, loff_t *pos)
1537 {
1538 struct trace_event_file *file;
1539 struct trace_array *tr = m->private;
1540 loff_t l;
1541
1542 mutex_lock(&event_mutex);
1543
1544 file = list_entry(&tr->events, struct trace_event_file, list);
1545 for (l = 0; l <= *pos; ) {
1546 file = t_next(m, file, &l);
1547 if (!file)
1548 break;
1549 }
1550 return file;
1551 }
1552
1553 enum set_event_iter_type {
1554 SET_EVENT_FILE,
1555 SET_EVENT_MOD,
1556 };
1557
1558 struct set_event_iter {
1559 enum set_event_iter_type type;
1560 union {
1561 struct trace_event_file *file;
1562 struct event_mod_load *event_mod;
1563 };
1564 };
1565
1566 static void *
s_next(struct seq_file * m,void * v,loff_t * pos)1567 s_next(struct seq_file *m, void *v, loff_t *pos)
1568 {
1569 struct set_event_iter *iter = v;
1570 struct trace_event_file *file;
1571 struct trace_array *tr = m->private;
1572
1573 (*pos)++;
1574
1575 if (iter->type == SET_EVENT_FILE) {
1576 file = iter->file;
1577 list_for_each_entry_continue(file, &tr->events, list) {
1578 if (file->flags & EVENT_FILE_FL_ENABLED) {
1579 iter->file = file;
1580 return iter;
1581 }
1582 }
1583 #ifdef CONFIG_MODULES
1584 iter->type = SET_EVENT_MOD;
1585 iter->event_mod = list_entry(&tr->mod_events, struct event_mod_load, list);
1586 #endif
1587 }
1588
1589 #ifdef CONFIG_MODULES
1590 list_for_each_entry_continue(iter->event_mod, &tr->mod_events, list)
1591 return iter;
1592 #endif
1593
1594 /*
1595 * The iter is allocated in s_start() and passed via the 'v'
1596 * parameter. To stop the iterator, NULL must be returned. But
1597 * the return value is what the 'v' parameter in s_stop() receives
1598 * and frees. Free iter here as it will no longer be used.
1599 */
1600 kfree(iter);
1601 return NULL;
1602 }
1603
s_start(struct seq_file * m,loff_t * pos)1604 static void *s_start(struct seq_file *m, loff_t *pos)
1605 {
1606 struct trace_array *tr = m->private;
1607 struct set_event_iter *iter;
1608 loff_t l;
1609
1610 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1611 if (!iter)
1612 return NULL;
1613
1614 mutex_lock(&event_mutex);
1615
1616 iter->type = SET_EVENT_FILE;
1617 iter->file = list_entry(&tr->events, struct trace_event_file, list);
1618
1619 for (l = 0; l <= *pos; ) {
1620 iter = s_next(m, iter, &l);
1621 if (!iter)
1622 break;
1623 }
1624 return iter;
1625 }
1626
t_show(struct seq_file * m,void * v)1627 static int t_show(struct seq_file *m, void *v)
1628 {
1629 struct trace_event_file *file = v;
1630 struct trace_event_call *call = file->event_call;
1631
1632 if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1633 seq_printf(m, "%s:", call->class->system);
1634 seq_printf(m, "%s\n", trace_event_name(call));
1635
1636 return 0;
1637 }
1638
t_stop(struct seq_file * m,void * p)1639 static void t_stop(struct seq_file *m, void *p)
1640 {
1641 mutex_unlock(&event_mutex);
1642 }
1643
1644 #ifdef CONFIG_MODULES
s_show(struct seq_file * m,void * v)1645 static int s_show(struct seq_file *m, void *v)
1646 {
1647 struct set_event_iter *iter = v;
1648 const char *system;
1649 const char *event;
1650
1651 if (iter->type == SET_EVENT_FILE)
1652 return t_show(m, iter->file);
1653
1654 /* When match is set, system and event are not */
1655 if (iter->event_mod->match) {
1656 seq_printf(m, "%s:mod:%s\n", iter->event_mod->match,
1657 iter->event_mod->module);
1658 return 0;
1659 }
1660
1661 system = iter->event_mod->system ? : "*";
1662 event = iter->event_mod->event ? : "*";
1663
1664 seq_printf(m, "%s:%s:mod:%s\n", system, event, iter->event_mod->module);
1665
1666 return 0;
1667 }
1668 #else /* CONFIG_MODULES */
s_show(struct seq_file * m,void * v)1669 static int s_show(struct seq_file *m, void *v)
1670 {
1671 struct set_event_iter *iter = v;
1672
1673 return t_show(m, iter->file);
1674 }
1675 #endif
1676
s_stop(struct seq_file * m,void * v)1677 static void s_stop(struct seq_file *m, void *v)
1678 {
1679 kfree(v);
1680 t_stop(m, NULL);
1681 }
1682
1683 static void *
__next(struct seq_file * m,void * v,loff_t * pos,int type)1684 __next(struct seq_file *m, void *v, loff_t *pos, int type)
1685 {
1686 struct trace_array *tr = m->private;
1687 struct trace_pid_list *pid_list;
1688
1689 if (type == TRACE_PIDS)
1690 pid_list = rcu_dereference_sched(tr->filtered_pids);
1691 else
1692 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1693
1694 return trace_pid_next(pid_list, v, pos);
1695 }
1696
1697 static void *
p_next(struct seq_file * m,void * v,loff_t * pos)1698 p_next(struct seq_file *m, void *v, loff_t *pos)
1699 {
1700 return __next(m, v, pos, TRACE_PIDS);
1701 }
1702
1703 static void *
np_next(struct seq_file * m,void * v,loff_t * pos)1704 np_next(struct seq_file *m, void *v, loff_t *pos)
1705 {
1706 return __next(m, v, pos, TRACE_NO_PIDS);
1707 }
1708
__start(struct seq_file * m,loff_t * pos,int type)1709 static void *__start(struct seq_file *m, loff_t *pos, int type)
1710 __acquires(RCU)
1711 {
1712 struct trace_pid_list *pid_list;
1713 struct trace_array *tr = m->private;
1714
1715 /*
1716 * Grab the mutex, to keep calls to p_next() having the same
1717 * tr->filtered_pids as p_start() has.
1718 * If we just passed the tr->filtered_pids around, then RCU would
1719 * have been enough, but doing that makes things more complex.
1720 */
1721 mutex_lock(&event_mutex);
1722 rcu_read_lock_sched();
1723
1724 if (type == TRACE_PIDS)
1725 pid_list = rcu_dereference_sched(tr->filtered_pids);
1726 else
1727 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1728
1729 if (!pid_list)
1730 return NULL;
1731
1732 return trace_pid_start(pid_list, pos);
1733 }
1734
p_start(struct seq_file * m,loff_t * pos)1735 static void *p_start(struct seq_file *m, loff_t *pos)
1736 __acquires(RCU)
1737 {
1738 return __start(m, pos, TRACE_PIDS);
1739 }
1740
np_start(struct seq_file * m,loff_t * pos)1741 static void *np_start(struct seq_file *m, loff_t *pos)
1742 __acquires(RCU)
1743 {
1744 return __start(m, pos, TRACE_NO_PIDS);
1745 }
1746
p_stop(struct seq_file * m,void * p)1747 static void p_stop(struct seq_file *m, void *p)
1748 __releases(RCU)
1749 {
1750 rcu_read_unlock_sched();
1751 mutex_unlock(&event_mutex);
1752 }
1753
1754 static ssize_t
event_enable_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1755 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1756 loff_t *ppos)
1757 {
1758 struct trace_event_file *file;
1759 unsigned long flags;
1760 char buf[4] = "0";
1761
1762 mutex_lock(&event_mutex);
1763 file = event_file_file(filp);
1764 if (likely(file))
1765 flags = file->flags;
1766 mutex_unlock(&event_mutex);
1767
1768 if (!file)
1769 return -ENODEV;
1770
1771 if (flags & EVENT_FILE_FL_ENABLED &&
1772 !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1773 strcpy(buf, "1");
1774
1775 if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1776 flags & EVENT_FILE_FL_SOFT_MODE)
1777 strcat(buf, "*");
1778
1779 strcat(buf, "\n");
1780
1781 return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1782 }
1783
1784 static ssize_t
event_enable_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1785 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1786 loff_t *ppos)
1787 {
1788 struct trace_event_file *file;
1789 unsigned long val;
1790 int ret;
1791
1792 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1793 if (ret)
1794 return ret;
1795
1796 guard(mutex)(&event_mutex);
1797
1798 switch (val) {
1799 case 0:
1800 case 1:
1801 file = event_file_file(filp);
1802 if (!file)
1803 return -ENODEV;
1804 ret = tracing_update_buffers(file->tr);
1805 if (ret < 0)
1806 return ret;
1807 ret = ftrace_event_enable_disable(file, val);
1808 if (ret < 0)
1809 return ret;
1810 break;
1811
1812 default:
1813 return -EINVAL;
1814 }
1815
1816 *ppos += cnt;
1817
1818 return cnt;
1819 }
1820
1821 static ssize_t
system_enable_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1822 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1823 loff_t *ppos)
1824 {
1825 const char set_to_char[4] = { '?', '0', '1', 'X' };
1826 struct trace_subsystem_dir *dir = filp->private_data;
1827 struct event_subsystem *system = dir->subsystem;
1828 struct trace_event_call *call;
1829 struct trace_event_file *file;
1830 struct trace_array *tr = dir->tr;
1831 char buf[2];
1832 int set = 0;
1833 int ret;
1834
1835 mutex_lock(&event_mutex);
1836 list_for_each_entry(file, &tr->events, list) {
1837 call = file->event_call;
1838 if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1839 !trace_event_name(call) || !call->class || !call->class->reg)
1840 continue;
1841
1842 if (system && strcmp(call->class->system, system->name) != 0)
1843 continue;
1844
1845 /*
1846 * We need to find out if all the events are set
1847 * or if all events or cleared, or if we have
1848 * a mixture.
1849 */
1850 set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1851
1852 /*
1853 * If we have a mixture, no need to look further.
1854 */
1855 if (set == 3)
1856 break;
1857 }
1858 mutex_unlock(&event_mutex);
1859
1860 buf[0] = set_to_char[set];
1861 buf[1] = '\n';
1862
1863 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1864
1865 return ret;
1866 }
1867
1868 static ssize_t
system_enable_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1869 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1870 loff_t *ppos)
1871 {
1872 struct trace_subsystem_dir *dir = filp->private_data;
1873 struct event_subsystem *system = dir->subsystem;
1874 const char *name = NULL;
1875 unsigned long val;
1876 ssize_t ret;
1877
1878 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1879 if (ret)
1880 return ret;
1881
1882 ret = tracing_update_buffers(dir->tr);
1883 if (ret < 0)
1884 return ret;
1885
1886 if (val != 0 && val != 1)
1887 return -EINVAL;
1888
1889 /*
1890 * Opening of "enable" adds a ref count to system,
1891 * so the name is safe to use.
1892 */
1893 if (system)
1894 name = system->name;
1895
1896 ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val, NULL);
1897 if (ret)
1898 goto out;
1899
1900 ret = cnt;
1901
1902 out:
1903 *ppos += cnt;
1904
1905 return ret;
1906 }
1907
1908 enum {
1909 FORMAT_HEADER = 1,
1910 FORMAT_FIELD_SEPERATOR = 2,
1911 FORMAT_PRINTFMT = 3,
1912 };
1913
f_next(struct seq_file * m,void * v,loff_t * pos)1914 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1915 {
1916 struct trace_event_file *file = event_file_data(m->private);
1917 struct trace_event_call *call = file->event_call;
1918 struct list_head *common_head = &ftrace_common_fields;
1919 struct list_head *head = trace_get_fields(call);
1920 struct list_head *node = v;
1921
1922 (*pos)++;
1923
1924 switch ((unsigned long)v) {
1925 case FORMAT_HEADER:
1926 node = common_head;
1927 break;
1928
1929 case FORMAT_FIELD_SEPERATOR:
1930 node = head;
1931 break;
1932
1933 case FORMAT_PRINTFMT:
1934 /* all done */
1935 return NULL;
1936 }
1937
1938 node = node->prev;
1939 if (node == common_head)
1940 return (void *)FORMAT_FIELD_SEPERATOR;
1941 else if (node == head)
1942 return (void *)FORMAT_PRINTFMT;
1943 else
1944 return node;
1945 }
1946
f_show(struct seq_file * m,void * v)1947 static int f_show(struct seq_file *m, void *v)
1948 {
1949 struct trace_event_file *file = event_file_data(m->private);
1950 struct trace_event_call *call = file->event_call;
1951 struct ftrace_event_field *field;
1952 const char *array_descriptor;
1953
1954 switch ((unsigned long)v) {
1955 case FORMAT_HEADER:
1956 seq_printf(m, "name: %s\n", trace_event_name(call));
1957 seq_printf(m, "ID: %d\n", call->event.type);
1958 seq_puts(m, "format:\n");
1959 return 0;
1960
1961 case FORMAT_FIELD_SEPERATOR:
1962 seq_putc(m, '\n');
1963 return 0;
1964
1965 case FORMAT_PRINTFMT:
1966 seq_printf(m, "\nprint fmt: %s\n",
1967 call->print_fmt);
1968 return 0;
1969 }
1970
1971 field = list_entry(v, struct ftrace_event_field, link);
1972 /*
1973 * Smartly shows the array type(except dynamic array).
1974 * Normal:
1975 * field:TYPE VAR
1976 * If TYPE := TYPE[LEN], it is shown:
1977 * field:TYPE VAR[LEN]
1978 */
1979 array_descriptor = strchr(field->type, '[');
1980
1981 if (str_has_prefix(field->type, "__data_loc"))
1982 array_descriptor = NULL;
1983
1984 if (!array_descriptor)
1985 seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1986 field->type, field->name, field->offset,
1987 field->size, !!field->is_signed);
1988 else if (field->len)
1989 seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1990 (int)(array_descriptor - field->type),
1991 field->type, field->name,
1992 field->len, field->offset,
1993 field->size, !!field->is_signed);
1994 else
1995 seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1996 (int)(array_descriptor - field->type),
1997 field->type, field->name,
1998 field->offset, field->size, !!field->is_signed);
1999
2000 return 0;
2001 }
2002
f_start(struct seq_file * m,loff_t * pos)2003 static void *f_start(struct seq_file *m, loff_t *pos)
2004 {
2005 struct trace_event_file *file;
2006 void *p = (void *)FORMAT_HEADER;
2007 loff_t l = 0;
2008
2009 /* ->stop() is called even if ->start() fails */
2010 mutex_lock(&event_mutex);
2011 file = event_file_file(m->private);
2012 if (!file)
2013 return ERR_PTR(-ENODEV);
2014
2015 while (l < *pos && p)
2016 p = f_next(m, p, &l);
2017
2018 return p;
2019 }
2020
f_stop(struct seq_file * m,void * p)2021 static void f_stop(struct seq_file *m, void *p)
2022 {
2023 mutex_unlock(&event_mutex);
2024 }
2025
2026 static const struct seq_operations trace_format_seq_ops = {
2027 .start = f_start,
2028 .next = f_next,
2029 .stop = f_stop,
2030 .show = f_show,
2031 };
2032
trace_format_open(struct inode * inode,struct file * file)2033 static int trace_format_open(struct inode *inode, struct file *file)
2034 {
2035 struct seq_file *m;
2036 int ret;
2037
2038 /* Do we want to hide event format files on tracefs lockdown? */
2039
2040 ret = seq_open(file, &trace_format_seq_ops);
2041 if (ret < 0)
2042 return ret;
2043
2044 m = file->private_data;
2045 m->private = file;
2046
2047 return 0;
2048 }
2049
2050 #ifdef CONFIG_PERF_EVENTS
2051 static ssize_t
event_id_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2052 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2053 {
2054 int id = (long)event_file_data(filp);
2055 char buf[32];
2056 int len;
2057
2058 if (unlikely(!id))
2059 return -ENODEV;
2060
2061 len = sprintf(buf, "%d\n", id);
2062
2063 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
2064 }
2065 #endif
2066
2067 static ssize_t
event_filter_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2068 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2069 loff_t *ppos)
2070 {
2071 struct trace_event_file *file;
2072 struct trace_seq *s;
2073 int r = -ENODEV;
2074
2075 if (*ppos)
2076 return 0;
2077
2078 s = kmalloc(sizeof(*s), GFP_KERNEL);
2079
2080 if (!s)
2081 return -ENOMEM;
2082
2083 trace_seq_init(s);
2084
2085 mutex_lock(&event_mutex);
2086 file = event_file_file(filp);
2087 if (file)
2088 print_event_filter(file, s);
2089 mutex_unlock(&event_mutex);
2090
2091 if (file)
2092 r = simple_read_from_buffer(ubuf, cnt, ppos,
2093 s->buffer, trace_seq_used(s));
2094
2095 kfree(s);
2096
2097 return r;
2098 }
2099
2100 static ssize_t
event_filter_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2101 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2102 loff_t *ppos)
2103 {
2104 struct trace_event_file *file;
2105 char *buf;
2106 int err = -ENODEV;
2107
2108 if (cnt >= PAGE_SIZE)
2109 return -EINVAL;
2110
2111 buf = memdup_user_nul(ubuf, cnt);
2112 if (IS_ERR(buf))
2113 return PTR_ERR(buf);
2114
2115 mutex_lock(&event_mutex);
2116 file = event_file_file(filp);
2117 if (file) {
2118 if (file->flags & EVENT_FILE_FL_FREED)
2119 err = -ENODEV;
2120 else
2121 err = apply_event_filter(file, buf);
2122 }
2123 mutex_unlock(&event_mutex);
2124
2125 kfree(buf);
2126 if (err < 0)
2127 return err;
2128
2129 *ppos += cnt;
2130
2131 return cnt;
2132 }
2133
2134 static LIST_HEAD(event_subsystems);
2135
subsystem_open(struct inode * inode,struct file * filp)2136 static int subsystem_open(struct inode *inode, struct file *filp)
2137 {
2138 struct trace_subsystem_dir *dir = NULL, *iter_dir;
2139 struct trace_array *tr = NULL, *iter_tr;
2140 struct event_subsystem *system = NULL;
2141 int ret;
2142
2143 if (tracing_is_disabled())
2144 return -ENODEV;
2145
2146 /* Make sure the system still exists */
2147 mutex_lock(&event_mutex);
2148 mutex_lock(&trace_types_lock);
2149 list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
2150 list_for_each_entry(iter_dir, &iter_tr->systems, list) {
2151 if (iter_dir == inode->i_private) {
2152 /* Don't open systems with no events */
2153 tr = iter_tr;
2154 dir = iter_dir;
2155 if (dir->nr_events) {
2156 __get_system_dir(dir);
2157 system = dir->subsystem;
2158 }
2159 goto exit_loop;
2160 }
2161 }
2162 }
2163 exit_loop:
2164 mutex_unlock(&trace_types_lock);
2165 mutex_unlock(&event_mutex);
2166
2167 if (!system)
2168 return -ENODEV;
2169
2170 /* Still need to increment the ref count of the system */
2171 if (trace_array_get(tr) < 0) {
2172 put_system(dir);
2173 return -ENODEV;
2174 }
2175
2176 ret = tracing_open_generic(inode, filp);
2177 if (ret < 0) {
2178 trace_array_put(tr);
2179 put_system(dir);
2180 }
2181
2182 return ret;
2183 }
2184
system_tr_open(struct inode * inode,struct file * filp)2185 static int system_tr_open(struct inode *inode, struct file *filp)
2186 {
2187 struct trace_subsystem_dir *dir;
2188 struct trace_array *tr = inode->i_private;
2189 int ret;
2190
2191 /* Make a temporary dir that has no system but points to tr */
2192 dir = kzalloc(sizeof(*dir), GFP_KERNEL);
2193 if (!dir)
2194 return -ENOMEM;
2195
2196 ret = tracing_open_generic_tr(inode, filp);
2197 if (ret < 0) {
2198 kfree(dir);
2199 return ret;
2200 }
2201 dir->tr = tr;
2202 filp->private_data = dir;
2203
2204 return 0;
2205 }
2206
subsystem_release(struct inode * inode,struct file * file)2207 static int subsystem_release(struct inode *inode, struct file *file)
2208 {
2209 struct trace_subsystem_dir *dir = file->private_data;
2210
2211 trace_array_put(dir->tr);
2212
2213 /*
2214 * If dir->subsystem is NULL, then this is a temporary
2215 * descriptor that was made for a trace_array to enable
2216 * all subsystems.
2217 */
2218 if (dir->subsystem)
2219 put_system(dir);
2220 else
2221 kfree(dir);
2222
2223 return 0;
2224 }
2225
2226 static ssize_t
subsystem_filter_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2227 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2228 loff_t *ppos)
2229 {
2230 struct trace_subsystem_dir *dir = filp->private_data;
2231 struct event_subsystem *system = dir->subsystem;
2232 struct trace_seq *s;
2233 int r;
2234
2235 if (*ppos)
2236 return 0;
2237
2238 s = kmalloc(sizeof(*s), GFP_KERNEL);
2239 if (!s)
2240 return -ENOMEM;
2241
2242 trace_seq_init(s);
2243
2244 print_subsystem_event_filter(system, s);
2245 r = simple_read_from_buffer(ubuf, cnt, ppos,
2246 s->buffer, trace_seq_used(s));
2247
2248 kfree(s);
2249
2250 return r;
2251 }
2252
2253 static ssize_t
subsystem_filter_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2254 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2255 loff_t *ppos)
2256 {
2257 struct trace_subsystem_dir *dir = filp->private_data;
2258 char *buf;
2259 int err;
2260
2261 if (cnt >= PAGE_SIZE)
2262 return -EINVAL;
2263
2264 buf = memdup_user_nul(ubuf, cnt);
2265 if (IS_ERR(buf))
2266 return PTR_ERR(buf);
2267
2268 err = apply_subsystem_event_filter(dir, buf);
2269 kfree(buf);
2270 if (err < 0)
2271 return err;
2272
2273 *ppos += cnt;
2274
2275 return cnt;
2276 }
2277
2278 static ssize_t
show_header_page_file(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2279 show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2280 {
2281 struct trace_array *tr = filp->private_data;
2282 struct trace_seq *s;
2283 int r;
2284
2285 if (*ppos)
2286 return 0;
2287
2288 s = kmalloc(sizeof(*s), GFP_KERNEL);
2289 if (!s)
2290 return -ENOMEM;
2291
2292 trace_seq_init(s);
2293
2294 ring_buffer_print_page_header(tr->array_buffer.buffer, s);
2295 r = simple_read_from_buffer(ubuf, cnt, ppos,
2296 s->buffer, trace_seq_used(s));
2297
2298 kfree(s);
2299
2300 return r;
2301 }
2302
2303 static ssize_t
show_header_event_file(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2304 show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2305 {
2306 struct trace_seq *s;
2307 int r;
2308
2309 if (*ppos)
2310 return 0;
2311
2312 s = kmalloc(sizeof(*s), GFP_KERNEL);
2313 if (!s)
2314 return -ENOMEM;
2315
2316 trace_seq_init(s);
2317
2318 ring_buffer_print_entry_header(s);
2319 r = simple_read_from_buffer(ubuf, cnt, ppos,
2320 s->buffer, trace_seq_used(s));
2321
2322 kfree(s);
2323
2324 return r;
2325 }
2326
ignore_task_cpu(void * data)2327 static void ignore_task_cpu(void *data)
2328 {
2329 struct trace_array *tr = data;
2330 struct trace_pid_list *pid_list;
2331 struct trace_pid_list *no_pid_list;
2332
2333 /*
2334 * This function is called by on_each_cpu() while the
2335 * event_mutex is held.
2336 */
2337 pid_list = rcu_dereference_protected(tr->filtered_pids,
2338 mutex_is_locked(&event_mutex));
2339 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2340 mutex_is_locked(&event_mutex));
2341
2342 this_cpu_write(tr->array_buffer.data->ignore_pid,
2343 trace_ignore_this_task(pid_list, no_pid_list, current));
2344 }
2345
register_pid_events(struct trace_array * tr)2346 static void register_pid_events(struct trace_array *tr)
2347 {
2348 /*
2349 * Register a probe that is called before all other probes
2350 * to set ignore_pid if next or prev do not match.
2351 * Register a probe this is called after all other probes
2352 * to only keep ignore_pid set if next pid matches.
2353 */
2354 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
2355 tr, INT_MAX);
2356 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
2357 tr, 0);
2358
2359 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
2360 tr, INT_MAX);
2361 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
2362 tr, 0);
2363
2364 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
2365 tr, INT_MAX);
2366 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
2367 tr, 0);
2368
2369 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
2370 tr, INT_MAX);
2371 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
2372 tr, 0);
2373 }
2374
2375 static ssize_t
event_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos,int type)2376 event_pid_write(struct file *filp, const char __user *ubuf,
2377 size_t cnt, loff_t *ppos, int type)
2378 {
2379 struct seq_file *m = filp->private_data;
2380 struct trace_array *tr = m->private;
2381 struct trace_pid_list *filtered_pids = NULL;
2382 struct trace_pid_list *other_pids = NULL;
2383 struct trace_pid_list *pid_list;
2384 struct trace_event_file *file;
2385 ssize_t ret;
2386
2387 if (!cnt)
2388 return 0;
2389
2390 ret = tracing_update_buffers(tr);
2391 if (ret < 0)
2392 return ret;
2393
2394 guard(mutex)(&event_mutex);
2395
2396 if (type == TRACE_PIDS) {
2397 filtered_pids = rcu_dereference_protected(tr->filtered_pids,
2398 lockdep_is_held(&event_mutex));
2399 other_pids = rcu_dereference_protected(tr->filtered_no_pids,
2400 lockdep_is_held(&event_mutex));
2401 } else {
2402 filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
2403 lockdep_is_held(&event_mutex));
2404 other_pids = rcu_dereference_protected(tr->filtered_pids,
2405 lockdep_is_held(&event_mutex));
2406 }
2407
2408 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
2409 if (ret < 0)
2410 return ret;
2411
2412 if (type == TRACE_PIDS)
2413 rcu_assign_pointer(tr->filtered_pids, pid_list);
2414 else
2415 rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2416
2417 list_for_each_entry(file, &tr->events, list) {
2418 set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2419 }
2420
2421 if (filtered_pids) {
2422 tracepoint_synchronize_unregister();
2423 trace_pid_list_free(filtered_pids);
2424 } else if (pid_list && !other_pids) {
2425 register_pid_events(tr);
2426 }
2427
2428 /*
2429 * Ignoring of pids is done at task switch. But we have to
2430 * check for those tasks that are currently running.
2431 * Always do this in case a pid was appended or removed.
2432 */
2433 on_each_cpu(ignore_task_cpu, tr, 1);
2434
2435 *ppos += ret;
2436
2437 return ret;
2438 }
2439
2440 static ssize_t
ftrace_event_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2441 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2442 size_t cnt, loff_t *ppos)
2443 {
2444 return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2445 }
2446
2447 static ssize_t
ftrace_event_npid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2448 ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2449 size_t cnt, loff_t *ppos)
2450 {
2451 return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2452 }
2453
2454 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2455 static int ftrace_event_set_open(struct inode *inode, struct file *file);
2456 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2457 static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2458 static int ftrace_event_release(struct inode *inode, struct file *file);
2459
2460 static const struct seq_operations show_event_seq_ops = {
2461 .start = t_start,
2462 .next = t_next,
2463 .show = t_show,
2464 .stop = t_stop,
2465 };
2466
2467 static const struct seq_operations show_set_event_seq_ops = {
2468 .start = s_start,
2469 .next = s_next,
2470 .show = s_show,
2471 .stop = s_stop,
2472 };
2473
2474 static const struct seq_operations show_set_pid_seq_ops = {
2475 .start = p_start,
2476 .next = p_next,
2477 .show = trace_pid_show,
2478 .stop = p_stop,
2479 };
2480
2481 static const struct seq_operations show_set_no_pid_seq_ops = {
2482 .start = np_start,
2483 .next = np_next,
2484 .show = trace_pid_show,
2485 .stop = p_stop,
2486 };
2487
2488 static const struct file_operations ftrace_avail_fops = {
2489 .open = ftrace_event_avail_open,
2490 .read = seq_read,
2491 .llseek = seq_lseek,
2492 .release = seq_release,
2493 };
2494
2495 static const struct file_operations ftrace_set_event_fops = {
2496 .open = ftrace_event_set_open,
2497 .read = seq_read,
2498 .write = ftrace_event_write,
2499 .llseek = seq_lseek,
2500 .release = ftrace_event_release,
2501 };
2502
2503 static const struct file_operations ftrace_set_event_pid_fops = {
2504 .open = ftrace_event_set_pid_open,
2505 .read = seq_read,
2506 .write = ftrace_event_pid_write,
2507 .llseek = seq_lseek,
2508 .release = ftrace_event_release,
2509 };
2510
2511 static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2512 .open = ftrace_event_set_npid_open,
2513 .read = seq_read,
2514 .write = ftrace_event_npid_write,
2515 .llseek = seq_lseek,
2516 .release = ftrace_event_release,
2517 };
2518
2519 static const struct file_operations ftrace_enable_fops = {
2520 .open = tracing_open_file_tr,
2521 .read = event_enable_read,
2522 .write = event_enable_write,
2523 .release = tracing_release_file_tr,
2524 .llseek = default_llseek,
2525 };
2526
2527 static const struct file_operations ftrace_event_format_fops = {
2528 .open = trace_format_open,
2529 .read = seq_read,
2530 .llseek = seq_lseek,
2531 .release = seq_release,
2532 };
2533
2534 #ifdef CONFIG_PERF_EVENTS
2535 static const struct file_operations ftrace_event_id_fops = {
2536 .read = event_id_read,
2537 .llseek = default_llseek,
2538 };
2539 #endif
2540
2541 static const struct file_operations ftrace_event_filter_fops = {
2542 .open = tracing_open_file_tr,
2543 .read = event_filter_read,
2544 .write = event_filter_write,
2545 .release = tracing_release_file_tr,
2546 .llseek = default_llseek,
2547 };
2548
2549 static const struct file_operations ftrace_subsystem_filter_fops = {
2550 .open = subsystem_open,
2551 .read = subsystem_filter_read,
2552 .write = subsystem_filter_write,
2553 .llseek = default_llseek,
2554 .release = subsystem_release,
2555 };
2556
2557 static const struct file_operations ftrace_system_enable_fops = {
2558 .open = subsystem_open,
2559 .read = system_enable_read,
2560 .write = system_enable_write,
2561 .llseek = default_llseek,
2562 .release = subsystem_release,
2563 };
2564
2565 static const struct file_operations ftrace_tr_enable_fops = {
2566 .open = system_tr_open,
2567 .read = system_enable_read,
2568 .write = system_enable_write,
2569 .llseek = default_llseek,
2570 .release = subsystem_release,
2571 };
2572
2573 static const struct file_operations ftrace_show_header_page_fops = {
2574 .open = tracing_open_generic_tr,
2575 .read = show_header_page_file,
2576 .llseek = default_llseek,
2577 .release = tracing_release_generic_tr,
2578 };
2579
2580 static const struct file_operations ftrace_show_header_event_fops = {
2581 .open = tracing_open_generic_tr,
2582 .read = show_header_event_file,
2583 .llseek = default_llseek,
2584 .release = tracing_release_generic_tr,
2585 };
2586
2587 static int
ftrace_event_open(struct inode * inode,struct file * file,const struct seq_operations * seq_ops)2588 ftrace_event_open(struct inode *inode, struct file *file,
2589 const struct seq_operations *seq_ops)
2590 {
2591 struct seq_file *m;
2592 int ret;
2593
2594 ret = security_locked_down(LOCKDOWN_TRACEFS);
2595 if (ret)
2596 return ret;
2597
2598 ret = seq_open(file, seq_ops);
2599 if (ret < 0)
2600 return ret;
2601 m = file->private_data;
2602 /* copy tr over to seq ops */
2603 m->private = inode->i_private;
2604
2605 return ret;
2606 }
2607
ftrace_event_release(struct inode * inode,struct file * file)2608 static int ftrace_event_release(struct inode *inode, struct file *file)
2609 {
2610 struct trace_array *tr = inode->i_private;
2611
2612 trace_array_put(tr);
2613
2614 return seq_release(inode, file);
2615 }
2616
2617 static int
ftrace_event_avail_open(struct inode * inode,struct file * file)2618 ftrace_event_avail_open(struct inode *inode, struct file *file)
2619 {
2620 const struct seq_operations *seq_ops = &show_event_seq_ops;
2621
2622 /* Checks for tracefs lockdown */
2623 return ftrace_event_open(inode, file, seq_ops);
2624 }
2625
2626 static int
ftrace_event_set_open(struct inode * inode,struct file * file)2627 ftrace_event_set_open(struct inode *inode, struct file *file)
2628 {
2629 const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2630 struct trace_array *tr = inode->i_private;
2631 int ret;
2632
2633 ret = tracing_check_open_get_tr(tr);
2634 if (ret)
2635 return ret;
2636
2637 if ((file->f_mode & FMODE_WRITE) &&
2638 (file->f_flags & O_TRUNC))
2639 ftrace_clear_events(tr);
2640
2641 ret = ftrace_event_open(inode, file, seq_ops);
2642 if (ret < 0)
2643 trace_array_put(tr);
2644 return ret;
2645 }
2646
2647 static int
ftrace_event_set_pid_open(struct inode * inode,struct file * file)2648 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2649 {
2650 const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2651 struct trace_array *tr = inode->i_private;
2652 int ret;
2653
2654 ret = tracing_check_open_get_tr(tr);
2655 if (ret)
2656 return ret;
2657
2658 if ((file->f_mode & FMODE_WRITE) &&
2659 (file->f_flags & O_TRUNC))
2660 ftrace_clear_event_pids(tr, TRACE_PIDS);
2661
2662 ret = ftrace_event_open(inode, file, seq_ops);
2663 if (ret < 0)
2664 trace_array_put(tr);
2665 return ret;
2666 }
2667
2668 static int
ftrace_event_set_npid_open(struct inode * inode,struct file * file)2669 ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2670 {
2671 const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2672 struct trace_array *tr = inode->i_private;
2673 int ret;
2674
2675 ret = tracing_check_open_get_tr(tr);
2676 if (ret)
2677 return ret;
2678
2679 if ((file->f_mode & FMODE_WRITE) &&
2680 (file->f_flags & O_TRUNC))
2681 ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2682
2683 ret = ftrace_event_open(inode, file, seq_ops);
2684 if (ret < 0)
2685 trace_array_put(tr);
2686 return ret;
2687 }
2688
2689 static struct event_subsystem *
create_new_subsystem(const char * name)2690 create_new_subsystem(const char *name)
2691 {
2692 struct event_subsystem *system;
2693
2694 /* need to create new entry */
2695 system = kmalloc(sizeof(*system), GFP_KERNEL);
2696 if (!system)
2697 return NULL;
2698
2699 system->ref_count = 1;
2700
2701 /* Only allocate if dynamic (kprobes and modules) */
2702 system->name = kstrdup_const(name, GFP_KERNEL);
2703 if (!system->name)
2704 goto out_free;
2705
2706 system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2707 if (!system->filter)
2708 goto out_free;
2709
2710 list_add(&system->list, &event_subsystems);
2711
2712 return system;
2713
2714 out_free:
2715 kfree_const(system->name);
2716 kfree(system);
2717 return NULL;
2718 }
2719
system_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)2720 static int system_callback(const char *name, umode_t *mode, void **data,
2721 const struct file_operations **fops)
2722 {
2723 if (strcmp(name, "filter") == 0)
2724 *fops = &ftrace_subsystem_filter_fops;
2725
2726 else if (strcmp(name, "enable") == 0)
2727 *fops = &ftrace_system_enable_fops;
2728
2729 else
2730 return 0;
2731
2732 *mode = TRACE_MODE_WRITE;
2733 return 1;
2734 }
2735
2736 static struct eventfs_inode *
event_subsystem_dir(struct trace_array * tr,const char * name,struct trace_event_file * file,struct eventfs_inode * parent)2737 event_subsystem_dir(struct trace_array *tr, const char *name,
2738 struct trace_event_file *file, struct eventfs_inode *parent)
2739 {
2740 struct event_subsystem *system, *iter;
2741 struct trace_subsystem_dir *dir;
2742 struct eventfs_inode *ei;
2743 int nr_entries;
2744 static struct eventfs_entry system_entries[] = {
2745 {
2746 .name = "filter",
2747 .callback = system_callback,
2748 },
2749 {
2750 .name = "enable",
2751 .callback = system_callback,
2752 }
2753 };
2754
2755 /* First see if we did not already create this dir */
2756 list_for_each_entry(dir, &tr->systems, list) {
2757 system = dir->subsystem;
2758 if (strcmp(system->name, name) == 0) {
2759 dir->nr_events++;
2760 file->system = dir;
2761 return dir->ei;
2762 }
2763 }
2764
2765 /* Now see if the system itself exists. */
2766 system = NULL;
2767 list_for_each_entry(iter, &event_subsystems, list) {
2768 if (strcmp(iter->name, name) == 0) {
2769 system = iter;
2770 break;
2771 }
2772 }
2773
2774 dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2775 if (!dir)
2776 goto out_fail;
2777
2778 if (!system) {
2779 system = create_new_subsystem(name);
2780 if (!system)
2781 goto out_free;
2782 } else
2783 __get_system(system);
2784
2785 /* ftrace only has directories no files */
2786 if (strcmp(name, "ftrace") == 0)
2787 nr_entries = 0;
2788 else
2789 nr_entries = ARRAY_SIZE(system_entries);
2790
2791 ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir);
2792 if (IS_ERR(ei)) {
2793 pr_warn("Failed to create system directory %s\n", name);
2794 __put_system(system);
2795 goto out_free;
2796 }
2797
2798 dir->ei = ei;
2799 dir->tr = tr;
2800 dir->ref_count = 1;
2801 dir->nr_events = 1;
2802 dir->subsystem = system;
2803 file->system = dir;
2804
2805 list_add(&dir->list, &tr->systems);
2806
2807 return dir->ei;
2808
2809 out_free:
2810 kfree(dir);
2811 out_fail:
2812 /* Only print this message if failed on memory allocation */
2813 if (!dir || !system)
2814 pr_warn("No memory to create event subsystem %s\n", name);
2815 return NULL;
2816 }
2817
2818 static int
event_define_fields(struct trace_event_call * call)2819 event_define_fields(struct trace_event_call *call)
2820 {
2821 struct list_head *head;
2822 int ret = 0;
2823
2824 /*
2825 * Other events may have the same class. Only update
2826 * the fields if they are not already defined.
2827 */
2828 head = trace_get_fields(call);
2829 if (list_empty(head)) {
2830 struct trace_event_fields *field = call->class->fields_array;
2831 unsigned int offset = sizeof(struct trace_entry);
2832
2833 for (; field->type; field++) {
2834 if (field->type == TRACE_FUNCTION_TYPE) {
2835 field->define_fields(call);
2836 break;
2837 }
2838
2839 offset = ALIGN(offset, field->align);
2840 ret = trace_define_field_ext(call, field->type, field->name,
2841 offset, field->size,
2842 field->is_signed, field->filter_type,
2843 field->len, field->needs_test);
2844 if (WARN_ON_ONCE(ret)) {
2845 pr_err("error code is %d\n", ret);
2846 break;
2847 }
2848
2849 offset += field->size;
2850 }
2851 }
2852
2853 return ret;
2854 }
2855
event_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)2856 static int event_callback(const char *name, umode_t *mode, void **data,
2857 const struct file_operations **fops)
2858 {
2859 struct trace_event_file *file = *data;
2860 struct trace_event_call *call = file->event_call;
2861
2862 if (strcmp(name, "format") == 0) {
2863 *mode = TRACE_MODE_READ;
2864 *fops = &ftrace_event_format_fops;
2865 return 1;
2866 }
2867
2868 /*
2869 * Only event directories that can be enabled should have
2870 * triggers or filters, with the exception of the "print"
2871 * event that can have a "trigger" file.
2872 */
2873 if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2874 if (call->class->reg && strcmp(name, "enable") == 0) {
2875 *mode = TRACE_MODE_WRITE;
2876 *fops = &ftrace_enable_fops;
2877 return 1;
2878 }
2879
2880 if (strcmp(name, "filter") == 0) {
2881 *mode = TRACE_MODE_WRITE;
2882 *fops = &ftrace_event_filter_fops;
2883 return 1;
2884 }
2885 }
2886
2887 if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
2888 strcmp(trace_event_name(call), "print") == 0) {
2889 if (strcmp(name, "trigger") == 0) {
2890 *mode = TRACE_MODE_WRITE;
2891 *fops = &event_trigger_fops;
2892 return 1;
2893 }
2894 }
2895
2896 #ifdef CONFIG_PERF_EVENTS
2897 if (call->event.type && call->class->reg &&
2898 strcmp(name, "id") == 0) {
2899 *mode = TRACE_MODE_READ;
2900 *data = (void *)(long)call->event.type;
2901 *fops = &ftrace_event_id_fops;
2902 return 1;
2903 }
2904 #endif
2905
2906 #ifdef CONFIG_HIST_TRIGGERS
2907 if (strcmp(name, "hist") == 0) {
2908 *mode = TRACE_MODE_READ;
2909 *fops = &event_hist_fops;
2910 return 1;
2911 }
2912 #endif
2913 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2914 if (strcmp(name, "hist_debug") == 0) {
2915 *mode = TRACE_MODE_READ;
2916 *fops = &event_hist_debug_fops;
2917 return 1;
2918 }
2919 #endif
2920 #ifdef CONFIG_TRACE_EVENT_INJECT
2921 if (call->event.type && call->class->reg &&
2922 strcmp(name, "inject") == 0) {
2923 *mode = 0200;
2924 *fops = &event_inject_fops;
2925 return 1;
2926 }
2927 #endif
2928 return 0;
2929 }
2930
2931 /* The file is incremented on creation and freeing the enable file decrements it */
event_release(const char * name,void * data)2932 static void event_release(const char *name, void *data)
2933 {
2934 struct trace_event_file *file = data;
2935
2936 event_file_put(file);
2937 }
2938
2939 static int
event_create_dir(struct eventfs_inode * parent,struct trace_event_file * file)2940 event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
2941 {
2942 struct trace_event_call *call = file->event_call;
2943 struct trace_array *tr = file->tr;
2944 struct eventfs_inode *e_events;
2945 struct eventfs_inode *ei;
2946 const char *name;
2947 int nr_entries;
2948 int ret;
2949 static struct eventfs_entry event_entries[] = {
2950 {
2951 .name = "enable",
2952 .callback = event_callback,
2953 .release = event_release,
2954 },
2955 {
2956 .name = "filter",
2957 .callback = event_callback,
2958 },
2959 {
2960 .name = "trigger",
2961 .callback = event_callback,
2962 },
2963 {
2964 .name = "format",
2965 .callback = event_callback,
2966 },
2967 #ifdef CONFIG_PERF_EVENTS
2968 {
2969 .name = "id",
2970 .callback = event_callback,
2971 },
2972 #endif
2973 #ifdef CONFIG_HIST_TRIGGERS
2974 {
2975 .name = "hist",
2976 .callback = event_callback,
2977 },
2978 #endif
2979 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2980 {
2981 .name = "hist_debug",
2982 .callback = event_callback,
2983 },
2984 #endif
2985 #ifdef CONFIG_TRACE_EVENT_INJECT
2986 {
2987 .name = "inject",
2988 .callback = event_callback,
2989 },
2990 #endif
2991 };
2992
2993 /*
2994 * If the trace point header did not define TRACE_SYSTEM
2995 * then the system would be called "TRACE_SYSTEM". This should
2996 * never happen.
2997 */
2998 if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
2999 return -ENODEV;
3000
3001 e_events = event_subsystem_dir(tr, call->class->system, file, parent);
3002 if (!e_events)
3003 return -ENOMEM;
3004
3005 nr_entries = ARRAY_SIZE(event_entries);
3006
3007 name = trace_event_name(call);
3008 ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file);
3009 if (IS_ERR(ei)) {
3010 pr_warn("Could not create tracefs '%s' directory\n", name);
3011 return -1;
3012 }
3013
3014 file->ei = ei;
3015
3016 ret = event_define_fields(call);
3017 if (ret < 0) {
3018 pr_warn("Could not initialize trace point events/%s\n", name);
3019 return ret;
3020 }
3021
3022 /* Gets decremented on freeing of the "enable" file */
3023 event_file_get(file);
3024
3025 return 0;
3026 }
3027
remove_event_from_tracers(struct trace_event_call * call)3028 static void remove_event_from_tracers(struct trace_event_call *call)
3029 {
3030 struct trace_event_file *file;
3031 struct trace_array *tr;
3032
3033 do_for_each_event_file_safe(tr, file) {
3034 if (file->event_call != call)
3035 continue;
3036
3037 remove_event_file_dir(file);
3038 /*
3039 * The do_for_each_event_file_safe() is
3040 * a double loop. After finding the call for this
3041 * trace_array, we use break to jump to the next
3042 * trace_array.
3043 */
3044 break;
3045 } while_for_each_event_file();
3046 }
3047
event_remove(struct trace_event_call * call)3048 static void event_remove(struct trace_event_call *call)
3049 {
3050 struct trace_array *tr;
3051 struct trace_event_file *file;
3052
3053 do_for_each_event_file(tr, file) {
3054 if (file->event_call != call)
3055 continue;
3056
3057 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3058 tr->clear_trace = true;
3059
3060 ftrace_event_enable_disable(file, 0);
3061 /*
3062 * The do_for_each_event_file() is
3063 * a double loop. After finding the call for this
3064 * trace_array, we use break to jump to the next
3065 * trace_array.
3066 */
3067 break;
3068 } while_for_each_event_file();
3069
3070 if (call->event.funcs)
3071 __unregister_trace_event(&call->event);
3072 remove_event_from_tracers(call);
3073 list_del(&call->list);
3074 }
3075
event_init(struct trace_event_call * call)3076 static int event_init(struct trace_event_call *call)
3077 {
3078 int ret = 0;
3079 const char *name;
3080
3081 name = trace_event_name(call);
3082 if (WARN_ON(!name))
3083 return -EINVAL;
3084
3085 if (call->class->raw_init) {
3086 ret = call->class->raw_init(call);
3087 if (ret < 0 && ret != -ENOSYS)
3088 pr_warn("Could not initialize trace events/%s\n", name);
3089 }
3090
3091 return ret;
3092 }
3093
3094 static int
__register_event(struct trace_event_call * call,struct module * mod)3095 __register_event(struct trace_event_call *call, struct module *mod)
3096 {
3097 int ret;
3098
3099 ret = event_init(call);
3100 if (ret < 0)
3101 return ret;
3102
3103 list_add(&call->list, &ftrace_events);
3104 if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3105 atomic_set(&call->refcnt, 0);
3106 else
3107 call->module = mod;
3108
3109 return 0;
3110 }
3111
eval_replace(char * ptr,struct trace_eval_map * map,int len)3112 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
3113 {
3114 int rlen;
3115 int elen;
3116
3117 /* Find the length of the eval value as a string */
3118 elen = snprintf(ptr, 0, "%ld", map->eval_value);
3119 /* Make sure there's enough room to replace the string with the value */
3120 if (len < elen)
3121 return NULL;
3122
3123 snprintf(ptr, elen + 1, "%ld", map->eval_value);
3124
3125 /* Get the rest of the string of ptr */
3126 rlen = strlen(ptr + len);
3127 memmove(ptr + elen, ptr + len, rlen);
3128 /* Make sure we end the new string */
3129 ptr[elen + rlen] = 0;
3130
3131 return ptr + elen;
3132 }
3133
update_event_printk(struct trace_event_call * call,struct trace_eval_map * map)3134 static void update_event_printk(struct trace_event_call *call,
3135 struct trace_eval_map *map)
3136 {
3137 char *ptr;
3138 int quote = 0;
3139 int len = strlen(map->eval_string);
3140
3141 for (ptr = call->print_fmt; *ptr; ptr++) {
3142 if (*ptr == '\\') {
3143 ptr++;
3144 /* paranoid */
3145 if (!*ptr)
3146 break;
3147 continue;
3148 }
3149 if (*ptr == '"') {
3150 quote ^= 1;
3151 continue;
3152 }
3153 if (quote)
3154 continue;
3155 if (isdigit(*ptr)) {
3156 /* skip numbers */
3157 do {
3158 ptr++;
3159 /* Check for alpha chars like ULL */
3160 } while (isalnum(*ptr));
3161 if (!*ptr)
3162 break;
3163 /*
3164 * A number must have some kind of delimiter after
3165 * it, and we can ignore that too.
3166 */
3167 continue;
3168 }
3169 if (isalpha(*ptr) || *ptr == '_') {
3170 if (strncmp(map->eval_string, ptr, len) == 0 &&
3171 !isalnum(ptr[len]) && ptr[len] != '_') {
3172 ptr = eval_replace(ptr, map, len);
3173 /* enum/sizeof string smaller than value */
3174 if (WARN_ON_ONCE(!ptr))
3175 return;
3176 /*
3177 * No need to decrement here, as eval_replace()
3178 * returns the pointer to the character passed
3179 * the eval, and two evals can not be placed
3180 * back to back without something in between.
3181 * We can skip that something in between.
3182 */
3183 continue;
3184 }
3185 skip_more:
3186 do {
3187 ptr++;
3188 } while (isalnum(*ptr) || *ptr == '_');
3189 if (!*ptr)
3190 break;
3191 /*
3192 * If what comes after this variable is a '.' or
3193 * '->' then we can continue to ignore that string.
3194 */
3195 if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
3196 ptr += *ptr == '.' ? 1 : 2;
3197 if (!*ptr)
3198 break;
3199 goto skip_more;
3200 }
3201 /*
3202 * Once again, we can skip the delimiter that came
3203 * after the string.
3204 */
3205 continue;
3206 }
3207 }
3208 }
3209
add_str_to_module(struct module * module,char * str)3210 static void add_str_to_module(struct module *module, char *str)
3211 {
3212 struct module_string *modstr;
3213
3214 modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
3215
3216 /*
3217 * If we failed to allocate memory here, then we'll just
3218 * let the str memory leak when the module is removed.
3219 * If this fails to allocate, there's worse problems than
3220 * a leaked string on module removal.
3221 */
3222 if (WARN_ON_ONCE(!modstr))
3223 return;
3224
3225 modstr->module = module;
3226 modstr->str = str;
3227
3228 list_add(&modstr->next, &module_strings);
3229 }
3230
update_event_fields(struct trace_event_call * call,struct trace_eval_map * map)3231 static void update_event_fields(struct trace_event_call *call,
3232 struct trace_eval_map *map)
3233 {
3234 struct ftrace_event_field *field;
3235 struct list_head *head;
3236 char *ptr;
3237 char *str;
3238 int len = strlen(map->eval_string);
3239
3240 /* Dynamic events should never have field maps */
3241 if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
3242 return;
3243
3244 head = trace_get_fields(call);
3245 list_for_each_entry(field, head, link) {
3246 ptr = strchr(field->type, '[');
3247 if (!ptr)
3248 continue;
3249 ptr++;
3250
3251 if (!isalpha(*ptr) && *ptr != '_')
3252 continue;
3253
3254 if (strncmp(map->eval_string, ptr, len) != 0)
3255 continue;
3256
3257 str = kstrdup(field->type, GFP_KERNEL);
3258 if (WARN_ON_ONCE(!str))
3259 return;
3260 ptr = str + (ptr - field->type);
3261 ptr = eval_replace(ptr, map, len);
3262 /* enum/sizeof string smaller than value */
3263 if (WARN_ON_ONCE(!ptr)) {
3264 kfree(str);
3265 continue;
3266 }
3267
3268 /*
3269 * If the event is part of a module, then we need to free the string
3270 * when the module is removed. Otherwise, it will stay allocated
3271 * until a reboot.
3272 */
3273 if (call->module)
3274 add_str_to_module(call->module, str);
3275
3276 field->type = str;
3277 }
3278 }
3279
trace_event_eval_update(struct trace_eval_map ** map,int len)3280 void trace_event_eval_update(struct trace_eval_map **map, int len)
3281 {
3282 struct trace_event_call *call, *p;
3283 const char *last_system = NULL;
3284 bool first = false;
3285 int last_i;
3286 int i;
3287
3288 down_write(&trace_event_sem);
3289 list_for_each_entry_safe(call, p, &ftrace_events, list) {
3290 /* events are usually grouped together with systems */
3291 if (!last_system || call->class->system != last_system) {
3292 first = true;
3293 last_i = 0;
3294 last_system = call->class->system;
3295 }
3296
3297 /*
3298 * Since calls are grouped by systems, the likelihood that the
3299 * next call in the iteration belongs to the same system as the
3300 * previous call is high. As an optimization, we skip searching
3301 * for a map[] that matches the call's system if the last call
3302 * was from the same system. That's what last_i is for. If the
3303 * call has the same system as the previous call, then last_i
3304 * will be the index of the first map[] that has a matching
3305 * system.
3306 */
3307 for (i = last_i; i < len; i++) {
3308 if (call->class->system == map[i]->system) {
3309 /* Save the first system if need be */
3310 if (first) {
3311 last_i = i;
3312 first = false;
3313 }
3314 update_event_printk(call, map[i]);
3315 update_event_fields(call, map[i]);
3316 }
3317 }
3318 cond_resched();
3319 }
3320 up_write(&trace_event_sem);
3321 }
3322
event_in_systems(struct trace_event_call * call,const char * systems)3323 static bool event_in_systems(struct trace_event_call *call,
3324 const char *systems)
3325 {
3326 const char *system;
3327 const char *p;
3328
3329 if (!systems)
3330 return true;
3331
3332 system = call->class->system;
3333 p = strstr(systems, system);
3334 if (!p)
3335 return false;
3336
3337 if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
3338 return false;
3339
3340 p += strlen(system);
3341 return !*p || isspace(*p) || *p == ',';
3342 }
3343
3344 #ifdef CONFIG_HIST_TRIGGERS
3345 /*
3346 * Wake up waiter on the hist_poll_wq from irq_work because the hist trigger
3347 * may happen in any context.
3348 */
hist_poll_event_irq_work(struct irq_work * work)3349 static void hist_poll_event_irq_work(struct irq_work *work)
3350 {
3351 wake_up_all(&hist_poll_wq);
3352 }
3353
3354 DEFINE_IRQ_WORK(hist_poll_work, hist_poll_event_irq_work);
3355 DECLARE_WAIT_QUEUE_HEAD(hist_poll_wq);
3356 #endif
3357
3358 static struct trace_event_file *
trace_create_new_event(struct trace_event_call * call,struct trace_array * tr)3359 trace_create_new_event(struct trace_event_call *call,
3360 struct trace_array *tr)
3361 {
3362 struct trace_pid_list *no_pid_list;
3363 struct trace_pid_list *pid_list;
3364 struct trace_event_file *file;
3365 unsigned int first;
3366
3367 if (!event_in_systems(call, tr->system_names))
3368 return NULL;
3369
3370 file = kmem_cache_alloc(file_cachep, GFP_TRACE);
3371 if (!file)
3372 return ERR_PTR(-ENOMEM);
3373
3374 pid_list = rcu_dereference_protected(tr->filtered_pids,
3375 lockdep_is_held(&event_mutex));
3376 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
3377 lockdep_is_held(&event_mutex));
3378
3379 if (!trace_pid_list_first(pid_list, &first) ||
3380 !trace_pid_list_first(no_pid_list, &first))
3381 file->flags |= EVENT_FILE_FL_PID_FILTER;
3382
3383 file->event_call = call;
3384 file->tr = tr;
3385 atomic_set(&file->sm_ref, 0);
3386 atomic_set(&file->tm_ref, 0);
3387 INIT_LIST_HEAD(&file->triggers);
3388 list_add(&file->list, &tr->events);
3389 refcount_set(&file->ref, 1);
3390
3391 return file;
3392 }
3393
3394 #define MAX_BOOT_TRIGGERS 32
3395
3396 static struct boot_triggers {
3397 const char *event;
3398 char *trigger;
3399 } bootup_triggers[MAX_BOOT_TRIGGERS];
3400
3401 static char bootup_trigger_buf[COMMAND_LINE_SIZE];
3402 static int nr_boot_triggers;
3403
setup_trace_triggers(char * str)3404 static __init int setup_trace_triggers(char *str)
3405 {
3406 char *trigger;
3407 char *buf;
3408 int i;
3409
3410 strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
3411 trace_set_ring_buffer_expanded(NULL);
3412 disable_tracing_selftest("running event triggers");
3413
3414 buf = bootup_trigger_buf;
3415 for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
3416 trigger = strsep(&buf, ",");
3417 if (!trigger)
3418 break;
3419 bootup_triggers[i].event = strsep(&trigger, ".");
3420 bootup_triggers[i].trigger = trigger;
3421 if (!bootup_triggers[i].trigger)
3422 break;
3423 }
3424
3425 nr_boot_triggers = i;
3426 return 1;
3427 }
3428 __setup("trace_trigger=", setup_trace_triggers);
3429
3430 /* Add an event to a trace directory */
3431 static int
__trace_add_new_event(struct trace_event_call * call,struct trace_array * tr)3432 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
3433 {
3434 struct trace_event_file *file;
3435
3436 file = trace_create_new_event(call, tr);
3437 /*
3438 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3439 * allocation, or NULL if the event is not part of the tr->system_names.
3440 * When the event is not part of the tr->system_names, return zero, not
3441 * an error.
3442 */
3443 if (!file)
3444 return 0;
3445
3446 if (IS_ERR(file))
3447 return PTR_ERR(file);
3448
3449 if (eventdir_initialized)
3450 return event_create_dir(tr->event_dir, file);
3451 else
3452 return event_define_fields(call);
3453 }
3454
trace_early_triggers(struct trace_event_file * file,const char * name)3455 static void trace_early_triggers(struct trace_event_file *file, const char *name)
3456 {
3457 int ret;
3458 int i;
3459
3460 for (i = 0; i < nr_boot_triggers; i++) {
3461 if (strcmp(name, bootup_triggers[i].event))
3462 continue;
3463 mutex_lock(&event_mutex);
3464 ret = trigger_process_regex(file, bootup_triggers[i].trigger);
3465 mutex_unlock(&event_mutex);
3466 if (ret)
3467 pr_err("Failed to register trigger '%s' on event %s\n",
3468 bootup_triggers[i].trigger,
3469 bootup_triggers[i].event);
3470 }
3471 }
3472
3473 /*
3474 * Just create a descriptor for early init. A descriptor is required
3475 * for enabling events at boot. We want to enable events before
3476 * the filesystem is initialized.
3477 */
3478 static int
__trace_early_add_new_event(struct trace_event_call * call,struct trace_array * tr)3479 __trace_early_add_new_event(struct trace_event_call *call,
3480 struct trace_array *tr)
3481 {
3482 struct trace_event_file *file;
3483 int ret;
3484
3485 file = trace_create_new_event(call, tr);
3486 /*
3487 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3488 * allocation, or NULL if the event is not part of the tr->system_names.
3489 * When the event is not part of the tr->system_names, return zero, not
3490 * an error.
3491 */
3492 if (!file)
3493 return 0;
3494
3495 if (IS_ERR(file))
3496 return PTR_ERR(file);
3497
3498 ret = event_define_fields(call);
3499 if (ret)
3500 return ret;
3501
3502 trace_early_triggers(file, trace_event_name(call));
3503
3504 return 0;
3505 }
3506
3507 struct ftrace_module_file_ops;
3508 static void __add_event_to_tracers(struct trace_event_call *call);
3509
3510 /* Add an additional event_call dynamically */
trace_add_event_call(struct trace_event_call * call)3511 int trace_add_event_call(struct trace_event_call *call)
3512 {
3513 int ret;
3514 lockdep_assert_held(&event_mutex);
3515
3516 guard(mutex)(&trace_types_lock);
3517
3518 ret = __register_event(call, NULL);
3519 if (ret < 0)
3520 return ret;
3521
3522 __add_event_to_tracers(call);
3523 return ret;
3524 }
3525 EXPORT_SYMBOL_GPL(trace_add_event_call);
3526
3527 /*
3528 * Must be called under locking of trace_types_lock, event_mutex and
3529 * trace_event_sem.
3530 */
__trace_remove_event_call(struct trace_event_call * call)3531 static void __trace_remove_event_call(struct trace_event_call *call)
3532 {
3533 event_remove(call);
3534 trace_destroy_fields(call);
3535 }
3536
probe_remove_event_call(struct trace_event_call * call)3537 static int probe_remove_event_call(struct trace_event_call *call)
3538 {
3539 struct trace_array *tr;
3540 struct trace_event_file *file;
3541
3542 #ifdef CONFIG_PERF_EVENTS
3543 if (call->perf_refcount)
3544 return -EBUSY;
3545 #endif
3546 do_for_each_event_file(tr, file) {
3547 if (file->event_call != call)
3548 continue;
3549 /*
3550 * We can't rely on ftrace_event_enable_disable(enable => 0)
3551 * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
3552 * TRACE_REG_UNREGISTER.
3553 */
3554 if (file->flags & EVENT_FILE_FL_ENABLED)
3555 goto busy;
3556
3557 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3558 tr->clear_trace = true;
3559 /*
3560 * The do_for_each_event_file_safe() is
3561 * a double loop. After finding the call for this
3562 * trace_array, we use break to jump to the next
3563 * trace_array.
3564 */
3565 break;
3566 } while_for_each_event_file();
3567
3568 __trace_remove_event_call(call);
3569
3570 return 0;
3571 busy:
3572 /* No need to clear the trace now */
3573 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
3574 tr->clear_trace = false;
3575 }
3576 return -EBUSY;
3577 }
3578
3579 /* Remove an event_call */
trace_remove_event_call(struct trace_event_call * call)3580 int trace_remove_event_call(struct trace_event_call *call)
3581 {
3582 int ret;
3583
3584 lockdep_assert_held(&event_mutex);
3585
3586 mutex_lock(&trace_types_lock);
3587 down_write(&trace_event_sem);
3588 ret = probe_remove_event_call(call);
3589 up_write(&trace_event_sem);
3590 mutex_unlock(&trace_types_lock);
3591
3592 return ret;
3593 }
3594 EXPORT_SYMBOL_GPL(trace_remove_event_call);
3595
3596 #define for_each_event(event, start, end) \
3597 for (event = start; \
3598 (unsigned long)event < (unsigned long)end; \
3599 event++)
3600
3601 #ifdef CONFIG_MODULES
update_mod_cache(struct trace_array * tr,struct module * mod)3602 static void update_mod_cache(struct trace_array *tr, struct module *mod)
3603 {
3604 struct event_mod_load *event_mod, *n;
3605
3606 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
3607 if (strcmp(event_mod->module, mod->name) != 0)
3608 continue;
3609
3610 __ftrace_set_clr_event_nolock(tr, event_mod->match,
3611 event_mod->system,
3612 event_mod->event, 1, mod->name);
3613 free_event_mod(event_mod);
3614 }
3615 }
3616
update_cache_events(struct module * mod)3617 static void update_cache_events(struct module *mod)
3618 {
3619 struct trace_array *tr;
3620
3621 list_for_each_entry(tr, &ftrace_trace_arrays, list)
3622 update_mod_cache(tr, mod);
3623 }
3624
trace_module_add_events(struct module * mod)3625 static void trace_module_add_events(struct module *mod)
3626 {
3627 struct trace_event_call **call, **start, **end;
3628
3629 if (!mod->num_trace_events)
3630 return;
3631
3632 /* Don't add infrastructure for mods without tracepoints */
3633 if (trace_module_has_bad_taint(mod)) {
3634 pr_err("%s: module has bad taint, not creating trace events\n",
3635 mod->name);
3636 return;
3637 }
3638
3639 start = mod->trace_events;
3640 end = mod->trace_events + mod->num_trace_events;
3641
3642 for_each_event(call, start, end) {
3643 __register_event(*call, mod);
3644 __add_event_to_tracers(*call);
3645 }
3646
3647 update_cache_events(mod);
3648 }
3649
trace_module_remove_events(struct module * mod)3650 static void trace_module_remove_events(struct module *mod)
3651 {
3652 struct trace_event_call *call, *p;
3653 struct module_string *modstr, *m;
3654
3655 down_write(&trace_event_sem);
3656 list_for_each_entry_safe(call, p, &ftrace_events, list) {
3657 if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3658 continue;
3659 if (call->module == mod)
3660 __trace_remove_event_call(call);
3661 }
3662 /* Check for any strings allocade for this module */
3663 list_for_each_entry_safe(modstr, m, &module_strings, next) {
3664 if (modstr->module != mod)
3665 continue;
3666 list_del(&modstr->next);
3667 kfree(modstr->str);
3668 kfree(modstr);
3669 }
3670 up_write(&trace_event_sem);
3671
3672 /*
3673 * It is safest to reset the ring buffer if the module being unloaded
3674 * registered any events that were used. The only worry is if
3675 * a new module gets loaded, and takes on the same id as the events
3676 * of this module. When printing out the buffer, traced events left
3677 * over from this module may be passed to the new module events and
3678 * unexpected results may occur.
3679 */
3680 tracing_reset_all_online_cpus_unlocked();
3681 }
3682
trace_module_notify(struct notifier_block * self,unsigned long val,void * data)3683 static int trace_module_notify(struct notifier_block *self,
3684 unsigned long val, void *data)
3685 {
3686 struct module *mod = data;
3687
3688 mutex_lock(&event_mutex);
3689 mutex_lock(&trace_types_lock);
3690 switch (val) {
3691 case MODULE_STATE_COMING:
3692 trace_module_add_events(mod);
3693 break;
3694 case MODULE_STATE_GOING:
3695 trace_module_remove_events(mod);
3696 break;
3697 }
3698 mutex_unlock(&trace_types_lock);
3699 mutex_unlock(&event_mutex);
3700
3701 return NOTIFY_OK;
3702 }
3703
3704 static struct notifier_block trace_module_nb = {
3705 .notifier_call = trace_module_notify,
3706 .priority = 1, /* higher than trace.c module notify */
3707 };
3708 #endif /* CONFIG_MODULES */
3709
3710 /* Create a new event directory structure for a trace directory. */
3711 static void
__trace_add_event_dirs(struct trace_array * tr)3712 __trace_add_event_dirs(struct trace_array *tr)
3713 {
3714 struct trace_event_call *call;
3715 int ret;
3716
3717 list_for_each_entry(call, &ftrace_events, list) {
3718 ret = __trace_add_new_event(call, tr);
3719 if (ret < 0)
3720 pr_warn("Could not create directory for event %s\n",
3721 trace_event_name(call));
3722 }
3723 }
3724
3725 /* Returns any file that matches the system and event */
3726 struct trace_event_file *
__find_event_file(struct trace_array * tr,const char * system,const char * event)3727 __find_event_file(struct trace_array *tr, const char *system, const char *event)
3728 {
3729 struct trace_event_file *file;
3730 struct trace_event_call *call;
3731 const char *name;
3732
3733 list_for_each_entry(file, &tr->events, list) {
3734
3735 call = file->event_call;
3736 name = trace_event_name(call);
3737
3738 if (!name || !call->class)
3739 continue;
3740
3741 if (strcmp(event, name) == 0 &&
3742 strcmp(system, call->class->system) == 0)
3743 return file;
3744 }
3745 return NULL;
3746 }
3747
3748 /* Returns valid trace event files that match system and event */
3749 struct trace_event_file *
find_event_file(struct trace_array * tr,const char * system,const char * event)3750 find_event_file(struct trace_array *tr, const char *system, const char *event)
3751 {
3752 struct trace_event_file *file;
3753
3754 file = __find_event_file(tr, system, event);
3755 if (!file || !file->event_call->class->reg ||
3756 file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3757 return NULL;
3758
3759 return file;
3760 }
3761
3762 /**
3763 * trace_get_event_file - Find and return a trace event file
3764 * @instance: The name of the trace instance containing the event
3765 * @system: The name of the system containing the event
3766 * @event: The name of the event
3767 *
3768 * Return a trace event file given the trace instance name, trace
3769 * system, and trace event name. If the instance name is NULL, it
3770 * refers to the top-level trace array.
3771 *
3772 * This function will look it up and return it if found, after calling
3773 * trace_array_get() to prevent the instance from going away, and
3774 * increment the event's module refcount to prevent it from being
3775 * removed.
3776 *
3777 * To release the file, call trace_put_event_file(), which will call
3778 * trace_array_put() and decrement the event's module refcount.
3779 *
3780 * Return: The trace event on success, ERR_PTR otherwise.
3781 */
trace_get_event_file(const char * instance,const char * system,const char * event)3782 struct trace_event_file *trace_get_event_file(const char *instance,
3783 const char *system,
3784 const char *event)
3785 {
3786 struct trace_array *tr = top_trace_array();
3787 struct trace_event_file *file = NULL;
3788 int ret = -EINVAL;
3789
3790 if (instance) {
3791 tr = trace_array_find_get(instance);
3792 if (!tr)
3793 return ERR_PTR(-ENOENT);
3794 } else {
3795 ret = trace_array_get(tr);
3796 if (ret)
3797 return ERR_PTR(ret);
3798 }
3799
3800 guard(mutex)(&event_mutex);
3801
3802 file = find_event_file(tr, system, event);
3803 if (!file) {
3804 trace_array_put(tr);
3805 return ERR_PTR(-EINVAL);
3806 }
3807
3808 /* Don't let event modules unload while in use */
3809 ret = trace_event_try_get_ref(file->event_call);
3810 if (!ret) {
3811 trace_array_put(tr);
3812 return ERR_PTR(-EBUSY);
3813 }
3814
3815 return file;
3816 }
3817 EXPORT_SYMBOL_GPL(trace_get_event_file);
3818
3819 /**
3820 * trace_put_event_file - Release a file from trace_get_event_file()
3821 * @file: The trace event file
3822 *
3823 * If a file was retrieved using trace_get_event_file(), this should
3824 * be called when it's no longer needed. It will cancel the previous
3825 * trace_array_get() called by that function, and decrement the
3826 * event's module refcount.
3827 */
trace_put_event_file(struct trace_event_file * file)3828 void trace_put_event_file(struct trace_event_file *file)
3829 {
3830 mutex_lock(&event_mutex);
3831 trace_event_put_ref(file->event_call);
3832 mutex_unlock(&event_mutex);
3833
3834 trace_array_put(file->tr);
3835 }
3836 EXPORT_SYMBOL_GPL(trace_put_event_file);
3837
3838 #ifdef CONFIG_DYNAMIC_FTRACE
3839
3840 /* Avoid typos */
3841 #define ENABLE_EVENT_STR "enable_event"
3842 #define DISABLE_EVENT_STR "disable_event"
3843
3844 struct event_probe_data {
3845 struct trace_event_file *file;
3846 unsigned long count;
3847 int ref;
3848 bool enable;
3849 };
3850
update_event_probe(struct event_probe_data * data)3851 static void update_event_probe(struct event_probe_data *data)
3852 {
3853 if (data->enable)
3854 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3855 else
3856 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3857 }
3858
3859 static void
event_enable_probe(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)3860 event_enable_probe(unsigned long ip, unsigned long parent_ip,
3861 struct trace_array *tr, struct ftrace_probe_ops *ops,
3862 void *data)
3863 {
3864 struct ftrace_func_mapper *mapper = data;
3865 struct event_probe_data *edata;
3866 void **pdata;
3867
3868 pdata = ftrace_func_mapper_find_ip(mapper, ip);
3869 if (!pdata || !*pdata)
3870 return;
3871
3872 edata = *pdata;
3873 update_event_probe(edata);
3874 }
3875
3876 static void
event_enable_count_probe(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)3877 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
3878 struct trace_array *tr, struct ftrace_probe_ops *ops,
3879 void *data)
3880 {
3881 struct ftrace_func_mapper *mapper = data;
3882 struct event_probe_data *edata;
3883 void **pdata;
3884
3885 pdata = ftrace_func_mapper_find_ip(mapper, ip);
3886 if (!pdata || !*pdata)
3887 return;
3888
3889 edata = *pdata;
3890
3891 if (!edata->count)
3892 return;
3893
3894 /* Skip if the event is in a state we want to switch to */
3895 if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
3896 return;
3897
3898 if (edata->count != -1)
3899 (edata->count)--;
3900
3901 update_event_probe(edata);
3902 }
3903
3904 static int
event_enable_print(struct seq_file * m,unsigned long ip,struct ftrace_probe_ops * ops,void * data)3905 event_enable_print(struct seq_file *m, unsigned long ip,
3906 struct ftrace_probe_ops *ops, void *data)
3907 {
3908 struct ftrace_func_mapper *mapper = data;
3909 struct event_probe_data *edata;
3910 void **pdata;
3911
3912 pdata = ftrace_func_mapper_find_ip(mapper, ip);
3913
3914 if (WARN_ON_ONCE(!pdata || !*pdata))
3915 return 0;
3916
3917 edata = *pdata;
3918
3919 seq_printf(m, "%ps:", (void *)ip);
3920
3921 seq_printf(m, "%s:%s:%s",
3922 edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
3923 edata->file->event_call->class->system,
3924 trace_event_name(edata->file->event_call));
3925
3926 if (edata->count == -1)
3927 seq_puts(m, ":unlimited\n");
3928 else
3929 seq_printf(m, ":count=%ld\n", edata->count);
3930
3931 return 0;
3932 }
3933
3934 static int
event_enable_init(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * init_data,void ** data)3935 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
3936 unsigned long ip, void *init_data, void **data)
3937 {
3938 struct ftrace_func_mapper *mapper = *data;
3939 struct event_probe_data *edata = init_data;
3940 int ret;
3941
3942 if (!mapper) {
3943 mapper = allocate_ftrace_func_mapper();
3944 if (!mapper)
3945 return -ENODEV;
3946 *data = mapper;
3947 }
3948
3949 ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
3950 if (ret < 0)
3951 return ret;
3952
3953 edata->ref++;
3954
3955 return 0;
3956 }
3957
free_probe_data(void * data)3958 static int free_probe_data(void *data)
3959 {
3960 struct event_probe_data *edata = data;
3961
3962 edata->ref--;
3963 if (!edata->ref) {
3964 /* Remove the SOFT_MODE flag */
3965 __ftrace_event_enable_disable(edata->file, 0, 1);
3966 trace_event_put_ref(edata->file->event_call);
3967 kfree(edata);
3968 }
3969 return 0;
3970 }
3971
3972 static void
event_enable_free(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * data)3973 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
3974 unsigned long ip, void *data)
3975 {
3976 struct ftrace_func_mapper *mapper = data;
3977 struct event_probe_data *edata;
3978
3979 if (!ip) {
3980 if (!mapper)
3981 return;
3982 free_ftrace_func_mapper(mapper, free_probe_data);
3983 return;
3984 }
3985
3986 edata = ftrace_func_mapper_remove_ip(mapper, ip);
3987
3988 if (WARN_ON_ONCE(!edata))
3989 return;
3990
3991 if (WARN_ON_ONCE(edata->ref <= 0))
3992 return;
3993
3994 free_probe_data(edata);
3995 }
3996
3997 static struct ftrace_probe_ops event_enable_probe_ops = {
3998 .func = event_enable_probe,
3999 .print = event_enable_print,
4000 .init = event_enable_init,
4001 .free = event_enable_free,
4002 };
4003
4004 static struct ftrace_probe_ops event_enable_count_probe_ops = {
4005 .func = event_enable_count_probe,
4006 .print = event_enable_print,
4007 .init = event_enable_init,
4008 .free = event_enable_free,
4009 };
4010
4011 static struct ftrace_probe_ops event_disable_probe_ops = {
4012 .func = event_enable_probe,
4013 .print = event_enable_print,
4014 .init = event_enable_init,
4015 .free = event_enable_free,
4016 };
4017
4018 static struct ftrace_probe_ops event_disable_count_probe_ops = {
4019 .func = event_enable_count_probe,
4020 .print = event_enable_print,
4021 .init = event_enable_init,
4022 .free = event_enable_free,
4023 };
4024
4025 static int
event_enable_func(struct trace_array * tr,struct ftrace_hash * hash,char * glob,char * cmd,char * param,int enabled)4026 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
4027 char *glob, char *cmd, char *param, int enabled)
4028 {
4029 struct trace_event_file *file;
4030 struct ftrace_probe_ops *ops;
4031 struct event_probe_data *data;
4032 unsigned long count = -1;
4033 const char *system;
4034 const char *event;
4035 char *number;
4036 bool enable;
4037 int ret;
4038
4039 if (!tr)
4040 return -ENODEV;
4041
4042 /* hash funcs only work with set_ftrace_filter */
4043 if (!enabled || !param)
4044 return -EINVAL;
4045
4046 system = strsep(¶m, ":");
4047 if (!param)
4048 return -EINVAL;
4049
4050 event = strsep(¶m, ":");
4051
4052 guard(mutex)(&event_mutex);
4053
4054 file = find_event_file(tr, system, event);
4055 if (!file)
4056 return -EINVAL;
4057
4058 enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
4059
4060 if (enable)
4061 ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
4062 else
4063 ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
4064
4065 if (glob[0] == '!')
4066 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
4067
4068 if (param) {
4069 number = strsep(¶m, ":");
4070
4071 if (!strlen(number))
4072 return -EINVAL;
4073
4074 /*
4075 * We use the callback data field (which is a pointer)
4076 * as our counter.
4077 */
4078 ret = kstrtoul(number, 0, &count);
4079 if (ret)
4080 return ret;
4081 }
4082
4083 /* Don't let event modules unload while probe registered */
4084 ret = trace_event_try_get_ref(file->event_call);
4085 if (!ret)
4086 return -EBUSY;
4087
4088 ret = __ftrace_event_enable_disable(file, 1, 1);
4089 if (ret < 0)
4090 goto out_put;
4091
4092 ret = -ENOMEM;
4093 data = kzalloc(sizeof(*data), GFP_KERNEL);
4094 if (!data)
4095 goto out_put;
4096
4097 data->enable = enable;
4098 data->count = count;
4099 data->file = file;
4100
4101 ret = register_ftrace_function_probe(glob, tr, ops, data);
4102 /*
4103 * The above returns on success the # of functions enabled,
4104 * but if it didn't find any functions it returns zero.
4105 * Consider no functions a failure too.
4106 */
4107
4108 /* Just return zero, not the number of enabled functions */
4109 if (ret > 0)
4110 return 0;
4111
4112 kfree(data);
4113
4114 if (!ret)
4115 ret = -ENOENT;
4116
4117 __ftrace_event_enable_disable(file, 0, 1);
4118 out_put:
4119 trace_event_put_ref(file->event_call);
4120 return ret;
4121 }
4122
4123 static struct ftrace_func_command event_enable_cmd = {
4124 .name = ENABLE_EVENT_STR,
4125 .func = event_enable_func,
4126 };
4127
4128 static struct ftrace_func_command event_disable_cmd = {
4129 .name = DISABLE_EVENT_STR,
4130 .func = event_enable_func,
4131 };
4132
register_event_cmds(void)4133 static __init int register_event_cmds(void)
4134 {
4135 int ret;
4136
4137 ret = register_ftrace_command(&event_enable_cmd);
4138 if (WARN_ON(ret < 0))
4139 return ret;
4140 ret = register_ftrace_command(&event_disable_cmd);
4141 if (WARN_ON(ret < 0))
4142 unregister_ftrace_command(&event_enable_cmd);
4143 return ret;
4144 }
4145 #else
register_event_cmds(void)4146 static inline int register_event_cmds(void) { return 0; }
4147 #endif /* CONFIG_DYNAMIC_FTRACE */
4148
4149 /*
4150 * The top level array and trace arrays created by boot-time tracing
4151 * have already had its trace_event_file descriptors created in order
4152 * to allow for early events to be recorded.
4153 * This function is called after the tracefs has been initialized,
4154 * and we now have to create the files associated to the events.
4155 */
__trace_early_add_event_dirs(struct trace_array * tr)4156 static void __trace_early_add_event_dirs(struct trace_array *tr)
4157 {
4158 struct trace_event_file *file;
4159 int ret;
4160
4161
4162 list_for_each_entry(file, &tr->events, list) {
4163 ret = event_create_dir(tr->event_dir, file);
4164 if (ret < 0)
4165 pr_warn("Could not create directory for event %s\n",
4166 trace_event_name(file->event_call));
4167 }
4168 }
4169
4170 /*
4171 * For early boot up, the top trace array and the trace arrays created
4172 * by boot-time tracing require to have a list of events that can be
4173 * enabled. This must be done before the filesystem is set up in order
4174 * to allow events to be traced early.
4175 */
__trace_early_add_events(struct trace_array * tr)4176 void __trace_early_add_events(struct trace_array *tr)
4177 {
4178 struct trace_event_call *call;
4179 int ret;
4180
4181 list_for_each_entry(call, &ftrace_events, list) {
4182 /* Early boot up should not have any modules loaded */
4183 if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
4184 WARN_ON_ONCE(call->module))
4185 continue;
4186
4187 ret = __trace_early_add_new_event(call, tr);
4188 if (ret < 0)
4189 pr_warn("Could not create early event %s\n",
4190 trace_event_name(call));
4191 }
4192 }
4193
4194 /* Remove the event directory structure for a trace directory. */
4195 static void
__trace_remove_event_dirs(struct trace_array * tr)4196 __trace_remove_event_dirs(struct trace_array *tr)
4197 {
4198 struct trace_event_file *file, *next;
4199
4200 list_for_each_entry_safe(file, next, &tr->events, list)
4201 remove_event_file_dir(file);
4202 }
4203
__add_event_to_tracers(struct trace_event_call * call)4204 static void __add_event_to_tracers(struct trace_event_call *call)
4205 {
4206 struct trace_array *tr;
4207
4208 list_for_each_entry(tr, &ftrace_trace_arrays, list)
4209 __trace_add_new_event(call, tr);
4210 }
4211
4212 extern struct trace_event_call *__start_ftrace_events[];
4213 extern struct trace_event_call *__stop_ftrace_events[];
4214
4215 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
4216
setup_trace_event(char * str)4217 static __init int setup_trace_event(char *str)
4218 {
4219 strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
4220 trace_set_ring_buffer_expanded(NULL);
4221 disable_tracing_selftest("running event tracing");
4222
4223 return 1;
4224 }
4225 __setup("trace_event=", setup_trace_event);
4226
events_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)4227 static int events_callback(const char *name, umode_t *mode, void **data,
4228 const struct file_operations **fops)
4229 {
4230 if (strcmp(name, "enable") == 0) {
4231 *mode = TRACE_MODE_WRITE;
4232 *fops = &ftrace_tr_enable_fops;
4233 return 1;
4234 }
4235
4236 if (strcmp(name, "header_page") == 0) {
4237 *mode = TRACE_MODE_READ;
4238 *fops = &ftrace_show_header_page_fops;
4239
4240 } else if (strcmp(name, "header_event") == 0) {
4241 *mode = TRACE_MODE_READ;
4242 *fops = &ftrace_show_header_event_fops;
4243 } else
4244 return 0;
4245
4246 return 1;
4247 }
4248
4249 /* Expects to have event_mutex held when called */
4250 static int
create_event_toplevel_files(struct dentry * parent,struct trace_array * tr)4251 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
4252 {
4253 struct eventfs_inode *e_events;
4254 struct dentry *entry;
4255 int nr_entries;
4256 static struct eventfs_entry events_entries[] = {
4257 {
4258 .name = "enable",
4259 .callback = events_callback,
4260 },
4261 {
4262 .name = "header_page",
4263 .callback = events_callback,
4264 },
4265 {
4266 .name = "header_event",
4267 .callback = events_callback,
4268 },
4269 };
4270
4271 entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
4272 tr, &ftrace_set_event_fops);
4273 if (!entry)
4274 return -ENOMEM;
4275
4276 nr_entries = ARRAY_SIZE(events_entries);
4277
4278 e_events = eventfs_create_events_dir("events", parent, events_entries,
4279 nr_entries, tr);
4280 if (IS_ERR(e_events)) {
4281 pr_warn("Could not create tracefs 'events' directory\n");
4282 return -ENOMEM;
4283 }
4284
4285 /* There are not as crucial, just warn if they are not created */
4286
4287 trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
4288 tr, &ftrace_set_event_pid_fops);
4289
4290 trace_create_file("set_event_notrace_pid",
4291 TRACE_MODE_WRITE, parent, tr,
4292 &ftrace_set_event_notrace_pid_fops);
4293
4294 tr->event_dir = e_events;
4295
4296 return 0;
4297 }
4298
4299 /**
4300 * event_trace_add_tracer - add a instance of a trace_array to events
4301 * @parent: The parent dentry to place the files/directories for events in
4302 * @tr: The trace array associated with these events
4303 *
4304 * When a new instance is created, it needs to set up its events
4305 * directory, as well as other files associated with events. It also
4306 * creates the event hierarchy in the @parent/events directory.
4307 *
4308 * Returns 0 on success.
4309 *
4310 * Must be called with event_mutex held.
4311 */
event_trace_add_tracer(struct dentry * parent,struct trace_array * tr)4312 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
4313 {
4314 int ret;
4315
4316 lockdep_assert_held(&event_mutex);
4317
4318 ret = create_event_toplevel_files(parent, tr);
4319 if (ret)
4320 goto out;
4321
4322 down_write(&trace_event_sem);
4323 /* If tr already has the event list, it is initialized in early boot. */
4324 if (unlikely(!list_empty(&tr->events)))
4325 __trace_early_add_event_dirs(tr);
4326 else
4327 __trace_add_event_dirs(tr);
4328 up_write(&trace_event_sem);
4329
4330 out:
4331 return ret;
4332 }
4333
4334 /*
4335 * The top trace array already had its file descriptors created.
4336 * Now the files themselves need to be created.
4337 */
4338 static __init int
early_event_add_tracer(struct dentry * parent,struct trace_array * tr)4339 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
4340 {
4341 int ret;
4342
4343 guard(mutex)(&event_mutex);
4344
4345 ret = create_event_toplevel_files(parent, tr);
4346 if (ret)
4347 return ret;
4348
4349 down_write(&trace_event_sem);
4350 __trace_early_add_event_dirs(tr);
4351 up_write(&trace_event_sem);
4352
4353 return 0;
4354 }
4355
4356 /* Must be called with event_mutex held */
event_trace_del_tracer(struct trace_array * tr)4357 int event_trace_del_tracer(struct trace_array *tr)
4358 {
4359 lockdep_assert_held(&event_mutex);
4360
4361 /* Disable any event triggers and associated soft-disabled events */
4362 clear_event_triggers(tr);
4363
4364 /* Clear the pid list */
4365 __ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
4366
4367 /* Disable any running events */
4368 __ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0, NULL);
4369
4370 /* Make sure no more events are being executed */
4371 tracepoint_synchronize_unregister();
4372
4373 down_write(&trace_event_sem);
4374 __trace_remove_event_dirs(tr);
4375 eventfs_remove_events_dir(tr->event_dir);
4376 up_write(&trace_event_sem);
4377
4378 tr->event_dir = NULL;
4379
4380 return 0;
4381 }
4382
event_trace_memsetup(void)4383 static __init int event_trace_memsetup(void)
4384 {
4385 field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
4386 file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
4387 return 0;
4388 }
4389
4390 __init void
early_enable_events(struct trace_array * tr,char * buf,bool disable_first)4391 early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
4392 {
4393 char *token;
4394 int ret;
4395
4396 while (true) {
4397 token = strsep(&buf, ",");
4398
4399 if (!token)
4400 break;
4401
4402 if (*token) {
4403 /* Restarting syscalls requires that we stop them first */
4404 if (disable_first)
4405 ftrace_set_clr_event(tr, token, 0);
4406
4407 ret = ftrace_set_clr_event(tr, token, 1);
4408 if (ret)
4409 pr_warn("Failed to enable trace event: %s\n", token);
4410 }
4411
4412 /* Put back the comma to allow this to be called again */
4413 if (buf)
4414 *(buf - 1) = ',';
4415 }
4416 }
4417
event_trace_enable(void)4418 static __init int event_trace_enable(void)
4419 {
4420 struct trace_array *tr = top_trace_array();
4421 struct trace_event_call **iter, *call;
4422 int ret;
4423
4424 if (!tr)
4425 return -ENODEV;
4426
4427 for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
4428
4429 call = *iter;
4430 ret = event_init(call);
4431 if (!ret)
4432 list_add(&call->list, &ftrace_events);
4433 }
4434
4435 register_trigger_cmds();
4436
4437 /*
4438 * We need the top trace array to have a working set of trace
4439 * points at early init, before the debug files and directories
4440 * are created. Create the file entries now, and attach them
4441 * to the actual file dentries later.
4442 */
4443 __trace_early_add_events(tr);
4444
4445 early_enable_events(tr, bootup_event_buf, false);
4446
4447 trace_printk_start_comm();
4448
4449 register_event_cmds();
4450
4451
4452 return 0;
4453 }
4454
4455 /*
4456 * event_trace_enable() is called from trace_event_init() first to
4457 * initialize events and perhaps start any events that are on the
4458 * command line. Unfortunately, there are some events that will not
4459 * start this early, like the system call tracepoints that need
4460 * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
4461 * event_trace_enable() is called before pid 1 starts, and this flag
4462 * is never set, making the syscall tracepoint never get reached, but
4463 * the event is enabled regardless (and not doing anything).
4464 */
event_trace_enable_again(void)4465 static __init int event_trace_enable_again(void)
4466 {
4467 struct trace_array *tr;
4468
4469 tr = top_trace_array();
4470 if (!tr)
4471 return -ENODEV;
4472
4473 early_enable_events(tr, bootup_event_buf, true);
4474
4475 return 0;
4476 }
4477
4478 early_initcall(event_trace_enable_again);
4479
4480 /* Init fields which doesn't related to the tracefs */
event_trace_init_fields(void)4481 static __init int event_trace_init_fields(void)
4482 {
4483 if (trace_define_generic_fields())
4484 pr_warn("tracing: Failed to allocated generic fields");
4485
4486 if (trace_define_common_fields())
4487 pr_warn("tracing: Failed to allocate common fields");
4488
4489 return 0;
4490 }
4491
event_trace_init(void)4492 __init int event_trace_init(void)
4493 {
4494 struct trace_array *tr;
4495 int ret;
4496
4497 tr = top_trace_array();
4498 if (!tr)
4499 return -ENODEV;
4500
4501 trace_create_file("available_events", TRACE_MODE_READ,
4502 NULL, tr, &ftrace_avail_fops);
4503
4504 ret = early_event_add_tracer(NULL, tr);
4505 if (ret)
4506 return ret;
4507
4508 #ifdef CONFIG_MODULES
4509 ret = register_module_notifier(&trace_module_nb);
4510 if (ret)
4511 pr_warn("Failed to register trace events module notifier\n");
4512 #endif
4513
4514 eventdir_initialized = true;
4515
4516 return 0;
4517 }
4518
trace_event_init(void)4519 void __init trace_event_init(void)
4520 {
4521 event_trace_memsetup();
4522 init_ftrace_syscalls();
4523 event_trace_enable();
4524 event_trace_init_fields();
4525 }
4526
4527 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4528
4529 static DEFINE_SPINLOCK(test_spinlock);
4530 static DEFINE_SPINLOCK(test_spinlock_irq);
4531 static DEFINE_MUTEX(test_mutex);
4532
test_work(struct work_struct * dummy)4533 static __init void test_work(struct work_struct *dummy)
4534 {
4535 spin_lock(&test_spinlock);
4536 spin_lock_irq(&test_spinlock_irq);
4537 udelay(1);
4538 spin_unlock_irq(&test_spinlock_irq);
4539 spin_unlock(&test_spinlock);
4540
4541 mutex_lock(&test_mutex);
4542 msleep(1);
4543 mutex_unlock(&test_mutex);
4544 }
4545
event_test_thread(void * unused)4546 static __init int event_test_thread(void *unused)
4547 {
4548 void *test_malloc;
4549
4550 test_malloc = kmalloc(1234, GFP_KERNEL);
4551 if (!test_malloc)
4552 pr_info("failed to kmalloc\n");
4553
4554 schedule_on_each_cpu(test_work);
4555
4556 kfree(test_malloc);
4557
4558 set_current_state(TASK_INTERRUPTIBLE);
4559 while (!kthread_should_stop()) {
4560 schedule();
4561 set_current_state(TASK_INTERRUPTIBLE);
4562 }
4563 __set_current_state(TASK_RUNNING);
4564
4565 return 0;
4566 }
4567
4568 /*
4569 * Do various things that may trigger events.
4570 */
event_test_stuff(void)4571 static __init void event_test_stuff(void)
4572 {
4573 struct task_struct *test_thread;
4574
4575 test_thread = kthread_run(event_test_thread, NULL, "test-events");
4576 msleep(1);
4577 kthread_stop(test_thread);
4578 }
4579
4580 /*
4581 * For every trace event defined, we will test each trace point separately,
4582 * and then by groups, and finally all trace points.
4583 */
event_trace_self_tests(void)4584 static __init void event_trace_self_tests(void)
4585 {
4586 struct trace_subsystem_dir *dir;
4587 struct trace_event_file *file;
4588 struct trace_event_call *call;
4589 struct event_subsystem *system;
4590 struct trace_array *tr;
4591 int ret;
4592
4593 tr = top_trace_array();
4594 if (!tr)
4595 return;
4596
4597 pr_info("Running tests on trace events:\n");
4598
4599 list_for_each_entry(file, &tr->events, list) {
4600
4601 call = file->event_call;
4602
4603 /* Only test those that have a probe */
4604 if (!call->class || !call->class->probe)
4605 continue;
4606
4607 /*
4608 * Testing syscall events here is pretty useless, but
4609 * we still do it if configured. But this is time consuming.
4610 * What we really need is a user thread to perform the
4611 * syscalls as we test.
4612 */
4613 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
4614 if (call->class->system &&
4615 strcmp(call->class->system, "syscalls") == 0)
4616 continue;
4617 #endif
4618
4619 pr_info("Testing event %s: ", trace_event_name(call));
4620
4621 /*
4622 * If an event is already enabled, someone is using
4623 * it and the self test should not be on.
4624 */
4625 if (file->flags & EVENT_FILE_FL_ENABLED) {
4626 pr_warn("Enabled event during self test!\n");
4627 WARN_ON_ONCE(1);
4628 continue;
4629 }
4630
4631 ftrace_event_enable_disable(file, 1);
4632 event_test_stuff();
4633 ftrace_event_enable_disable(file, 0);
4634
4635 pr_cont("OK\n");
4636 }
4637
4638 /* Now test at the sub system level */
4639
4640 pr_info("Running tests on trace event systems:\n");
4641
4642 list_for_each_entry(dir, &tr->systems, list) {
4643
4644 system = dir->subsystem;
4645
4646 /* the ftrace system is special, skip it */
4647 if (strcmp(system->name, "ftrace") == 0)
4648 continue;
4649
4650 pr_info("Testing event system %s: ", system->name);
4651
4652 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1, NULL);
4653 if (WARN_ON_ONCE(ret)) {
4654 pr_warn("error enabling system %s\n",
4655 system->name);
4656 continue;
4657 }
4658
4659 event_test_stuff();
4660
4661 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0, NULL);
4662 if (WARN_ON_ONCE(ret)) {
4663 pr_warn("error disabling system %s\n",
4664 system->name);
4665 continue;
4666 }
4667
4668 pr_cont("OK\n");
4669 }
4670
4671 /* Test with all events enabled */
4672
4673 pr_info("Running tests on all trace events:\n");
4674 pr_info("Testing all events: ");
4675
4676 ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1, NULL);
4677 if (WARN_ON_ONCE(ret)) {
4678 pr_warn("error enabling all events\n");
4679 return;
4680 }
4681
4682 event_test_stuff();
4683
4684 /* reset sysname */
4685 ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0, NULL);
4686 if (WARN_ON_ONCE(ret)) {
4687 pr_warn("error disabling all events\n");
4688 return;
4689 }
4690
4691 pr_cont("OK\n");
4692 }
4693
4694 #ifdef CONFIG_FUNCTION_TRACER
4695
4696 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4697
4698 static struct trace_event_file event_trace_file __initdata;
4699
4700 static void __init
function_test_events_call(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * regs)4701 function_test_events_call(unsigned long ip, unsigned long parent_ip,
4702 struct ftrace_ops *op, struct ftrace_regs *regs)
4703 {
4704 struct trace_buffer *buffer;
4705 struct ring_buffer_event *event;
4706 struct ftrace_entry *entry;
4707 unsigned int trace_ctx;
4708 long disabled;
4709 int cpu;
4710
4711 trace_ctx = tracing_gen_ctx();
4712 preempt_disable_notrace();
4713 cpu = raw_smp_processor_id();
4714 disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4715
4716 if (disabled != 1)
4717 goto out;
4718
4719 event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4720 TRACE_FN, sizeof(*entry),
4721 trace_ctx);
4722 if (!event)
4723 goto out;
4724 entry = ring_buffer_event_data(event);
4725 entry->ip = ip;
4726 entry->parent_ip = parent_ip;
4727
4728 event_trigger_unlock_commit(&event_trace_file, buffer, event,
4729 entry, trace_ctx);
4730 out:
4731 atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4732 preempt_enable_notrace();
4733 }
4734
4735 static struct ftrace_ops trace_ops __initdata =
4736 {
4737 .func = function_test_events_call,
4738 };
4739
event_trace_self_test_with_function(void)4740 static __init void event_trace_self_test_with_function(void)
4741 {
4742 int ret;
4743
4744 event_trace_file.tr = top_trace_array();
4745 if (WARN_ON(!event_trace_file.tr))
4746 return;
4747
4748 ret = register_ftrace_function(&trace_ops);
4749 if (WARN_ON(ret < 0)) {
4750 pr_info("Failed to enable function tracer for event tests\n");
4751 return;
4752 }
4753 pr_info("Running tests again, along with the function tracer\n");
4754 event_trace_self_tests();
4755 unregister_ftrace_function(&trace_ops);
4756 }
4757 #else
event_trace_self_test_with_function(void)4758 static __init void event_trace_self_test_with_function(void)
4759 {
4760 }
4761 #endif
4762
event_trace_self_tests_init(void)4763 static __init int event_trace_self_tests_init(void)
4764 {
4765 if (!tracing_selftest_disabled) {
4766 event_trace_self_tests();
4767 event_trace_self_test_with_function();
4768 }
4769
4770 return 0;
4771 }
4772
4773 late_initcall(event_trace_self_tests_init);
4774
4775 #endif
4776