xref: /linux/kernel/trace/trace_events.c (revision 744fab2d9ff9177a27647c3710e86d43f2efe68c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * event tracer
4  *
5  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
6  *
7  *  - Added format output of fields of the trace point.
8  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
9  *
10  */
11 
12 #define pr_fmt(fmt) fmt
13 
14 #include <linux/workqueue.h>
15 #include <linux/security.h>
16 #include <linux/spinlock.h>
17 #include <linux/kthread.h>
18 #include <linux/tracefs.h>
19 #include <linux/uaccess.h>
20 #include <linux/module.h>
21 #include <linux/ctype.h>
22 #include <linux/sort.h>
23 #include <linux/slab.h>
24 #include <linux/delay.h>
25 
26 #include <trace/events/sched.h>
27 #include <trace/syscall.h>
28 
29 #include <asm/setup.h>
30 
31 #include "trace_output.h"
32 
33 #undef TRACE_SYSTEM
34 #define TRACE_SYSTEM "TRACE_SYSTEM"
35 
36 DEFINE_MUTEX(event_mutex);
37 
38 LIST_HEAD(ftrace_events);
39 static LIST_HEAD(ftrace_generic_fields);
40 static LIST_HEAD(ftrace_common_fields);
41 static bool eventdir_initialized;
42 
43 static LIST_HEAD(module_strings);
44 
45 struct module_string {
46 	struct list_head	next;
47 	struct module		*module;
48 	char			*str;
49 };
50 
51 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
52 
53 static struct kmem_cache *field_cachep;
54 static struct kmem_cache *file_cachep;
55 
system_refcount(struct event_subsystem * system)56 static inline int system_refcount(struct event_subsystem *system)
57 {
58 	return system->ref_count;
59 }
60 
system_refcount_inc(struct event_subsystem * system)61 static int system_refcount_inc(struct event_subsystem *system)
62 {
63 	return system->ref_count++;
64 }
65 
system_refcount_dec(struct event_subsystem * system)66 static int system_refcount_dec(struct event_subsystem *system)
67 {
68 	return --system->ref_count;
69 }
70 
71 /* Double loops, do not use break, only goto's work */
72 #define do_for_each_event_file(tr, file)			\
73 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
74 		list_for_each_entry(file, &tr->events, list)
75 
76 #define do_for_each_event_file_safe(tr, file)			\
77 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
78 		struct trace_event_file *___n;				\
79 		list_for_each_entry_safe(file, ___n, &tr->events, list)
80 
81 #define while_for_each_event_file()		\
82 	}
83 
84 static struct ftrace_event_field *
__find_event_field(struct list_head * head,const char * name)85 __find_event_field(struct list_head *head, const char *name)
86 {
87 	struct ftrace_event_field *field;
88 
89 	list_for_each_entry(field, head, link) {
90 		if (!strcmp(field->name, name))
91 			return field;
92 	}
93 
94 	return NULL;
95 }
96 
97 struct ftrace_event_field *
trace_find_event_field(struct trace_event_call * call,char * name)98 trace_find_event_field(struct trace_event_call *call, char *name)
99 {
100 	struct ftrace_event_field *field;
101 	struct list_head *head;
102 
103 	head = trace_get_fields(call);
104 	field = __find_event_field(head, name);
105 	if (field)
106 		return field;
107 
108 	field = __find_event_field(&ftrace_generic_fields, name);
109 	if (field)
110 		return field;
111 
112 	return __find_event_field(&ftrace_common_fields, name);
113 }
114 
__trace_define_field(struct list_head * head,const char * type,const char * name,int offset,int size,int is_signed,int filter_type,int len,int need_test)115 static int __trace_define_field(struct list_head *head, const char *type,
116 				const char *name, int offset, int size,
117 				int is_signed, int filter_type, int len,
118 				int need_test)
119 {
120 	struct ftrace_event_field *field;
121 
122 	field = kmem_cache_alloc(field_cachep, GFP_TRACE);
123 	if (!field)
124 		return -ENOMEM;
125 
126 	field->name = name;
127 	field->type = type;
128 
129 	if (filter_type == FILTER_OTHER)
130 		field->filter_type = filter_assign_type(type);
131 	else
132 		field->filter_type = filter_type;
133 
134 	field->offset = offset;
135 	field->size = size;
136 	field->is_signed = is_signed;
137 	field->needs_test = need_test;
138 	field->len = len;
139 
140 	list_add(&field->link, head);
141 
142 	return 0;
143 }
144 
trace_define_field(struct trace_event_call * call,const char * type,const char * name,int offset,int size,int is_signed,int filter_type)145 int trace_define_field(struct trace_event_call *call, const char *type,
146 		       const char *name, int offset, int size, int is_signed,
147 		       int filter_type)
148 {
149 	struct list_head *head;
150 
151 	if (WARN_ON(!call->class))
152 		return 0;
153 
154 	head = trace_get_fields(call);
155 	return __trace_define_field(head, type, name, offset, size,
156 				    is_signed, filter_type, 0, 0);
157 }
158 EXPORT_SYMBOL_GPL(trace_define_field);
159 
trace_define_field_ext(struct trace_event_call * call,const char * type,const char * name,int offset,int size,int is_signed,int filter_type,int len,int need_test)160 static int trace_define_field_ext(struct trace_event_call *call, const char *type,
161 		       const char *name, int offset, int size, int is_signed,
162 		       int filter_type, int len, int need_test)
163 {
164 	struct list_head *head;
165 
166 	if (WARN_ON(!call->class))
167 		return 0;
168 
169 	head = trace_get_fields(call);
170 	return __trace_define_field(head, type, name, offset, size,
171 				    is_signed, filter_type, len, need_test);
172 }
173 
174 #define __generic_field(type, item, filter_type)			\
175 	ret = __trace_define_field(&ftrace_generic_fields, #type,	\
176 				   #item, 0, 0, is_signed_type(type),	\
177 				   filter_type, 0, 0);			\
178 	if (ret)							\
179 		return ret;
180 
181 #define __common_field(type, item)					\
182 	ret = __trace_define_field(&ftrace_common_fields, #type,	\
183 				   "common_" #item,			\
184 				   offsetof(typeof(ent), item),		\
185 				   sizeof(ent.item),			\
186 				   is_signed_type(type), FILTER_OTHER,	\
187 				   0, 0);				\
188 	if (ret)							\
189 		return ret;
190 
trace_define_generic_fields(void)191 static int trace_define_generic_fields(void)
192 {
193 	int ret;
194 
195 	__generic_field(int, CPU, FILTER_CPU);
196 	__generic_field(int, cpu, FILTER_CPU);
197 	__generic_field(int, common_cpu, FILTER_CPU);
198 	__generic_field(char *, COMM, FILTER_COMM);
199 	__generic_field(char *, comm, FILTER_COMM);
200 	__generic_field(char *, stacktrace, FILTER_STACKTRACE);
201 	__generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
202 
203 	return ret;
204 }
205 
trace_define_common_fields(void)206 static int trace_define_common_fields(void)
207 {
208 	int ret;
209 	struct trace_entry ent;
210 
211 	__common_field(unsigned short, type);
212 	__common_field(unsigned char, flags);
213 	/* Holds both preempt_count and migrate_disable */
214 	__common_field(unsigned char, preempt_count);
215 	__common_field(int, pid);
216 
217 	return ret;
218 }
219 
trace_destroy_fields(struct trace_event_call * call)220 static void trace_destroy_fields(struct trace_event_call *call)
221 {
222 	struct ftrace_event_field *field, *next;
223 	struct list_head *head;
224 
225 	head = trace_get_fields(call);
226 	list_for_each_entry_safe(field, next, head, link) {
227 		list_del(&field->link);
228 		kmem_cache_free(field_cachep, field);
229 	}
230 }
231 
232 /*
233  * run-time version of trace_event_get_offsets_<call>() that returns the last
234  * accessible offset of trace fields excluding __dynamic_array bytes
235  */
trace_event_get_offsets(struct trace_event_call * call)236 int trace_event_get_offsets(struct trace_event_call *call)
237 {
238 	struct ftrace_event_field *tail;
239 	struct list_head *head;
240 
241 	head = trace_get_fields(call);
242 	/*
243 	 * head->next points to the last field with the largest offset,
244 	 * since it was added last by trace_define_field()
245 	 */
246 	tail = list_first_entry(head, struct ftrace_event_field, link);
247 	return tail->offset + tail->size;
248 }
249 
250 
find_event_field(const char * fmt,struct trace_event_call * call)251 static struct trace_event_fields *find_event_field(const char *fmt,
252 						   struct trace_event_call *call)
253 {
254 	struct trace_event_fields *field = call->class->fields_array;
255 	const char *p = fmt;
256 	int len;
257 
258 	if (!(len = str_has_prefix(fmt, "REC->")))
259 		return NULL;
260 	fmt += len;
261 	for (p = fmt; *p; p++) {
262 		if (!isalnum(*p) && *p != '_')
263 			break;
264 	}
265 	len = p - fmt;
266 
267 	for (; field->type; field++) {
268 		if (strncmp(field->name, fmt, len) || field->name[len])
269 			continue;
270 
271 		return field;
272 	}
273 	return NULL;
274 }
275 
276 /*
277  * Check if the referenced field is an array and return true,
278  * as arrays are OK to dereference.
279  */
test_field(const char * fmt,struct trace_event_call * call)280 static bool test_field(const char *fmt, struct trace_event_call *call)
281 {
282 	struct trace_event_fields *field;
283 
284 	field = find_event_field(fmt, call);
285 	if (!field)
286 		return false;
287 
288 	/* This is an array and is OK to dereference. */
289 	return strchr(field->type, '[') != NULL;
290 }
291 
292 /* Look for a string within an argument */
find_print_string(const char * arg,const char * str,const char * end)293 static bool find_print_string(const char *arg, const char *str, const char *end)
294 {
295 	const char *r;
296 
297 	r = strstr(arg, str);
298 	return r && r < end;
299 }
300 
301 /* Return true if the argument pointer is safe */
process_pointer(const char * fmt,int len,struct trace_event_call * call)302 static bool process_pointer(const char *fmt, int len, struct trace_event_call *call)
303 {
304 	const char *r, *e, *a;
305 
306 	e = fmt + len;
307 
308 	/* Find the REC-> in the argument */
309 	r = strstr(fmt, "REC->");
310 	if (r && r < e) {
311 		/*
312 		 * Addresses of events on the buffer, or an array on the buffer is
313 		 * OK to dereference. There's ways to fool this, but
314 		 * this is to catch common mistakes, not malicious code.
315 		 */
316 		a = strchr(fmt, '&');
317 		if ((a && (a < r)) || test_field(r, call))
318 			return true;
319 	} else if (find_print_string(fmt, "__get_dynamic_array(", e)) {
320 		return true;
321 	} else if (find_print_string(fmt, "__get_rel_dynamic_array(", e)) {
322 		return true;
323 	} else if (find_print_string(fmt, "__get_dynamic_array_len(", e)) {
324 		return true;
325 	} else if (find_print_string(fmt, "__get_rel_dynamic_array_len(", e)) {
326 		return true;
327 	} else if (find_print_string(fmt, "__get_sockaddr(", e)) {
328 		return true;
329 	} else if (find_print_string(fmt, "__get_rel_sockaddr(", e)) {
330 		return true;
331 	}
332 	return false;
333 }
334 
335 /* Return true if the string is safe */
process_string(const char * fmt,int len,struct trace_event_call * call)336 static bool process_string(const char *fmt, int len, struct trace_event_call *call)
337 {
338 	struct trace_event_fields *field;
339 	const char *r, *e, *s;
340 
341 	e = fmt + len;
342 
343 	/*
344 	 * There are several helper functions that return strings.
345 	 * If the argument contains a function, then assume its field is valid.
346 	 * It is considered that the argument has a function if it has:
347 	 *   alphanumeric or '_' before a parenthesis.
348 	 */
349 	s = fmt;
350 	do {
351 		r = strstr(s, "(");
352 		if (!r || r >= e)
353 			break;
354 		for (int i = 1; r - i >= s; i++) {
355 			char ch = *(r - i);
356 			if (isspace(ch))
357 				continue;
358 			if (isalnum(ch) || ch == '_')
359 				return true;
360 			/* Anything else, this isn't a function */
361 			break;
362 		}
363 		/* A function could be wrapped in parethesis, try the next one */
364 		s = r + 1;
365 	} while (s < e);
366 
367 	/*
368 	 * Check for arrays. If the argument has: foo[REC->val]
369 	 * then it is very likely that foo is an array of strings
370 	 * that are safe to use.
371 	 */
372 	r = strstr(s, "[");
373 	if (r && r < e) {
374 		r = strstr(r, "REC->");
375 		if (r && r < e)
376 			return true;
377 	}
378 
379 	/*
380 	 * If there's any strings in the argument consider this arg OK as it
381 	 * could be: REC->field ? "foo" : "bar" and we don't want to get into
382 	 * verifying that logic here.
383 	 */
384 	if (find_print_string(fmt, "\"", e))
385 		return true;
386 
387 	/* Dereferenced strings are also valid like any other pointer */
388 	if (process_pointer(fmt, len, call))
389 		return true;
390 
391 	/* Make sure the field is found */
392 	field = find_event_field(fmt, call);
393 	if (!field)
394 		return false;
395 
396 	/* Test this field's string before printing the event */
397 	call->flags |= TRACE_EVENT_FL_TEST_STR;
398 	field->needs_test = 1;
399 
400 	return true;
401 }
402 
403 /*
404  * Examine the print fmt of the event looking for unsafe dereference
405  * pointers using %p* that could be recorded in the trace event and
406  * much later referenced after the pointer was freed. Dereferencing
407  * pointers are OK, if it is dereferenced into the event itself.
408  */
test_event_printk(struct trace_event_call * call)409 static void test_event_printk(struct trace_event_call *call)
410 {
411 	u64 dereference_flags = 0;
412 	u64 string_flags = 0;
413 	bool first = true;
414 	const char *fmt;
415 	int parens = 0;
416 	char in_quote = 0;
417 	int start_arg = 0;
418 	int arg = 0;
419 	int i, e;
420 
421 	fmt = call->print_fmt;
422 
423 	if (!fmt)
424 		return;
425 
426 	for (i = 0; fmt[i]; i++) {
427 		switch (fmt[i]) {
428 		case '\\':
429 			i++;
430 			if (!fmt[i])
431 				return;
432 			continue;
433 		case '"':
434 		case '\'':
435 			/*
436 			 * The print fmt starts with a string that
437 			 * is processed first to find %p* usage,
438 			 * then after the first string, the print fmt
439 			 * contains arguments that are used to check
440 			 * if the dereferenced %p* usage is safe.
441 			 */
442 			if (first) {
443 				if (fmt[i] == '\'')
444 					continue;
445 				if (in_quote) {
446 					arg = 0;
447 					first = false;
448 					/*
449 					 * If there was no %p* uses
450 					 * the fmt is OK.
451 					 */
452 					if (!dereference_flags)
453 						return;
454 				}
455 			}
456 			if (in_quote) {
457 				if (in_quote == fmt[i])
458 					in_quote = 0;
459 			} else {
460 				in_quote = fmt[i];
461 			}
462 			continue;
463 		case '%':
464 			if (!first || !in_quote)
465 				continue;
466 			i++;
467 			if (!fmt[i])
468 				return;
469 			switch (fmt[i]) {
470 			case '%':
471 				continue;
472 			case 'p':
473 				/* Find dereferencing fields */
474 				switch (fmt[i + 1]) {
475 				case 'B': case 'R': case 'r':
476 				case 'b': case 'M': case 'm':
477 				case 'I': case 'i': case 'E':
478 				case 'U': case 'V': case 'N':
479 				case 'a': case 'd': case 'D':
480 				case 'g': case 't': case 'C':
481 				case 'O': case 'f':
482 					if (WARN_ONCE(arg == 63,
483 						      "Too many args for event: %s",
484 						      trace_event_name(call)))
485 						return;
486 					dereference_flags |= 1ULL << arg;
487 				}
488 				break;
489 			default:
490 			{
491 				bool star = false;
492 				int j;
493 
494 				/* Increment arg if %*s exists. */
495 				for (j = 0; fmt[i + j]; j++) {
496 					if (isdigit(fmt[i + j]) ||
497 					    fmt[i + j] == '.')
498 						continue;
499 					if (fmt[i + j] == '*') {
500 						star = true;
501 						continue;
502 					}
503 					if ((fmt[i + j] == 's')) {
504 						if (star)
505 							arg++;
506 						if (WARN_ONCE(arg == 63,
507 							      "Too many args for event: %s",
508 							      trace_event_name(call)))
509 							return;
510 						dereference_flags |= 1ULL << arg;
511 						string_flags |= 1ULL << arg;
512 					}
513 					break;
514 				}
515 				break;
516 			} /* default */
517 
518 			} /* switch */
519 			arg++;
520 			continue;
521 		case '(':
522 			if (in_quote)
523 				continue;
524 			parens++;
525 			continue;
526 		case ')':
527 			if (in_quote)
528 				continue;
529 			parens--;
530 			if (WARN_ONCE(parens < 0,
531 				      "Paren mismatch for event: %s\narg='%s'\n%*s",
532 				      trace_event_name(call),
533 				      fmt + start_arg,
534 				      (i - start_arg) + 5, "^"))
535 				return;
536 			continue;
537 		case ',':
538 			if (in_quote || parens)
539 				continue;
540 			e = i;
541 			i++;
542 			while (isspace(fmt[i]))
543 				i++;
544 
545 			/*
546 			 * If start_arg is zero, then this is the start of the
547 			 * first argument. The processing of the argument happens
548 			 * when the end of the argument is found, as it needs to
549 			 * handle paranthesis and such.
550 			 */
551 			if (!start_arg) {
552 				start_arg = i;
553 				/* Balance out the i++ in the for loop */
554 				i--;
555 				continue;
556 			}
557 
558 			if (dereference_flags & (1ULL << arg)) {
559 				if (string_flags & (1ULL << arg)) {
560 					if (process_string(fmt + start_arg, e - start_arg, call))
561 						dereference_flags &= ~(1ULL << arg);
562 				} else if (process_pointer(fmt + start_arg, e - start_arg, call))
563 					dereference_flags &= ~(1ULL << arg);
564 			}
565 
566 			start_arg = i;
567 			arg++;
568 			/* Balance out the i++ in the for loop */
569 			i--;
570 		}
571 	}
572 
573 	if (dereference_flags & (1ULL << arg)) {
574 		if (string_flags & (1ULL << arg)) {
575 			if (process_string(fmt + start_arg, i - start_arg, call))
576 				dereference_flags &= ~(1ULL << arg);
577 		} else if (process_pointer(fmt + start_arg, i - start_arg, call))
578 			dereference_flags &= ~(1ULL << arg);
579 	}
580 
581 	/*
582 	 * If you triggered the below warning, the trace event reported
583 	 * uses an unsafe dereference pointer %p*. As the data stored
584 	 * at the trace event time may no longer exist when the trace
585 	 * event is printed, dereferencing to the original source is
586 	 * unsafe. The source of the dereference must be copied into the
587 	 * event itself, and the dereference must access the copy instead.
588 	 */
589 	if (WARN_ON_ONCE(dereference_flags)) {
590 		arg = 1;
591 		while (!(dereference_flags & 1)) {
592 			dereference_flags >>= 1;
593 			arg++;
594 		}
595 		pr_warn("event %s has unsafe dereference of argument %d\n",
596 			trace_event_name(call), arg);
597 		pr_warn("print_fmt: %s\n", fmt);
598 	}
599 }
600 
trace_event_raw_init(struct trace_event_call * call)601 int trace_event_raw_init(struct trace_event_call *call)
602 {
603 	int id;
604 
605 	id = register_trace_event(&call->event);
606 	if (!id)
607 		return -ENODEV;
608 
609 	test_event_printk(call);
610 
611 	return 0;
612 }
613 EXPORT_SYMBOL_GPL(trace_event_raw_init);
614 
trace_event_ignore_this_pid(struct trace_event_file * trace_file)615 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
616 {
617 	struct trace_array *tr = trace_file->tr;
618 	struct trace_array_cpu *data;
619 	struct trace_pid_list *no_pid_list;
620 	struct trace_pid_list *pid_list;
621 
622 	pid_list = rcu_dereference_raw(tr->filtered_pids);
623 	no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
624 
625 	if (!pid_list && !no_pid_list)
626 		return false;
627 
628 	data = this_cpu_ptr(tr->array_buffer.data);
629 
630 	return data->ignore_pid;
631 }
632 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
633 
trace_event_buffer_reserve(struct trace_event_buffer * fbuffer,struct trace_event_file * trace_file,unsigned long len)634 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
635 				 struct trace_event_file *trace_file,
636 				 unsigned long len)
637 {
638 	struct trace_event_call *event_call = trace_file->event_call;
639 
640 	if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
641 	    trace_event_ignore_this_pid(trace_file))
642 		return NULL;
643 
644 	/*
645 	 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
646 	 * preemption (adding one to the preempt_count). Since we are
647 	 * interested in the preempt_count at the time the tracepoint was
648 	 * hit, we need to subtract one to offset the increment.
649 	 */
650 	fbuffer->trace_ctx = tracing_gen_ctx_dec();
651 	fbuffer->trace_file = trace_file;
652 
653 	fbuffer->event =
654 		trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
655 						event_call->event.type, len,
656 						fbuffer->trace_ctx);
657 	if (!fbuffer->event)
658 		return NULL;
659 
660 	fbuffer->regs = NULL;
661 	fbuffer->entry = ring_buffer_event_data(fbuffer->event);
662 	return fbuffer->entry;
663 }
664 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
665 
trace_event_reg(struct trace_event_call * call,enum trace_reg type,void * data)666 int trace_event_reg(struct trace_event_call *call,
667 		    enum trace_reg type, void *data)
668 {
669 	struct trace_event_file *file = data;
670 
671 	WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
672 	switch (type) {
673 	case TRACE_REG_REGISTER:
674 		return tracepoint_probe_register(call->tp,
675 						 call->class->probe,
676 						 file);
677 	case TRACE_REG_UNREGISTER:
678 		tracepoint_probe_unregister(call->tp,
679 					    call->class->probe,
680 					    file);
681 		return 0;
682 
683 #ifdef CONFIG_PERF_EVENTS
684 	case TRACE_REG_PERF_REGISTER:
685 		return tracepoint_probe_register(call->tp,
686 						 call->class->perf_probe,
687 						 call);
688 	case TRACE_REG_PERF_UNREGISTER:
689 		tracepoint_probe_unregister(call->tp,
690 					    call->class->perf_probe,
691 					    call);
692 		return 0;
693 	case TRACE_REG_PERF_OPEN:
694 	case TRACE_REG_PERF_CLOSE:
695 	case TRACE_REG_PERF_ADD:
696 	case TRACE_REG_PERF_DEL:
697 		return 0;
698 #endif
699 	}
700 	return 0;
701 }
702 EXPORT_SYMBOL_GPL(trace_event_reg);
703 
trace_event_enable_cmd_record(bool enable)704 void trace_event_enable_cmd_record(bool enable)
705 {
706 	struct trace_event_file *file;
707 	struct trace_array *tr;
708 
709 	lockdep_assert_held(&event_mutex);
710 
711 	do_for_each_event_file(tr, file) {
712 
713 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
714 			continue;
715 
716 		if (enable) {
717 			tracing_start_cmdline_record();
718 			set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
719 		} else {
720 			tracing_stop_cmdline_record();
721 			clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
722 		}
723 	} while_for_each_event_file();
724 }
725 
trace_event_enable_tgid_record(bool enable)726 void trace_event_enable_tgid_record(bool enable)
727 {
728 	struct trace_event_file *file;
729 	struct trace_array *tr;
730 
731 	lockdep_assert_held(&event_mutex);
732 
733 	do_for_each_event_file(tr, file) {
734 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
735 			continue;
736 
737 		if (enable) {
738 			tracing_start_tgid_record();
739 			set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
740 		} else {
741 			tracing_stop_tgid_record();
742 			clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
743 				  &file->flags);
744 		}
745 	} while_for_each_event_file();
746 }
747 
__ftrace_event_enable_disable(struct trace_event_file * file,int enable,int soft_disable)748 static int __ftrace_event_enable_disable(struct trace_event_file *file,
749 					 int enable, int soft_disable)
750 {
751 	struct trace_event_call *call = file->event_call;
752 	struct trace_array *tr = file->tr;
753 	int ret = 0;
754 	int disable;
755 
756 	switch (enable) {
757 	case 0:
758 		/*
759 		 * When soft_disable is set and enable is cleared, the sm_ref
760 		 * reference counter is decremented. If it reaches 0, we want
761 		 * to clear the SOFT_DISABLED flag but leave the event in the
762 		 * state that it was. That is, if the event was enabled and
763 		 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
764 		 * is set we do not want the event to be enabled before we
765 		 * clear the bit.
766 		 *
767 		 * When soft_disable is not set but the SOFT_MODE flag is,
768 		 * we do nothing. Do not disable the tracepoint, otherwise
769 		 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
770 		 */
771 		if (soft_disable) {
772 			if (atomic_dec_return(&file->sm_ref) > 0)
773 				break;
774 			disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
775 			clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
776 			/* Disable use of trace_buffered_event */
777 			trace_buffered_event_disable();
778 		} else
779 			disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
780 
781 		if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
782 			clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
783 			if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
784 				tracing_stop_cmdline_record();
785 				clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
786 			}
787 
788 			if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
789 				tracing_stop_tgid_record();
790 				clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
791 			}
792 
793 			ret = call->class->reg(call, TRACE_REG_UNREGISTER, file);
794 
795 			WARN_ON_ONCE(ret);
796 		}
797 		/* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
798 		if (file->flags & EVENT_FILE_FL_SOFT_MODE)
799 			set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
800 		else
801 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
802 		break;
803 	case 1:
804 		/*
805 		 * When soft_disable is set and enable is set, we want to
806 		 * register the tracepoint for the event, but leave the event
807 		 * as is. That means, if the event was already enabled, we do
808 		 * nothing (but set SOFT_MODE). If the event is disabled, we
809 		 * set SOFT_DISABLED before enabling the event tracepoint, so
810 		 * it still seems to be disabled.
811 		 */
812 		if (!soft_disable)
813 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
814 		else {
815 			if (atomic_inc_return(&file->sm_ref) > 1)
816 				break;
817 			set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
818 			/* Enable use of trace_buffered_event */
819 			trace_buffered_event_enable();
820 		}
821 
822 		if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
823 			bool cmd = false, tgid = false;
824 
825 			/* Keep the event disabled, when going to SOFT_MODE. */
826 			if (soft_disable)
827 				set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
828 
829 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
830 				cmd = true;
831 				tracing_start_cmdline_record();
832 				set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
833 			}
834 
835 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
836 				tgid = true;
837 				tracing_start_tgid_record();
838 				set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
839 			}
840 
841 			ret = call->class->reg(call, TRACE_REG_REGISTER, file);
842 			if (ret) {
843 				if (cmd)
844 					tracing_stop_cmdline_record();
845 				if (tgid)
846 					tracing_stop_tgid_record();
847 				pr_info("event trace: Could not enable event "
848 					"%s\n", trace_event_name(call));
849 				break;
850 			}
851 			set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
852 
853 			/* WAS_ENABLED gets set but never cleared. */
854 			set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
855 		}
856 		break;
857 	}
858 
859 	return ret;
860 }
861 
trace_event_enable_disable(struct trace_event_file * file,int enable,int soft_disable)862 int trace_event_enable_disable(struct trace_event_file *file,
863 			       int enable, int soft_disable)
864 {
865 	return __ftrace_event_enable_disable(file, enable, soft_disable);
866 }
867 
ftrace_event_enable_disable(struct trace_event_file * file,int enable)868 static int ftrace_event_enable_disable(struct trace_event_file *file,
869 				       int enable)
870 {
871 	return __ftrace_event_enable_disable(file, enable, 0);
872 }
873 
874 #ifdef CONFIG_MODULES
875 struct event_mod_load {
876 	struct list_head	list;
877 	char			*module;
878 	char			*match;
879 	char			*system;
880 	char			*event;
881 };
882 
free_event_mod(struct event_mod_load * event_mod)883 static void free_event_mod(struct event_mod_load *event_mod)
884 {
885 	list_del(&event_mod->list);
886 	kfree(event_mod->module);
887 	kfree(event_mod->match);
888 	kfree(event_mod->system);
889 	kfree(event_mod->event);
890 	kfree(event_mod);
891 }
892 
clear_mod_events(struct trace_array * tr)893 static void clear_mod_events(struct trace_array *tr)
894 {
895 	struct event_mod_load *event_mod, *n;
896 
897 	list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
898 		free_event_mod(event_mod);
899 	}
900 }
901 
remove_cache_mod(struct trace_array * tr,const char * mod,const char * match,const char * system,const char * event)902 static int remove_cache_mod(struct trace_array *tr, const char *mod,
903 			    const char *match, const char *system, const char *event)
904 {
905 	struct event_mod_load *event_mod, *n;
906 	int ret = -EINVAL;
907 
908 	list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
909 		if (strcmp(event_mod->module, mod) != 0)
910 			continue;
911 
912 		if (match && strcmp(event_mod->match, match) != 0)
913 			continue;
914 
915 		if (system &&
916 		    (!event_mod->system || strcmp(event_mod->system, system) != 0))
917 			continue;
918 
919 		if (event &&
920 		    (!event_mod->event || strcmp(event_mod->event, event) != 0))
921 			continue;
922 
923 		free_event_mod(event_mod);
924 		ret = 0;
925 	}
926 
927 	return ret;
928 }
929 
cache_mod(struct trace_array * tr,const char * mod,int set,const char * match,const char * system,const char * event)930 static int cache_mod(struct trace_array *tr, const char *mod, int set,
931 		     const char *match, const char *system, const char *event)
932 {
933 	struct event_mod_load *event_mod;
934 
935 	/* If the module exists, then this just failed to find an event */
936 	if (module_exists(mod))
937 		return -EINVAL;
938 
939 	/* See if this is to remove a cached filter */
940 	if (!set)
941 		return remove_cache_mod(tr, mod, match, system, event);
942 
943 	event_mod = kzalloc(sizeof(*event_mod), GFP_KERNEL);
944 	if (!event_mod)
945 		return -ENOMEM;
946 
947 	INIT_LIST_HEAD(&event_mod->list);
948 	event_mod->module = kstrdup(mod, GFP_KERNEL);
949 	if (!event_mod->module)
950 		goto out_free;
951 
952 	if (match) {
953 		event_mod->match = kstrdup(match, GFP_KERNEL);
954 		if (!event_mod->match)
955 			goto out_free;
956 	}
957 
958 	if (system) {
959 		event_mod->system = kstrdup(system, GFP_KERNEL);
960 		if (!event_mod->system)
961 			goto out_free;
962 	}
963 
964 	if (event) {
965 		event_mod->event = kstrdup(event, GFP_KERNEL);
966 		if (!event_mod->event)
967 			goto out_free;
968 	}
969 
970 	list_add(&event_mod->list, &tr->mod_events);
971 
972 	return 0;
973 
974  out_free:
975 	free_event_mod(event_mod);
976 
977 	return -ENOMEM;
978 }
979 #else /* CONFIG_MODULES */
clear_mod_events(struct trace_array * tr)980 static inline void clear_mod_events(struct trace_array *tr) { }
cache_mod(struct trace_array * tr,const char * mod,int set,const char * match,const char * system,const char * event)981 static int cache_mod(struct trace_array *tr, const char *mod, int set,
982 		     const char *match, const char *system, const char *event)
983 {
984 	return -EINVAL;
985 }
986 #endif
987 
ftrace_clear_events(struct trace_array * tr)988 static void ftrace_clear_events(struct trace_array *tr)
989 {
990 	struct trace_event_file *file;
991 
992 	mutex_lock(&event_mutex);
993 	list_for_each_entry(file, &tr->events, list) {
994 		ftrace_event_enable_disable(file, 0);
995 	}
996 	clear_mod_events(tr);
997 	mutex_unlock(&event_mutex);
998 }
999 
1000 static void
event_filter_pid_sched_process_exit(void * data,struct task_struct * task)1001 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
1002 {
1003 	struct trace_pid_list *pid_list;
1004 	struct trace_array *tr = data;
1005 
1006 	pid_list = rcu_dereference_raw(tr->filtered_pids);
1007 	trace_filter_add_remove_task(pid_list, NULL, task);
1008 
1009 	pid_list = rcu_dereference_raw(tr->filtered_no_pids);
1010 	trace_filter_add_remove_task(pid_list, NULL, task);
1011 }
1012 
1013 static void
event_filter_pid_sched_process_fork(void * data,struct task_struct * self,struct task_struct * task)1014 event_filter_pid_sched_process_fork(void *data,
1015 				    struct task_struct *self,
1016 				    struct task_struct *task)
1017 {
1018 	struct trace_pid_list *pid_list;
1019 	struct trace_array *tr = data;
1020 
1021 	pid_list = rcu_dereference_sched(tr->filtered_pids);
1022 	trace_filter_add_remove_task(pid_list, self, task);
1023 
1024 	pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1025 	trace_filter_add_remove_task(pid_list, self, task);
1026 }
1027 
trace_event_follow_fork(struct trace_array * tr,bool enable)1028 void trace_event_follow_fork(struct trace_array *tr, bool enable)
1029 {
1030 	if (enable) {
1031 		register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
1032 						       tr, INT_MIN);
1033 		register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
1034 						       tr, INT_MAX);
1035 	} else {
1036 		unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
1037 						    tr);
1038 		unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
1039 						    tr);
1040 	}
1041 }
1042 
1043 static void
event_filter_pid_sched_switch_probe_pre(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)1044 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
1045 					struct task_struct *prev,
1046 					struct task_struct *next,
1047 					unsigned int prev_state)
1048 {
1049 	struct trace_array *tr = data;
1050 	struct trace_pid_list *no_pid_list;
1051 	struct trace_pid_list *pid_list;
1052 	bool ret;
1053 
1054 	pid_list = rcu_dereference_sched(tr->filtered_pids);
1055 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1056 
1057 	/*
1058 	 * Sched switch is funny, as we only want to ignore it
1059 	 * in the notrace case if both prev and next should be ignored.
1060 	 */
1061 	ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
1062 		trace_ignore_this_task(NULL, no_pid_list, next);
1063 
1064 	this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
1065 		       (trace_ignore_this_task(pid_list, NULL, prev) &&
1066 			trace_ignore_this_task(pid_list, NULL, next)));
1067 }
1068 
1069 static void
event_filter_pid_sched_switch_probe_post(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)1070 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
1071 					 struct task_struct *prev,
1072 					 struct task_struct *next,
1073 					 unsigned int prev_state)
1074 {
1075 	struct trace_array *tr = data;
1076 	struct trace_pid_list *no_pid_list;
1077 	struct trace_pid_list *pid_list;
1078 
1079 	pid_list = rcu_dereference_sched(tr->filtered_pids);
1080 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1081 
1082 	this_cpu_write(tr->array_buffer.data->ignore_pid,
1083 		       trace_ignore_this_task(pid_list, no_pid_list, next));
1084 }
1085 
1086 static void
event_filter_pid_sched_wakeup_probe_pre(void * data,struct task_struct * task)1087 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
1088 {
1089 	struct trace_array *tr = data;
1090 	struct trace_pid_list *no_pid_list;
1091 	struct trace_pid_list *pid_list;
1092 
1093 	/* Nothing to do if we are already tracing */
1094 	if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
1095 		return;
1096 
1097 	pid_list = rcu_dereference_sched(tr->filtered_pids);
1098 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1099 
1100 	this_cpu_write(tr->array_buffer.data->ignore_pid,
1101 		       trace_ignore_this_task(pid_list, no_pid_list, task));
1102 }
1103 
1104 static void
event_filter_pid_sched_wakeup_probe_post(void * data,struct task_struct * task)1105 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
1106 {
1107 	struct trace_array *tr = data;
1108 	struct trace_pid_list *no_pid_list;
1109 	struct trace_pid_list *pid_list;
1110 
1111 	/* Nothing to do if we are not tracing */
1112 	if (this_cpu_read(tr->array_buffer.data->ignore_pid))
1113 		return;
1114 
1115 	pid_list = rcu_dereference_sched(tr->filtered_pids);
1116 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1117 
1118 	/* Set tracing if current is enabled */
1119 	this_cpu_write(tr->array_buffer.data->ignore_pid,
1120 		       trace_ignore_this_task(pid_list, no_pid_list, current));
1121 }
1122 
unregister_pid_events(struct trace_array * tr)1123 static void unregister_pid_events(struct trace_array *tr)
1124 {
1125 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
1126 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
1127 
1128 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
1129 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
1130 
1131 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
1132 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
1133 
1134 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
1135 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
1136 }
1137 
__ftrace_clear_event_pids(struct trace_array * tr,int type)1138 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
1139 {
1140 	struct trace_pid_list *pid_list;
1141 	struct trace_pid_list *no_pid_list;
1142 	struct trace_event_file *file;
1143 	int cpu;
1144 
1145 	pid_list = rcu_dereference_protected(tr->filtered_pids,
1146 					     lockdep_is_held(&event_mutex));
1147 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1148 					     lockdep_is_held(&event_mutex));
1149 
1150 	/* Make sure there's something to do */
1151 	if (!pid_type_enabled(type, pid_list, no_pid_list))
1152 		return;
1153 
1154 	if (!still_need_pid_events(type, pid_list, no_pid_list)) {
1155 		unregister_pid_events(tr);
1156 
1157 		list_for_each_entry(file, &tr->events, list) {
1158 			clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1159 		}
1160 
1161 		for_each_possible_cpu(cpu)
1162 			per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
1163 	}
1164 
1165 	if (type & TRACE_PIDS)
1166 		rcu_assign_pointer(tr->filtered_pids, NULL);
1167 
1168 	if (type & TRACE_NO_PIDS)
1169 		rcu_assign_pointer(tr->filtered_no_pids, NULL);
1170 
1171 	/* Wait till all users are no longer using pid filtering */
1172 	tracepoint_synchronize_unregister();
1173 
1174 	if ((type & TRACE_PIDS) && pid_list)
1175 		trace_pid_list_free(pid_list);
1176 
1177 	if ((type & TRACE_NO_PIDS) && no_pid_list)
1178 		trace_pid_list_free(no_pid_list);
1179 }
1180 
ftrace_clear_event_pids(struct trace_array * tr,int type)1181 static void ftrace_clear_event_pids(struct trace_array *tr, int type)
1182 {
1183 	mutex_lock(&event_mutex);
1184 	__ftrace_clear_event_pids(tr, type);
1185 	mutex_unlock(&event_mutex);
1186 }
1187 
__put_system(struct event_subsystem * system)1188 static void __put_system(struct event_subsystem *system)
1189 {
1190 	struct event_filter *filter = system->filter;
1191 
1192 	WARN_ON_ONCE(system_refcount(system) == 0);
1193 	if (system_refcount_dec(system))
1194 		return;
1195 
1196 	list_del(&system->list);
1197 
1198 	if (filter) {
1199 		kfree(filter->filter_string);
1200 		kfree(filter);
1201 	}
1202 	kfree_const(system->name);
1203 	kfree(system);
1204 }
1205 
__get_system(struct event_subsystem * system)1206 static void __get_system(struct event_subsystem *system)
1207 {
1208 	WARN_ON_ONCE(system_refcount(system) == 0);
1209 	system_refcount_inc(system);
1210 }
1211 
__get_system_dir(struct trace_subsystem_dir * dir)1212 static void __get_system_dir(struct trace_subsystem_dir *dir)
1213 {
1214 	WARN_ON_ONCE(dir->ref_count == 0);
1215 	dir->ref_count++;
1216 	__get_system(dir->subsystem);
1217 }
1218 
__put_system_dir(struct trace_subsystem_dir * dir)1219 static void __put_system_dir(struct trace_subsystem_dir *dir)
1220 {
1221 	WARN_ON_ONCE(dir->ref_count == 0);
1222 	/* If the subsystem is about to be freed, the dir must be too */
1223 	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
1224 
1225 	__put_system(dir->subsystem);
1226 	if (!--dir->ref_count)
1227 		kfree(dir);
1228 }
1229 
put_system(struct trace_subsystem_dir * dir)1230 static void put_system(struct trace_subsystem_dir *dir)
1231 {
1232 	mutex_lock(&event_mutex);
1233 	__put_system_dir(dir);
1234 	mutex_unlock(&event_mutex);
1235 }
1236 
remove_subsystem(struct trace_subsystem_dir * dir)1237 static void remove_subsystem(struct trace_subsystem_dir *dir)
1238 {
1239 	if (!dir)
1240 		return;
1241 
1242 	if (!--dir->nr_events) {
1243 		eventfs_remove_dir(dir->ei);
1244 		list_del(&dir->list);
1245 		__put_system_dir(dir);
1246 	}
1247 }
1248 
event_file_get(struct trace_event_file * file)1249 void event_file_get(struct trace_event_file *file)
1250 {
1251 	refcount_inc(&file->ref);
1252 }
1253 
event_file_put(struct trace_event_file * file)1254 void event_file_put(struct trace_event_file *file)
1255 {
1256 	if (WARN_ON_ONCE(!refcount_read(&file->ref))) {
1257 		if (file->flags & EVENT_FILE_FL_FREED)
1258 			kmem_cache_free(file_cachep, file);
1259 		return;
1260 	}
1261 
1262 	if (refcount_dec_and_test(&file->ref)) {
1263 		/* Count should only go to zero when it is freed */
1264 		if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1265 			return;
1266 		kmem_cache_free(file_cachep, file);
1267 	}
1268 }
1269 
remove_event_file_dir(struct trace_event_file * file)1270 static void remove_event_file_dir(struct trace_event_file *file)
1271 {
1272 	eventfs_remove_dir(file->ei);
1273 	list_del(&file->list);
1274 	remove_subsystem(file->system);
1275 	free_event_filter(file->filter);
1276 	file->flags |= EVENT_FILE_FL_FREED;
1277 	event_file_put(file);
1278 }
1279 
1280 /*
1281  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1282  */
1283 static int
__ftrace_set_clr_event_nolock(struct trace_array * tr,const char * match,const char * sub,const char * event,int set,const char * mod)1284 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1285 			      const char *sub, const char *event, int set,
1286 			      const char *mod)
1287 {
1288 	struct trace_event_file *file;
1289 	struct trace_event_call *call;
1290 	char *module __free(kfree) = NULL;
1291 	const char *name;
1292 	int ret = -EINVAL;
1293 	int eret = 0;
1294 
1295 	if (mod) {
1296 		char *p;
1297 
1298 		module = kstrdup(mod, GFP_KERNEL);
1299 		if (!module)
1300 			return -ENOMEM;
1301 
1302 		/* Replace all '-' with '_' as that's what modules do */
1303 		for (p = strchr(module, '-'); p; p = strchr(p + 1, '-'))
1304 			*p = '_';
1305 	}
1306 
1307 	list_for_each_entry(file, &tr->events, list) {
1308 
1309 		call = file->event_call;
1310 
1311 		/* If a module is specified, skip events that are not that module */
1312 		if (module && (!call->module || strcmp(module_name(call->module), module)))
1313 			continue;
1314 
1315 		name = trace_event_name(call);
1316 
1317 		if (!name || !call->class || !call->class->reg)
1318 			continue;
1319 
1320 		if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1321 			continue;
1322 
1323 		if (match &&
1324 		    strcmp(match, name) != 0 &&
1325 		    strcmp(match, call->class->system) != 0)
1326 			continue;
1327 
1328 		if (sub && strcmp(sub, call->class->system) != 0)
1329 			continue;
1330 
1331 		if (event && strcmp(event, name) != 0)
1332 			continue;
1333 
1334 		ret = ftrace_event_enable_disable(file, set);
1335 
1336 		/*
1337 		 * Save the first error and return that. Some events
1338 		 * may still have been enabled, but let the user
1339 		 * know that something went wrong.
1340 		 */
1341 		if (ret && !eret)
1342 			eret = ret;
1343 
1344 		ret = eret;
1345 	}
1346 
1347 	/*
1348 	 * If this is a module setting and nothing was found,
1349 	 * check if the module was loaded. If it wasn't cache it.
1350 	 */
1351 	if (module && ret == -EINVAL && !eret)
1352 		ret = cache_mod(tr, module, set, match, sub, event);
1353 
1354 	return ret;
1355 }
1356 
__ftrace_set_clr_event(struct trace_array * tr,const char * match,const char * sub,const char * event,int set,const char * mod)1357 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1358 				  const char *sub, const char *event, int set,
1359 				  const char *mod)
1360 {
1361 	int ret;
1362 
1363 	mutex_lock(&event_mutex);
1364 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set, mod);
1365 	mutex_unlock(&event_mutex);
1366 
1367 	return ret;
1368 }
1369 
ftrace_set_clr_event(struct trace_array * tr,char * buf,int set)1370 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1371 {
1372 	char *event = NULL, *sub = NULL, *match, *mod;
1373 	int ret;
1374 
1375 	if (!tr)
1376 		return -ENOENT;
1377 
1378 	/* Modules events can be appened with :mod:<module> */
1379 	mod = strstr(buf, ":mod:");
1380 	if (mod) {
1381 		*mod = '\0';
1382 		/* move to the module name */
1383 		mod += 5;
1384 	}
1385 
1386 	/*
1387 	 * The buf format can be <subsystem>:<event-name>
1388 	 *  *:<event-name> means any event by that name.
1389 	 *  :<event-name> is the same.
1390 	 *
1391 	 *  <subsystem>:* means all events in that subsystem
1392 	 *  <subsystem>: means the same.
1393 	 *
1394 	 *  <name> (no ':') means all events in a subsystem with
1395 	 *  the name <name> or any event that matches <name>
1396 	 */
1397 
1398 	match = strsep(&buf, ":");
1399 	if (buf) {
1400 		sub = match;
1401 		event = buf;
1402 		match = NULL;
1403 
1404 		if (!strlen(sub) || strcmp(sub, "*") == 0)
1405 			sub = NULL;
1406 		if (!strlen(event) || strcmp(event, "*") == 0)
1407 			event = NULL;
1408 	} else if (mod) {
1409 		/* Allow wildcard for no length or star */
1410 		if (!strlen(match) || strcmp(match, "*") == 0)
1411 			match = NULL;
1412 	}
1413 
1414 	ret = __ftrace_set_clr_event(tr, match, sub, event, set, mod);
1415 
1416 	/* Put back the colon to allow this to be called again */
1417 	if (buf)
1418 		*(buf - 1) = ':';
1419 
1420 	return ret;
1421 }
1422 
1423 /**
1424  * trace_set_clr_event - enable or disable an event
1425  * @system: system name to match (NULL for any system)
1426  * @event: event name to match (NULL for all events, within system)
1427  * @set: 1 to enable, 0 to disable
1428  *
1429  * This is a way for other parts of the kernel to enable or disable
1430  * event recording.
1431  *
1432  * Returns 0 on success, -EINVAL if the parameters do not match any
1433  * registered events.
1434  */
trace_set_clr_event(const char * system,const char * event,int set)1435 int trace_set_clr_event(const char *system, const char *event, int set)
1436 {
1437 	struct trace_array *tr = top_trace_array();
1438 
1439 	if (!tr)
1440 		return -ENODEV;
1441 
1442 	return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1443 }
1444 EXPORT_SYMBOL_GPL(trace_set_clr_event);
1445 
1446 /**
1447  * trace_array_set_clr_event - enable or disable an event for a trace array.
1448  * @tr: concerned trace array.
1449  * @system: system name to match (NULL for any system)
1450  * @event: event name to match (NULL for all events, within system)
1451  * @enable: true to enable, false to disable
1452  *
1453  * This is a way for other parts of the kernel to enable or disable
1454  * event recording.
1455  *
1456  * Returns 0 on success, -EINVAL if the parameters do not match any
1457  * registered events.
1458  */
trace_array_set_clr_event(struct trace_array * tr,const char * system,const char * event,bool enable)1459 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1460 		const char *event, bool enable)
1461 {
1462 	int set;
1463 
1464 	if (!tr)
1465 		return -ENOENT;
1466 
1467 	set = (enable == true) ? 1 : 0;
1468 	return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1469 }
1470 EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1471 
1472 /* 128 should be much more than enough */
1473 #define EVENT_BUF_SIZE		127
1474 
1475 static ssize_t
ftrace_event_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)1476 ftrace_event_write(struct file *file, const char __user *ubuf,
1477 		   size_t cnt, loff_t *ppos)
1478 {
1479 	struct trace_parser parser;
1480 	struct seq_file *m = file->private_data;
1481 	struct trace_array *tr = m->private;
1482 	ssize_t read, ret;
1483 
1484 	if (!cnt)
1485 		return 0;
1486 
1487 	ret = tracing_update_buffers(tr);
1488 	if (ret < 0)
1489 		return ret;
1490 
1491 	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1492 		return -ENOMEM;
1493 
1494 	read = trace_get_user(&parser, ubuf, cnt, ppos);
1495 
1496 	if (read >= 0 && trace_parser_loaded((&parser))) {
1497 		int set = 1;
1498 
1499 		if (*parser.buffer == '!')
1500 			set = 0;
1501 
1502 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1503 		if (ret)
1504 			goto out_put;
1505 	}
1506 
1507 	ret = read;
1508 
1509  out_put:
1510 	trace_parser_put(&parser);
1511 
1512 	return ret;
1513 }
1514 
1515 static void *
t_next(struct seq_file * m,void * v,loff_t * pos)1516 t_next(struct seq_file *m, void *v, loff_t *pos)
1517 {
1518 	struct trace_event_file *file = v;
1519 	struct trace_event_call *call;
1520 	struct trace_array *tr = m->private;
1521 
1522 	(*pos)++;
1523 
1524 	list_for_each_entry_continue(file, &tr->events, list) {
1525 		call = file->event_call;
1526 		/*
1527 		 * The ftrace subsystem is for showing formats only.
1528 		 * They can not be enabled or disabled via the event files.
1529 		 */
1530 		if (call->class && call->class->reg &&
1531 		    !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1532 			return file;
1533 	}
1534 
1535 	return NULL;
1536 }
1537 
t_start(struct seq_file * m,loff_t * pos)1538 static void *t_start(struct seq_file *m, loff_t *pos)
1539 {
1540 	struct trace_event_file *file;
1541 	struct trace_array *tr = m->private;
1542 	loff_t l;
1543 
1544 	mutex_lock(&event_mutex);
1545 
1546 	file = list_entry(&tr->events, struct trace_event_file, list);
1547 	for (l = 0; l <= *pos; ) {
1548 		file = t_next(m, file, &l);
1549 		if (!file)
1550 			break;
1551 	}
1552 	return file;
1553 }
1554 
1555 enum set_event_iter_type {
1556 	SET_EVENT_FILE,
1557 	SET_EVENT_MOD,
1558 };
1559 
1560 struct set_event_iter {
1561 	enum set_event_iter_type	type;
1562 	union {
1563 		struct trace_event_file	*file;
1564 		struct event_mod_load	*event_mod;
1565 	};
1566 };
1567 
1568 static void *
s_next(struct seq_file * m,void * v,loff_t * pos)1569 s_next(struct seq_file *m, void *v, loff_t *pos)
1570 {
1571 	struct set_event_iter *iter = v;
1572 	struct trace_event_file *file;
1573 	struct trace_array *tr = m->private;
1574 
1575 	(*pos)++;
1576 
1577 	if (iter->type == SET_EVENT_FILE) {
1578 		file = iter->file;
1579 		list_for_each_entry_continue(file, &tr->events, list) {
1580 			if (file->flags & EVENT_FILE_FL_ENABLED) {
1581 				iter->file = file;
1582 				return iter;
1583 			}
1584 		}
1585 #ifdef CONFIG_MODULES
1586 		iter->type = SET_EVENT_MOD;
1587 		iter->event_mod = list_entry(&tr->mod_events, struct event_mod_load, list);
1588 #endif
1589 	}
1590 
1591 #ifdef CONFIG_MODULES
1592 	list_for_each_entry_continue(iter->event_mod, &tr->mod_events, list)
1593 		return iter;
1594 #endif
1595 
1596 	/*
1597 	 * The iter is allocated in s_start() and passed via the 'v'
1598 	 * parameter. To stop the iterator, NULL must be returned. But
1599 	 * the return value is what the 'v' parameter in s_stop() receives
1600 	 * and frees. Free iter here as it will no longer be used.
1601 	 */
1602 	kfree(iter);
1603 	return NULL;
1604 }
1605 
s_start(struct seq_file * m,loff_t * pos)1606 static void *s_start(struct seq_file *m, loff_t *pos)
1607 {
1608 	struct trace_array *tr = m->private;
1609 	struct set_event_iter *iter;
1610 	loff_t l;
1611 
1612 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1613 	if (!iter)
1614 		return NULL;
1615 
1616 	mutex_lock(&event_mutex);
1617 
1618 	iter->type = SET_EVENT_FILE;
1619 	iter->file = list_entry(&tr->events, struct trace_event_file, list);
1620 
1621 	for (l = 0; l <= *pos; ) {
1622 		iter = s_next(m, iter, &l);
1623 		if (!iter)
1624 			break;
1625 	}
1626 	return iter;
1627 }
1628 
t_show(struct seq_file * m,void * v)1629 static int t_show(struct seq_file *m, void *v)
1630 {
1631 	struct trace_event_file *file = v;
1632 	struct trace_event_call *call = file->event_call;
1633 
1634 	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1635 		seq_printf(m, "%s:", call->class->system);
1636 	seq_printf(m, "%s\n", trace_event_name(call));
1637 
1638 	return 0;
1639 }
1640 
t_stop(struct seq_file * m,void * p)1641 static void t_stop(struct seq_file *m, void *p)
1642 {
1643 	mutex_unlock(&event_mutex);
1644 }
1645 
1646 #ifdef CONFIG_MODULES
s_show(struct seq_file * m,void * v)1647 static int s_show(struct seq_file *m, void *v)
1648 {
1649 	struct set_event_iter *iter = v;
1650 	const char *system;
1651 	const char *event;
1652 
1653 	if (iter->type == SET_EVENT_FILE)
1654 		return t_show(m, iter->file);
1655 
1656 	/* When match is set, system and event are not */
1657 	if (iter->event_mod->match) {
1658 		seq_printf(m, "%s:mod:%s\n", iter->event_mod->match,
1659 			   iter->event_mod->module);
1660 		return 0;
1661 	}
1662 
1663 	system = iter->event_mod->system ? : "*";
1664 	event = iter->event_mod->event ? : "*";
1665 
1666 	seq_printf(m, "%s:%s:mod:%s\n", system, event, iter->event_mod->module);
1667 
1668 	return 0;
1669 }
1670 #else /* CONFIG_MODULES */
s_show(struct seq_file * m,void * v)1671 static int s_show(struct seq_file *m, void *v)
1672 {
1673 	struct set_event_iter *iter = v;
1674 
1675 	return t_show(m, iter->file);
1676 }
1677 #endif
1678 
s_stop(struct seq_file * m,void * v)1679 static void s_stop(struct seq_file *m, void *v)
1680 {
1681 	kfree(v);
1682 	t_stop(m, NULL);
1683 }
1684 
1685 static void *
__next(struct seq_file * m,void * v,loff_t * pos,int type)1686 __next(struct seq_file *m, void *v, loff_t *pos, int type)
1687 {
1688 	struct trace_array *tr = m->private;
1689 	struct trace_pid_list *pid_list;
1690 
1691 	if (type == TRACE_PIDS)
1692 		pid_list = rcu_dereference_sched(tr->filtered_pids);
1693 	else
1694 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1695 
1696 	return trace_pid_next(pid_list, v, pos);
1697 }
1698 
1699 static void *
p_next(struct seq_file * m,void * v,loff_t * pos)1700 p_next(struct seq_file *m, void *v, loff_t *pos)
1701 {
1702 	return __next(m, v, pos, TRACE_PIDS);
1703 }
1704 
1705 static void *
np_next(struct seq_file * m,void * v,loff_t * pos)1706 np_next(struct seq_file *m, void *v, loff_t *pos)
1707 {
1708 	return __next(m, v, pos, TRACE_NO_PIDS);
1709 }
1710 
__start(struct seq_file * m,loff_t * pos,int type)1711 static void *__start(struct seq_file *m, loff_t *pos, int type)
1712 	__acquires(RCU)
1713 {
1714 	struct trace_pid_list *pid_list;
1715 	struct trace_array *tr = m->private;
1716 
1717 	/*
1718 	 * Grab the mutex, to keep calls to p_next() having the same
1719 	 * tr->filtered_pids as p_start() has.
1720 	 * If we just passed the tr->filtered_pids around, then RCU would
1721 	 * have been enough, but doing that makes things more complex.
1722 	 */
1723 	mutex_lock(&event_mutex);
1724 	rcu_read_lock_sched();
1725 
1726 	if (type == TRACE_PIDS)
1727 		pid_list = rcu_dereference_sched(tr->filtered_pids);
1728 	else
1729 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1730 
1731 	if (!pid_list)
1732 		return NULL;
1733 
1734 	return trace_pid_start(pid_list, pos);
1735 }
1736 
p_start(struct seq_file * m,loff_t * pos)1737 static void *p_start(struct seq_file *m, loff_t *pos)
1738 	__acquires(RCU)
1739 {
1740 	return __start(m, pos, TRACE_PIDS);
1741 }
1742 
np_start(struct seq_file * m,loff_t * pos)1743 static void *np_start(struct seq_file *m, loff_t *pos)
1744 	__acquires(RCU)
1745 {
1746 	return __start(m, pos, TRACE_NO_PIDS);
1747 }
1748 
p_stop(struct seq_file * m,void * p)1749 static void p_stop(struct seq_file *m, void *p)
1750 	__releases(RCU)
1751 {
1752 	rcu_read_unlock_sched();
1753 	mutex_unlock(&event_mutex);
1754 }
1755 
1756 static ssize_t
event_enable_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1757 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1758 		  loff_t *ppos)
1759 {
1760 	struct trace_event_file *file;
1761 	unsigned long flags;
1762 	char buf[4] = "0";
1763 
1764 	mutex_lock(&event_mutex);
1765 	file = event_file_file(filp);
1766 	if (likely(file))
1767 		flags = file->flags;
1768 	mutex_unlock(&event_mutex);
1769 
1770 	if (!file)
1771 		return -ENODEV;
1772 
1773 	if (flags & EVENT_FILE_FL_ENABLED &&
1774 	    !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1775 		strcpy(buf, "1");
1776 
1777 	if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1778 	    flags & EVENT_FILE_FL_SOFT_MODE)
1779 		strcat(buf, "*");
1780 
1781 	strcat(buf, "\n");
1782 
1783 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1784 }
1785 
1786 static ssize_t
event_enable_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1787 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1788 		   loff_t *ppos)
1789 {
1790 	struct trace_event_file *file;
1791 	unsigned long val;
1792 	int ret;
1793 
1794 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1795 	if (ret)
1796 		return ret;
1797 
1798 	guard(mutex)(&event_mutex);
1799 
1800 	switch (val) {
1801 	case 0:
1802 	case 1:
1803 		file = event_file_file(filp);
1804 		if (!file)
1805 			return -ENODEV;
1806 		ret = tracing_update_buffers(file->tr);
1807 		if (ret < 0)
1808 			return ret;
1809 		ret = ftrace_event_enable_disable(file, val);
1810 		if (ret < 0)
1811 			return ret;
1812 		break;
1813 
1814 	default:
1815 		return -EINVAL;
1816 	}
1817 
1818 	*ppos += cnt;
1819 
1820 	return cnt;
1821 }
1822 
1823 static ssize_t
system_enable_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1824 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1825 		   loff_t *ppos)
1826 {
1827 	const char set_to_char[4] = { '?', '0', '1', 'X' };
1828 	struct trace_subsystem_dir *dir = filp->private_data;
1829 	struct event_subsystem *system = dir->subsystem;
1830 	struct trace_event_call *call;
1831 	struct trace_event_file *file;
1832 	struct trace_array *tr = dir->tr;
1833 	char buf[2];
1834 	int set = 0;
1835 	int ret;
1836 
1837 	mutex_lock(&event_mutex);
1838 	list_for_each_entry(file, &tr->events, list) {
1839 		call = file->event_call;
1840 		if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1841 		    !trace_event_name(call) || !call->class || !call->class->reg)
1842 			continue;
1843 
1844 		if (system && strcmp(call->class->system, system->name) != 0)
1845 			continue;
1846 
1847 		/*
1848 		 * We need to find out if all the events are set
1849 		 * or if all events or cleared, or if we have
1850 		 * a mixture.
1851 		 */
1852 		set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1853 
1854 		/*
1855 		 * If we have a mixture, no need to look further.
1856 		 */
1857 		if (set == 3)
1858 			break;
1859 	}
1860 	mutex_unlock(&event_mutex);
1861 
1862 	buf[0] = set_to_char[set];
1863 	buf[1] = '\n';
1864 
1865 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1866 
1867 	return ret;
1868 }
1869 
1870 static ssize_t
system_enable_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1871 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1872 		    loff_t *ppos)
1873 {
1874 	struct trace_subsystem_dir *dir = filp->private_data;
1875 	struct event_subsystem *system = dir->subsystem;
1876 	const char *name = NULL;
1877 	unsigned long val;
1878 	ssize_t ret;
1879 
1880 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1881 	if (ret)
1882 		return ret;
1883 
1884 	ret = tracing_update_buffers(dir->tr);
1885 	if (ret < 0)
1886 		return ret;
1887 
1888 	if (val != 0 && val != 1)
1889 		return -EINVAL;
1890 
1891 	/*
1892 	 * Opening of "enable" adds a ref count to system,
1893 	 * so the name is safe to use.
1894 	 */
1895 	if (system)
1896 		name = system->name;
1897 
1898 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val, NULL);
1899 	if (ret)
1900 		goto out;
1901 
1902 	ret = cnt;
1903 
1904 out:
1905 	*ppos += cnt;
1906 
1907 	return ret;
1908 }
1909 
1910 enum {
1911 	FORMAT_HEADER		= 1,
1912 	FORMAT_FIELD_SEPERATOR	= 2,
1913 	FORMAT_PRINTFMT		= 3,
1914 };
1915 
f_next(struct seq_file * m,void * v,loff_t * pos)1916 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1917 {
1918 	struct trace_event_file *file = event_file_data(m->private);
1919 	struct trace_event_call *call = file->event_call;
1920 	struct list_head *common_head = &ftrace_common_fields;
1921 	struct list_head *head = trace_get_fields(call);
1922 	struct list_head *node = v;
1923 
1924 	(*pos)++;
1925 
1926 	switch ((unsigned long)v) {
1927 	case FORMAT_HEADER:
1928 		node = common_head;
1929 		break;
1930 
1931 	case FORMAT_FIELD_SEPERATOR:
1932 		node = head;
1933 		break;
1934 
1935 	case FORMAT_PRINTFMT:
1936 		/* all done */
1937 		return NULL;
1938 	}
1939 
1940 	node = node->prev;
1941 	if (node == common_head)
1942 		return (void *)FORMAT_FIELD_SEPERATOR;
1943 	else if (node == head)
1944 		return (void *)FORMAT_PRINTFMT;
1945 	else
1946 		return node;
1947 }
1948 
f_show(struct seq_file * m,void * v)1949 static int f_show(struct seq_file *m, void *v)
1950 {
1951 	struct trace_event_file *file = event_file_data(m->private);
1952 	struct trace_event_call *call = file->event_call;
1953 	struct ftrace_event_field *field;
1954 	const char *array_descriptor;
1955 
1956 	switch ((unsigned long)v) {
1957 	case FORMAT_HEADER:
1958 		seq_printf(m, "name: %s\n", trace_event_name(call));
1959 		seq_printf(m, "ID: %d\n", call->event.type);
1960 		seq_puts(m, "format:\n");
1961 		return 0;
1962 
1963 	case FORMAT_FIELD_SEPERATOR:
1964 		seq_putc(m, '\n');
1965 		return 0;
1966 
1967 	case FORMAT_PRINTFMT:
1968 		seq_printf(m, "\nprint fmt: %s\n",
1969 			   call->print_fmt);
1970 		return 0;
1971 	}
1972 
1973 	field = list_entry(v, struct ftrace_event_field, link);
1974 	/*
1975 	 * Smartly shows the array type(except dynamic array).
1976 	 * Normal:
1977 	 *	field:TYPE VAR
1978 	 * If TYPE := TYPE[LEN], it is shown:
1979 	 *	field:TYPE VAR[LEN]
1980 	 */
1981 	array_descriptor = strchr(field->type, '[');
1982 
1983 	if (str_has_prefix(field->type, "__data_loc"))
1984 		array_descriptor = NULL;
1985 
1986 	if (!array_descriptor)
1987 		seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1988 			   field->type, field->name, field->offset,
1989 			   field->size, !!field->is_signed);
1990 	else if (field->len)
1991 		seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1992 			   (int)(array_descriptor - field->type),
1993 			   field->type, field->name,
1994 			   field->len, field->offset,
1995 			   field->size, !!field->is_signed);
1996 	else
1997 		seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1998 				(int)(array_descriptor - field->type),
1999 				field->type, field->name,
2000 				field->offset, field->size, !!field->is_signed);
2001 
2002 	return 0;
2003 }
2004 
f_start(struct seq_file * m,loff_t * pos)2005 static void *f_start(struct seq_file *m, loff_t *pos)
2006 {
2007 	struct trace_event_file *file;
2008 	void *p = (void *)FORMAT_HEADER;
2009 	loff_t l = 0;
2010 
2011 	/* ->stop() is called even if ->start() fails */
2012 	mutex_lock(&event_mutex);
2013 	file = event_file_file(m->private);
2014 	if (!file)
2015 		return ERR_PTR(-ENODEV);
2016 
2017 	while (l < *pos && p)
2018 		p = f_next(m, p, &l);
2019 
2020 	return p;
2021 }
2022 
f_stop(struct seq_file * m,void * p)2023 static void f_stop(struct seq_file *m, void *p)
2024 {
2025 	mutex_unlock(&event_mutex);
2026 }
2027 
2028 static const struct seq_operations trace_format_seq_ops = {
2029 	.start		= f_start,
2030 	.next		= f_next,
2031 	.stop		= f_stop,
2032 	.show		= f_show,
2033 };
2034 
trace_format_open(struct inode * inode,struct file * file)2035 static int trace_format_open(struct inode *inode, struct file *file)
2036 {
2037 	struct seq_file *m;
2038 	int ret;
2039 
2040 	/* Do we want to hide event format files on tracefs lockdown? */
2041 
2042 	ret = seq_open(file, &trace_format_seq_ops);
2043 	if (ret < 0)
2044 		return ret;
2045 
2046 	m = file->private_data;
2047 	m->private = file;
2048 
2049 	return 0;
2050 }
2051 
2052 #ifdef CONFIG_PERF_EVENTS
2053 static ssize_t
event_id_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2054 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2055 {
2056 	int id = (long)event_file_data(filp);
2057 	char buf[32];
2058 	int len;
2059 
2060 	if (unlikely(!id))
2061 		return -ENODEV;
2062 
2063 	len = sprintf(buf, "%d\n", id);
2064 
2065 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
2066 }
2067 #endif
2068 
2069 static ssize_t
event_filter_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2070 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2071 		  loff_t *ppos)
2072 {
2073 	struct trace_event_file *file;
2074 	struct trace_seq *s;
2075 	int r = -ENODEV;
2076 
2077 	if (*ppos)
2078 		return 0;
2079 
2080 	s = kmalloc(sizeof(*s), GFP_KERNEL);
2081 
2082 	if (!s)
2083 		return -ENOMEM;
2084 
2085 	trace_seq_init(s);
2086 
2087 	mutex_lock(&event_mutex);
2088 	file = event_file_file(filp);
2089 	if (file)
2090 		print_event_filter(file, s);
2091 	mutex_unlock(&event_mutex);
2092 
2093 	if (file)
2094 		r = simple_read_from_buffer(ubuf, cnt, ppos,
2095 					    s->buffer, trace_seq_used(s));
2096 
2097 	kfree(s);
2098 
2099 	return r;
2100 }
2101 
2102 static ssize_t
event_filter_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2103 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2104 		   loff_t *ppos)
2105 {
2106 	struct trace_event_file *file;
2107 	char *buf;
2108 	int err = -ENODEV;
2109 
2110 	if (cnt >= PAGE_SIZE)
2111 		return -EINVAL;
2112 
2113 	buf = memdup_user_nul(ubuf, cnt);
2114 	if (IS_ERR(buf))
2115 		return PTR_ERR(buf);
2116 
2117 	mutex_lock(&event_mutex);
2118 	file = event_file_file(filp);
2119 	if (file) {
2120 		if (file->flags & EVENT_FILE_FL_FREED)
2121 			err = -ENODEV;
2122 		else
2123 			err = apply_event_filter(file, buf);
2124 	}
2125 	mutex_unlock(&event_mutex);
2126 
2127 	kfree(buf);
2128 	if (err < 0)
2129 		return err;
2130 
2131 	*ppos += cnt;
2132 
2133 	return cnt;
2134 }
2135 
2136 static LIST_HEAD(event_subsystems);
2137 
subsystem_open(struct inode * inode,struct file * filp)2138 static int subsystem_open(struct inode *inode, struct file *filp)
2139 {
2140 	struct trace_subsystem_dir *dir = NULL, *iter_dir;
2141 	struct trace_array *tr = NULL, *iter_tr;
2142 	struct event_subsystem *system = NULL;
2143 	int ret;
2144 
2145 	if (tracing_is_disabled())
2146 		return -ENODEV;
2147 
2148 	/* Make sure the system still exists */
2149 	mutex_lock(&event_mutex);
2150 	mutex_lock(&trace_types_lock);
2151 	list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
2152 		list_for_each_entry(iter_dir, &iter_tr->systems, list) {
2153 			if (iter_dir == inode->i_private) {
2154 				/* Don't open systems with no events */
2155 				tr = iter_tr;
2156 				dir = iter_dir;
2157 				if (dir->nr_events) {
2158 					__get_system_dir(dir);
2159 					system = dir->subsystem;
2160 				}
2161 				goto exit_loop;
2162 			}
2163 		}
2164 	}
2165  exit_loop:
2166 	mutex_unlock(&trace_types_lock);
2167 	mutex_unlock(&event_mutex);
2168 
2169 	if (!system)
2170 		return -ENODEV;
2171 
2172 	/* Still need to increment the ref count of the system */
2173 	if (trace_array_get(tr) < 0) {
2174 		put_system(dir);
2175 		return -ENODEV;
2176 	}
2177 
2178 	ret = tracing_open_generic(inode, filp);
2179 	if (ret < 0) {
2180 		trace_array_put(tr);
2181 		put_system(dir);
2182 	}
2183 
2184 	return ret;
2185 }
2186 
system_tr_open(struct inode * inode,struct file * filp)2187 static int system_tr_open(struct inode *inode, struct file *filp)
2188 {
2189 	struct trace_subsystem_dir *dir;
2190 	struct trace_array *tr = inode->i_private;
2191 	int ret;
2192 
2193 	/* Make a temporary dir that has no system but points to tr */
2194 	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
2195 	if (!dir)
2196 		return -ENOMEM;
2197 
2198 	ret = tracing_open_generic_tr(inode, filp);
2199 	if (ret < 0) {
2200 		kfree(dir);
2201 		return ret;
2202 	}
2203 	dir->tr = tr;
2204 	filp->private_data = dir;
2205 
2206 	return 0;
2207 }
2208 
subsystem_release(struct inode * inode,struct file * file)2209 static int subsystem_release(struct inode *inode, struct file *file)
2210 {
2211 	struct trace_subsystem_dir *dir = file->private_data;
2212 
2213 	trace_array_put(dir->tr);
2214 
2215 	/*
2216 	 * If dir->subsystem is NULL, then this is a temporary
2217 	 * descriptor that was made for a trace_array to enable
2218 	 * all subsystems.
2219 	 */
2220 	if (dir->subsystem)
2221 		put_system(dir);
2222 	else
2223 		kfree(dir);
2224 
2225 	return 0;
2226 }
2227 
2228 static ssize_t
subsystem_filter_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2229 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2230 		      loff_t *ppos)
2231 {
2232 	struct trace_subsystem_dir *dir = filp->private_data;
2233 	struct event_subsystem *system = dir->subsystem;
2234 	struct trace_seq *s;
2235 	int r;
2236 
2237 	if (*ppos)
2238 		return 0;
2239 
2240 	s = kmalloc(sizeof(*s), GFP_KERNEL);
2241 	if (!s)
2242 		return -ENOMEM;
2243 
2244 	trace_seq_init(s);
2245 
2246 	print_subsystem_event_filter(system, s);
2247 	r = simple_read_from_buffer(ubuf, cnt, ppos,
2248 				    s->buffer, trace_seq_used(s));
2249 
2250 	kfree(s);
2251 
2252 	return r;
2253 }
2254 
2255 static ssize_t
subsystem_filter_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2256 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2257 		       loff_t *ppos)
2258 {
2259 	struct trace_subsystem_dir *dir = filp->private_data;
2260 	char *buf;
2261 	int err;
2262 
2263 	if (cnt >= PAGE_SIZE)
2264 		return -EINVAL;
2265 
2266 	buf = memdup_user_nul(ubuf, cnt);
2267 	if (IS_ERR(buf))
2268 		return PTR_ERR(buf);
2269 
2270 	err = apply_subsystem_event_filter(dir, buf);
2271 	kfree(buf);
2272 	if (err < 0)
2273 		return err;
2274 
2275 	*ppos += cnt;
2276 
2277 	return cnt;
2278 }
2279 
2280 static ssize_t
show_header_page_file(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2281 show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2282 {
2283 	struct trace_array *tr = filp->private_data;
2284 	struct trace_seq *s;
2285 	int r;
2286 
2287 	if (*ppos)
2288 		return 0;
2289 
2290 	s = kmalloc(sizeof(*s), GFP_KERNEL);
2291 	if (!s)
2292 		return -ENOMEM;
2293 
2294 	trace_seq_init(s);
2295 
2296 	ring_buffer_print_page_header(tr->array_buffer.buffer, s);
2297 	r = simple_read_from_buffer(ubuf, cnt, ppos,
2298 				    s->buffer, trace_seq_used(s));
2299 
2300 	kfree(s);
2301 
2302 	return r;
2303 }
2304 
2305 static ssize_t
show_header_event_file(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)2306 show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2307 {
2308 	struct trace_seq *s;
2309 	int r;
2310 
2311 	if (*ppos)
2312 		return 0;
2313 
2314 	s = kmalloc(sizeof(*s), GFP_KERNEL);
2315 	if (!s)
2316 		return -ENOMEM;
2317 
2318 	trace_seq_init(s);
2319 
2320 	ring_buffer_print_entry_header(s);
2321 	r = simple_read_from_buffer(ubuf, cnt, ppos,
2322 				    s->buffer, trace_seq_used(s));
2323 
2324 	kfree(s);
2325 
2326 	return r;
2327 }
2328 
ignore_task_cpu(void * data)2329 static void ignore_task_cpu(void *data)
2330 {
2331 	struct trace_array *tr = data;
2332 	struct trace_pid_list *pid_list;
2333 	struct trace_pid_list *no_pid_list;
2334 
2335 	/*
2336 	 * This function is called by on_each_cpu() while the
2337 	 * event_mutex is held.
2338 	 */
2339 	pid_list = rcu_dereference_protected(tr->filtered_pids,
2340 					     mutex_is_locked(&event_mutex));
2341 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2342 					     mutex_is_locked(&event_mutex));
2343 
2344 	this_cpu_write(tr->array_buffer.data->ignore_pid,
2345 		       trace_ignore_this_task(pid_list, no_pid_list, current));
2346 }
2347 
register_pid_events(struct trace_array * tr)2348 static void register_pid_events(struct trace_array *tr)
2349 {
2350 	/*
2351 	 * Register a probe that is called before all other probes
2352 	 * to set ignore_pid if next or prev do not match.
2353 	 * Register a probe this is called after all other probes
2354 	 * to only keep ignore_pid set if next pid matches.
2355 	 */
2356 	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
2357 					 tr, INT_MAX);
2358 	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
2359 					 tr, 0);
2360 
2361 	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
2362 					 tr, INT_MAX);
2363 	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
2364 					 tr, 0);
2365 
2366 	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
2367 					     tr, INT_MAX);
2368 	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
2369 					     tr, 0);
2370 
2371 	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
2372 					 tr, INT_MAX);
2373 	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
2374 					 tr, 0);
2375 }
2376 
2377 static ssize_t
event_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos,int type)2378 event_pid_write(struct file *filp, const char __user *ubuf,
2379 		size_t cnt, loff_t *ppos, int type)
2380 {
2381 	struct seq_file *m = filp->private_data;
2382 	struct trace_array *tr = m->private;
2383 	struct trace_pid_list *filtered_pids = NULL;
2384 	struct trace_pid_list *other_pids = NULL;
2385 	struct trace_pid_list *pid_list;
2386 	struct trace_event_file *file;
2387 	ssize_t ret;
2388 
2389 	if (!cnt)
2390 		return 0;
2391 
2392 	ret = tracing_update_buffers(tr);
2393 	if (ret < 0)
2394 		return ret;
2395 
2396 	guard(mutex)(&event_mutex);
2397 
2398 	if (type == TRACE_PIDS) {
2399 		filtered_pids = rcu_dereference_protected(tr->filtered_pids,
2400 							  lockdep_is_held(&event_mutex));
2401 		other_pids = rcu_dereference_protected(tr->filtered_no_pids,
2402 							  lockdep_is_held(&event_mutex));
2403 	} else {
2404 		filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
2405 							  lockdep_is_held(&event_mutex));
2406 		other_pids = rcu_dereference_protected(tr->filtered_pids,
2407 							  lockdep_is_held(&event_mutex));
2408 	}
2409 
2410 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
2411 	if (ret < 0)
2412 		return ret;
2413 
2414 	if (type == TRACE_PIDS)
2415 		rcu_assign_pointer(tr->filtered_pids, pid_list);
2416 	else
2417 		rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2418 
2419 	list_for_each_entry(file, &tr->events, list) {
2420 		set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2421 	}
2422 
2423 	if (filtered_pids) {
2424 		tracepoint_synchronize_unregister();
2425 		trace_pid_list_free(filtered_pids);
2426 	} else if (pid_list && !other_pids) {
2427 		register_pid_events(tr);
2428 	}
2429 
2430 	/*
2431 	 * Ignoring of pids is done at task switch. But we have to
2432 	 * check for those tasks that are currently running.
2433 	 * Always do this in case a pid was appended or removed.
2434 	 */
2435 	on_each_cpu(ignore_task_cpu, tr, 1);
2436 
2437 	*ppos += ret;
2438 
2439 	return ret;
2440 }
2441 
2442 static ssize_t
ftrace_event_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2443 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2444 		       size_t cnt, loff_t *ppos)
2445 {
2446 	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2447 }
2448 
2449 static ssize_t
ftrace_event_npid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2450 ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2451 			size_t cnt, loff_t *ppos)
2452 {
2453 	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2454 }
2455 
2456 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2457 static int ftrace_event_set_open(struct inode *inode, struct file *file);
2458 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2459 static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2460 static int ftrace_event_release(struct inode *inode, struct file *file);
2461 
2462 static const struct seq_operations show_event_seq_ops = {
2463 	.start = t_start,
2464 	.next = t_next,
2465 	.show = t_show,
2466 	.stop = t_stop,
2467 };
2468 
2469 static const struct seq_operations show_set_event_seq_ops = {
2470 	.start = s_start,
2471 	.next = s_next,
2472 	.show = s_show,
2473 	.stop = s_stop,
2474 };
2475 
2476 static const struct seq_operations show_set_pid_seq_ops = {
2477 	.start = p_start,
2478 	.next = p_next,
2479 	.show = trace_pid_show,
2480 	.stop = p_stop,
2481 };
2482 
2483 static const struct seq_operations show_set_no_pid_seq_ops = {
2484 	.start = np_start,
2485 	.next = np_next,
2486 	.show = trace_pid_show,
2487 	.stop = p_stop,
2488 };
2489 
2490 static const struct file_operations ftrace_avail_fops = {
2491 	.open = ftrace_event_avail_open,
2492 	.read = seq_read,
2493 	.llseek = seq_lseek,
2494 	.release = seq_release,
2495 };
2496 
2497 static const struct file_operations ftrace_set_event_fops = {
2498 	.open = ftrace_event_set_open,
2499 	.read = seq_read,
2500 	.write = ftrace_event_write,
2501 	.llseek = seq_lseek,
2502 	.release = ftrace_event_release,
2503 };
2504 
2505 static const struct file_operations ftrace_set_event_pid_fops = {
2506 	.open = ftrace_event_set_pid_open,
2507 	.read = seq_read,
2508 	.write = ftrace_event_pid_write,
2509 	.llseek = seq_lseek,
2510 	.release = ftrace_event_release,
2511 };
2512 
2513 static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2514 	.open = ftrace_event_set_npid_open,
2515 	.read = seq_read,
2516 	.write = ftrace_event_npid_write,
2517 	.llseek = seq_lseek,
2518 	.release = ftrace_event_release,
2519 };
2520 
2521 static const struct file_operations ftrace_enable_fops = {
2522 	.open = tracing_open_file_tr,
2523 	.read = event_enable_read,
2524 	.write = event_enable_write,
2525 	.release = tracing_release_file_tr,
2526 	.llseek = default_llseek,
2527 };
2528 
2529 static const struct file_operations ftrace_event_format_fops = {
2530 	.open = trace_format_open,
2531 	.read = seq_read,
2532 	.llseek = seq_lseek,
2533 	.release = seq_release,
2534 };
2535 
2536 #ifdef CONFIG_PERF_EVENTS
2537 static const struct file_operations ftrace_event_id_fops = {
2538 	.read = event_id_read,
2539 	.llseek = default_llseek,
2540 };
2541 #endif
2542 
2543 static const struct file_operations ftrace_event_filter_fops = {
2544 	.open = tracing_open_file_tr,
2545 	.read = event_filter_read,
2546 	.write = event_filter_write,
2547 	.release = tracing_release_file_tr,
2548 	.llseek = default_llseek,
2549 };
2550 
2551 static const struct file_operations ftrace_subsystem_filter_fops = {
2552 	.open = subsystem_open,
2553 	.read = subsystem_filter_read,
2554 	.write = subsystem_filter_write,
2555 	.llseek = default_llseek,
2556 	.release = subsystem_release,
2557 };
2558 
2559 static const struct file_operations ftrace_system_enable_fops = {
2560 	.open = subsystem_open,
2561 	.read = system_enable_read,
2562 	.write = system_enable_write,
2563 	.llseek = default_llseek,
2564 	.release = subsystem_release,
2565 };
2566 
2567 static const struct file_operations ftrace_tr_enable_fops = {
2568 	.open = system_tr_open,
2569 	.read = system_enable_read,
2570 	.write = system_enable_write,
2571 	.llseek = default_llseek,
2572 	.release = subsystem_release,
2573 };
2574 
2575 static const struct file_operations ftrace_show_header_page_fops = {
2576 	.open = tracing_open_generic_tr,
2577 	.read = show_header_page_file,
2578 	.llseek = default_llseek,
2579 	.release = tracing_release_generic_tr,
2580 };
2581 
2582 static const struct file_operations ftrace_show_header_event_fops = {
2583 	.open = tracing_open_generic_tr,
2584 	.read = show_header_event_file,
2585 	.llseek = default_llseek,
2586 	.release = tracing_release_generic_tr,
2587 };
2588 
2589 static int
ftrace_event_open(struct inode * inode,struct file * file,const struct seq_operations * seq_ops)2590 ftrace_event_open(struct inode *inode, struct file *file,
2591 		  const struct seq_operations *seq_ops)
2592 {
2593 	struct seq_file *m;
2594 	int ret;
2595 
2596 	ret = security_locked_down(LOCKDOWN_TRACEFS);
2597 	if (ret)
2598 		return ret;
2599 
2600 	ret = seq_open(file, seq_ops);
2601 	if (ret < 0)
2602 		return ret;
2603 	m = file->private_data;
2604 	/* copy tr over to seq ops */
2605 	m->private = inode->i_private;
2606 
2607 	return ret;
2608 }
2609 
ftrace_event_release(struct inode * inode,struct file * file)2610 static int ftrace_event_release(struct inode *inode, struct file *file)
2611 {
2612 	struct trace_array *tr = inode->i_private;
2613 
2614 	trace_array_put(tr);
2615 
2616 	return seq_release(inode, file);
2617 }
2618 
2619 static int
ftrace_event_avail_open(struct inode * inode,struct file * file)2620 ftrace_event_avail_open(struct inode *inode, struct file *file)
2621 {
2622 	const struct seq_operations *seq_ops = &show_event_seq_ops;
2623 
2624 	/* Checks for tracefs lockdown */
2625 	return ftrace_event_open(inode, file, seq_ops);
2626 }
2627 
2628 static int
ftrace_event_set_open(struct inode * inode,struct file * file)2629 ftrace_event_set_open(struct inode *inode, struct file *file)
2630 {
2631 	const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2632 	struct trace_array *tr = inode->i_private;
2633 	int ret;
2634 
2635 	ret = tracing_check_open_get_tr(tr);
2636 	if (ret)
2637 		return ret;
2638 
2639 	if ((file->f_mode & FMODE_WRITE) &&
2640 	    (file->f_flags & O_TRUNC))
2641 		ftrace_clear_events(tr);
2642 
2643 	ret = ftrace_event_open(inode, file, seq_ops);
2644 	if (ret < 0)
2645 		trace_array_put(tr);
2646 	return ret;
2647 }
2648 
2649 static int
ftrace_event_set_pid_open(struct inode * inode,struct file * file)2650 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2651 {
2652 	const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2653 	struct trace_array *tr = inode->i_private;
2654 	int ret;
2655 
2656 	ret = tracing_check_open_get_tr(tr);
2657 	if (ret)
2658 		return ret;
2659 
2660 	if ((file->f_mode & FMODE_WRITE) &&
2661 	    (file->f_flags & O_TRUNC))
2662 		ftrace_clear_event_pids(tr, TRACE_PIDS);
2663 
2664 	ret = ftrace_event_open(inode, file, seq_ops);
2665 	if (ret < 0)
2666 		trace_array_put(tr);
2667 	return ret;
2668 }
2669 
2670 static int
ftrace_event_set_npid_open(struct inode * inode,struct file * file)2671 ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2672 {
2673 	const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2674 	struct trace_array *tr = inode->i_private;
2675 	int ret;
2676 
2677 	ret = tracing_check_open_get_tr(tr);
2678 	if (ret)
2679 		return ret;
2680 
2681 	if ((file->f_mode & FMODE_WRITE) &&
2682 	    (file->f_flags & O_TRUNC))
2683 		ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2684 
2685 	ret = ftrace_event_open(inode, file, seq_ops);
2686 	if (ret < 0)
2687 		trace_array_put(tr);
2688 	return ret;
2689 }
2690 
2691 static struct event_subsystem *
create_new_subsystem(const char * name)2692 create_new_subsystem(const char *name)
2693 {
2694 	struct event_subsystem *system;
2695 
2696 	/* need to create new entry */
2697 	system = kmalloc(sizeof(*system), GFP_KERNEL);
2698 	if (!system)
2699 		return NULL;
2700 
2701 	system->ref_count = 1;
2702 
2703 	/* Only allocate if dynamic (kprobes and modules) */
2704 	system->name = kstrdup_const(name, GFP_KERNEL);
2705 	if (!system->name)
2706 		goto out_free;
2707 
2708 	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2709 	if (!system->filter)
2710 		goto out_free;
2711 
2712 	list_add(&system->list, &event_subsystems);
2713 
2714 	return system;
2715 
2716  out_free:
2717 	kfree_const(system->name);
2718 	kfree(system);
2719 	return NULL;
2720 }
2721 
system_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)2722 static int system_callback(const char *name, umode_t *mode, void **data,
2723 		    const struct file_operations **fops)
2724 {
2725 	if (strcmp(name, "filter") == 0)
2726 		*fops = &ftrace_subsystem_filter_fops;
2727 
2728 	else if (strcmp(name, "enable") == 0)
2729 		*fops = &ftrace_system_enable_fops;
2730 
2731 	else
2732 		return 0;
2733 
2734 	*mode = TRACE_MODE_WRITE;
2735 	return 1;
2736 }
2737 
2738 static struct eventfs_inode *
event_subsystem_dir(struct trace_array * tr,const char * name,struct trace_event_file * file,struct eventfs_inode * parent)2739 event_subsystem_dir(struct trace_array *tr, const char *name,
2740 		    struct trace_event_file *file, struct eventfs_inode *parent)
2741 {
2742 	struct event_subsystem *system, *iter;
2743 	struct trace_subsystem_dir *dir;
2744 	struct eventfs_inode *ei;
2745 	int nr_entries;
2746 	static struct eventfs_entry system_entries[] = {
2747 		{
2748 			.name		= "filter",
2749 			.callback	= system_callback,
2750 		},
2751 		{
2752 			.name		= "enable",
2753 			.callback	= system_callback,
2754 		}
2755 	};
2756 
2757 	/* First see if we did not already create this dir */
2758 	list_for_each_entry(dir, &tr->systems, list) {
2759 		system = dir->subsystem;
2760 		if (strcmp(system->name, name) == 0) {
2761 			dir->nr_events++;
2762 			file->system = dir;
2763 			return dir->ei;
2764 		}
2765 	}
2766 
2767 	/* Now see if the system itself exists. */
2768 	system = NULL;
2769 	list_for_each_entry(iter, &event_subsystems, list) {
2770 		if (strcmp(iter->name, name) == 0) {
2771 			system = iter;
2772 			break;
2773 		}
2774 	}
2775 
2776 	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2777 	if (!dir)
2778 		goto out_fail;
2779 
2780 	if (!system) {
2781 		system = create_new_subsystem(name);
2782 		if (!system)
2783 			goto out_free;
2784 	} else
2785 		__get_system(system);
2786 
2787 	/* ftrace only has directories no files */
2788 	if (strcmp(name, "ftrace") == 0)
2789 		nr_entries = 0;
2790 	else
2791 		nr_entries = ARRAY_SIZE(system_entries);
2792 
2793 	ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir);
2794 	if (IS_ERR(ei)) {
2795 		pr_warn("Failed to create system directory %s\n", name);
2796 		__put_system(system);
2797 		goto out_free;
2798 	}
2799 
2800 	dir->ei = ei;
2801 	dir->tr = tr;
2802 	dir->ref_count = 1;
2803 	dir->nr_events = 1;
2804 	dir->subsystem = system;
2805 	file->system = dir;
2806 
2807 	list_add(&dir->list, &tr->systems);
2808 
2809 	return dir->ei;
2810 
2811  out_free:
2812 	kfree(dir);
2813  out_fail:
2814 	/* Only print this message if failed on memory allocation */
2815 	if (!dir || !system)
2816 		pr_warn("No memory to create event subsystem %s\n", name);
2817 	return NULL;
2818 }
2819 
2820 static int
event_define_fields(struct trace_event_call * call)2821 event_define_fields(struct trace_event_call *call)
2822 {
2823 	struct list_head *head;
2824 	int ret = 0;
2825 
2826 	/*
2827 	 * Other events may have the same class. Only update
2828 	 * the fields if they are not already defined.
2829 	 */
2830 	head = trace_get_fields(call);
2831 	if (list_empty(head)) {
2832 		struct trace_event_fields *field = call->class->fields_array;
2833 		unsigned int offset = sizeof(struct trace_entry);
2834 
2835 		for (; field->type; field++) {
2836 			if (field->type == TRACE_FUNCTION_TYPE) {
2837 				field->define_fields(call);
2838 				break;
2839 			}
2840 
2841 			offset = ALIGN(offset, field->align);
2842 			ret = trace_define_field_ext(call, field->type, field->name,
2843 						 offset, field->size,
2844 						 field->is_signed, field->filter_type,
2845 						 field->len, field->needs_test);
2846 			if (WARN_ON_ONCE(ret)) {
2847 				pr_err("error code is %d\n", ret);
2848 				break;
2849 			}
2850 
2851 			offset += field->size;
2852 		}
2853 	}
2854 
2855 	return ret;
2856 }
2857 
event_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)2858 static int event_callback(const char *name, umode_t *mode, void **data,
2859 			  const struct file_operations **fops)
2860 {
2861 	struct trace_event_file *file = *data;
2862 	struct trace_event_call *call = file->event_call;
2863 
2864 	if (strcmp(name, "format") == 0) {
2865 		*mode = TRACE_MODE_READ;
2866 		*fops = &ftrace_event_format_fops;
2867 		return 1;
2868 	}
2869 
2870 	/*
2871 	 * Only event directories that can be enabled should have
2872 	 * triggers or filters, with the exception of the "print"
2873 	 * event that can have a "trigger" file.
2874 	 */
2875 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2876 		if (call->class->reg && strcmp(name, "enable") == 0) {
2877 			*mode = TRACE_MODE_WRITE;
2878 			*fops = &ftrace_enable_fops;
2879 			return 1;
2880 		}
2881 
2882 		if (strcmp(name, "filter") == 0) {
2883 			*mode = TRACE_MODE_WRITE;
2884 			*fops = &ftrace_event_filter_fops;
2885 			return 1;
2886 		}
2887 	}
2888 
2889 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
2890 	    strcmp(trace_event_name(call), "print") == 0) {
2891 		if (strcmp(name, "trigger") == 0) {
2892 			*mode = TRACE_MODE_WRITE;
2893 			*fops = &event_trigger_fops;
2894 			return 1;
2895 		}
2896 	}
2897 
2898 #ifdef CONFIG_PERF_EVENTS
2899 	if (call->event.type && call->class->reg &&
2900 	    strcmp(name, "id") == 0) {
2901 		*mode = TRACE_MODE_READ;
2902 		*data = (void *)(long)call->event.type;
2903 		*fops = &ftrace_event_id_fops;
2904 		return 1;
2905 	}
2906 #endif
2907 
2908 #ifdef CONFIG_HIST_TRIGGERS
2909 	if (strcmp(name, "hist") == 0) {
2910 		*mode = TRACE_MODE_READ;
2911 		*fops = &event_hist_fops;
2912 		return 1;
2913 	}
2914 #endif
2915 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2916 	if (strcmp(name, "hist_debug") == 0) {
2917 		*mode = TRACE_MODE_READ;
2918 		*fops = &event_hist_debug_fops;
2919 		return 1;
2920 	}
2921 #endif
2922 #ifdef CONFIG_TRACE_EVENT_INJECT
2923 	if (call->event.type && call->class->reg &&
2924 	    strcmp(name, "inject") == 0) {
2925 		*mode = 0200;
2926 		*fops = &event_inject_fops;
2927 		return 1;
2928 	}
2929 #endif
2930 	return 0;
2931 }
2932 
2933 /* The file is incremented on creation and freeing the enable file decrements it */
event_release(const char * name,void * data)2934 static void event_release(const char *name, void *data)
2935 {
2936 	struct trace_event_file *file = data;
2937 
2938 	event_file_put(file);
2939 }
2940 
2941 static int
event_create_dir(struct eventfs_inode * parent,struct trace_event_file * file)2942 event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
2943 {
2944 	struct trace_event_call *call = file->event_call;
2945 	struct trace_array *tr = file->tr;
2946 	struct eventfs_inode *e_events;
2947 	struct eventfs_inode *ei;
2948 	const char *name;
2949 	int nr_entries;
2950 	int ret;
2951 	static struct eventfs_entry event_entries[] = {
2952 		{
2953 			.name		= "enable",
2954 			.callback	= event_callback,
2955 			.release	= event_release,
2956 		},
2957 		{
2958 			.name		= "filter",
2959 			.callback	= event_callback,
2960 		},
2961 		{
2962 			.name		= "trigger",
2963 			.callback	= event_callback,
2964 		},
2965 		{
2966 			.name		= "format",
2967 			.callback	= event_callback,
2968 		},
2969 #ifdef CONFIG_PERF_EVENTS
2970 		{
2971 			.name		= "id",
2972 			.callback	= event_callback,
2973 		},
2974 #endif
2975 #ifdef CONFIG_HIST_TRIGGERS
2976 		{
2977 			.name		= "hist",
2978 			.callback	= event_callback,
2979 		},
2980 #endif
2981 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2982 		{
2983 			.name		= "hist_debug",
2984 			.callback	= event_callback,
2985 		},
2986 #endif
2987 #ifdef CONFIG_TRACE_EVENT_INJECT
2988 		{
2989 			.name		= "inject",
2990 			.callback	= event_callback,
2991 		},
2992 #endif
2993 	};
2994 
2995 	/*
2996 	 * If the trace point header did not define TRACE_SYSTEM
2997 	 * then the system would be called "TRACE_SYSTEM". This should
2998 	 * never happen.
2999 	 */
3000 	if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
3001 		return -ENODEV;
3002 
3003 	e_events = event_subsystem_dir(tr, call->class->system, file, parent);
3004 	if (!e_events)
3005 		return -ENOMEM;
3006 
3007 	nr_entries = ARRAY_SIZE(event_entries);
3008 
3009 	name = trace_event_name(call);
3010 	ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file);
3011 	if (IS_ERR(ei)) {
3012 		pr_warn("Could not create tracefs '%s' directory\n", name);
3013 		return -1;
3014 	}
3015 
3016 	file->ei = ei;
3017 
3018 	ret = event_define_fields(call);
3019 	if (ret < 0) {
3020 		pr_warn("Could not initialize trace point events/%s\n", name);
3021 		return ret;
3022 	}
3023 
3024 	/* Gets decremented on freeing of the "enable" file */
3025 	event_file_get(file);
3026 
3027 	return 0;
3028 }
3029 
remove_event_from_tracers(struct trace_event_call * call)3030 static void remove_event_from_tracers(struct trace_event_call *call)
3031 {
3032 	struct trace_event_file *file;
3033 	struct trace_array *tr;
3034 
3035 	do_for_each_event_file_safe(tr, file) {
3036 		if (file->event_call != call)
3037 			continue;
3038 
3039 		remove_event_file_dir(file);
3040 		/*
3041 		 * The do_for_each_event_file_safe() is
3042 		 * a double loop. After finding the call for this
3043 		 * trace_array, we use break to jump to the next
3044 		 * trace_array.
3045 		 */
3046 		break;
3047 	} while_for_each_event_file();
3048 }
3049 
event_remove(struct trace_event_call * call)3050 static void event_remove(struct trace_event_call *call)
3051 {
3052 	struct trace_array *tr;
3053 	struct trace_event_file *file;
3054 
3055 	do_for_each_event_file(tr, file) {
3056 		if (file->event_call != call)
3057 			continue;
3058 
3059 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3060 			tr->clear_trace = true;
3061 
3062 		ftrace_event_enable_disable(file, 0);
3063 		/*
3064 		 * The do_for_each_event_file() is
3065 		 * a double loop. After finding the call for this
3066 		 * trace_array, we use break to jump to the next
3067 		 * trace_array.
3068 		 */
3069 		break;
3070 	} while_for_each_event_file();
3071 
3072 	if (call->event.funcs)
3073 		__unregister_trace_event(&call->event);
3074 	remove_event_from_tracers(call);
3075 	list_del(&call->list);
3076 }
3077 
event_init(struct trace_event_call * call)3078 static int event_init(struct trace_event_call *call)
3079 {
3080 	int ret = 0;
3081 	const char *name;
3082 
3083 	name = trace_event_name(call);
3084 	if (WARN_ON(!name))
3085 		return -EINVAL;
3086 
3087 	if (call->class->raw_init) {
3088 		ret = call->class->raw_init(call);
3089 		if (ret < 0 && ret != -ENOSYS)
3090 			pr_warn("Could not initialize trace events/%s\n", name);
3091 	}
3092 
3093 	return ret;
3094 }
3095 
3096 static int
__register_event(struct trace_event_call * call,struct module * mod)3097 __register_event(struct trace_event_call *call, struct module *mod)
3098 {
3099 	int ret;
3100 
3101 	ret = event_init(call);
3102 	if (ret < 0)
3103 		return ret;
3104 
3105 	list_add(&call->list, &ftrace_events);
3106 	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3107 		atomic_set(&call->refcnt, 0);
3108 	else
3109 		call->module = mod;
3110 
3111 	return 0;
3112 }
3113 
eval_replace(char * ptr,struct trace_eval_map * map,int len)3114 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
3115 {
3116 	int rlen;
3117 	int elen;
3118 
3119 	/* Find the length of the eval value as a string */
3120 	elen = snprintf(ptr, 0, "%ld", map->eval_value);
3121 	/* Make sure there's enough room to replace the string with the value */
3122 	if (len < elen)
3123 		return NULL;
3124 
3125 	snprintf(ptr, elen + 1, "%ld", map->eval_value);
3126 
3127 	/* Get the rest of the string of ptr */
3128 	rlen = strlen(ptr + len);
3129 	memmove(ptr + elen, ptr + len, rlen);
3130 	/* Make sure we end the new string */
3131 	ptr[elen + rlen] = 0;
3132 
3133 	return ptr + elen;
3134 }
3135 
update_event_printk(struct trace_event_call * call,struct trace_eval_map * map)3136 static void update_event_printk(struct trace_event_call *call,
3137 				struct trace_eval_map *map)
3138 {
3139 	char *ptr;
3140 	int quote = 0;
3141 	int len = strlen(map->eval_string);
3142 
3143 	for (ptr = call->print_fmt; *ptr; ptr++) {
3144 		if (*ptr == '\\') {
3145 			ptr++;
3146 			/* paranoid */
3147 			if (!*ptr)
3148 				break;
3149 			continue;
3150 		}
3151 		if (*ptr == '"') {
3152 			quote ^= 1;
3153 			continue;
3154 		}
3155 		if (quote)
3156 			continue;
3157 		if (isdigit(*ptr)) {
3158 			/* skip numbers */
3159 			do {
3160 				ptr++;
3161 				/* Check for alpha chars like ULL */
3162 			} while (isalnum(*ptr));
3163 			if (!*ptr)
3164 				break;
3165 			/*
3166 			 * A number must have some kind of delimiter after
3167 			 * it, and we can ignore that too.
3168 			 */
3169 			continue;
3170 		}
3171 		if (isalpha(*ptr) || *ptr == '_') {
3172 			if (strncmp(map->eval_string, ptr, len) == 0 &&
3173 			    !isalnum(ptr[len]) && ptr[len] != '_') {
3174 				ptr = eval_replace(ptr, map, len);
3175 				/* enum/sizeof string smaller than value */
3176 				if (WARN_ON_ONCE(!ptr))
3177 					return;
3178 				/*
3179 				 * No need to decrement here, as eval_replace()
3180 				 * returns the pointer to the character passed
3181 				 * the eval, and two evals can not be placed
3182 				 * back to back without something in between.
3183 				 * We can skip that something in between.
3184 				 */
3185 				continue;
3186 			}
3187 		skip_more:
3188 			do {
3189 				ptr++;
3190 			} while (isalnum(*ptr) || *ptr == '_');
3191 			if (!*ptr)
3192 				break;
3193 			/*
3194 			 * If what comes after this variable is a '.' or
3195 			 * '->' then we can continue to ignore that string.
3196 			 */
3197 			if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
3198 				ptr += *ptr == '.' ? 1 : 2;
3199 				if (!*ptr)
3200 					break;
3201 				goto skip_more;
3202 			}
3203 			/*
3204 			 * Once again, we can skip the delimiter that came
3205 			 * after the string.
3206 			 */
3207 			continue;
3208 		}
3209 	}
3210 }
3211 
add_str_to_module(struct module * module,char * str)3212 static void add_str_to_module(struct module *module, char *str)
3213 {
3214 	struct module_string *modstr;
3215 
3216 	modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
3217 
3218 	/*
3219 	 * If we failed to allocate memory here, then we'll just
3220 	 * let the str memory leak when the module is removed.
3221 	 * If this fails to allocate, there's worse problems than
3222 	 * a leaked string on module removal.
3223 	 */
3224 	if (WARN_ON_ONCE(!modstr))
3225 		return;
3226 
3227 	modstr->module = module;
3228 	modstr->str = str;
3229 
3230 	list_add(&modstr->next, &module_strings);
3231 }
3232 
update_event_fields(struct trace_event_call * call,struct trace_eval_map * map)3233 static void update_event_fields(struct trace_event_call *call,
3234 				struct trace_eval_map *map)
3235 {
3236 	struct ftrace_event_field *field;
3237 	struct list_head *head;
3238 	char *ptr;
3239 	char *str;
3240 	int len = strlen(map->eval_string);
3241 
3242 	/* Dynamic events should never have field maps */
3243 	if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
3244 		return;
3245 
3246 	head = trace_get_fields(call);
3247 	list_for_each_entry(field, head, link) {
3248 		ptr = strchr(field->type, '[');
3249 		if (!ptr)
3250 			continue;
3251 		ptr++;
3252 
3253 		if (!isalpha(*ptr) && *ptr != '_')
3254 			continue;
3255 
3256 		if (strncmp(map->eval_string, ptr, len) != 0)
3257 			continue;
3258 
3259 		str = kstrdup(field->type, GFP_KERNEL);
3260 		if (WARN_ON_ONCE(!str))
3261 			return;
3262 		ptr = str + (ptr - field->type);
3263 		ptr = eval_replace(ptr, map, len);
3264 		/* enum/sizeof string smaller than value */
3265 		if (WARN_ON_ONCE(!ptr)) {
3266 			kfree(str);
3267 			continue;
3268 		}
3269 
3270 		/*
3271 		 * If the event is part of a module, then we need to free the string
3272 		 * when the module is removed. Otherwise, it will stay allocated
3273 		 * until a reboot.
3274 		 */
3275 		if (call->module)
3276 			add_str_to_module(call->module, str);
3277 
3278 		field->type = str;
3279 	}
3280 }
3281 
trace_event_eval_update(struct trace_eval_map ** map,int len)3282 void trace_event_eval_update(struct trace_eval_map **map, int len)
3283 {
3284 	struct trace_event_call *call, *p;
3285 	const char *last_system = NULL;
3286 	bool first = false;
3287 	int last_i;
3288 	int i;
3289 
3290 	down_write(&trace_event_sem);
3291 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
3292 		/* events are usually grouped together with systems */
3293 		if (!last_system || call->class->system != last_system) {
3294 			first = true;
3295 			last_i = 0;
3296 			last_system = call->class->system;
3297 		}
3298 
3299 		/*
3300 		 * Since calls are grouped by systems, the likelihood that the
3301 		 * next call in the iteration belongs to the same system as the
3302 		 * previous call is high. As an optimization, we skip searching
3303 		 * for a map[] that matches the call's system if the last call
3304 		 * was from the same system. That's what last_i is for. If the
3305 		 * call has the same system as the previous call, then last_i
3306 		 * will be the index of the first map[] that has a matching
3307 		 * system.
3308 		 */
3309 		for (i = last_i; i < len; i++) {
3310 			if (call->class->system == map[i]->system) {
3311 				/* Save the first system if need be */
3312 				if (first) {
3313 					last_i = i;
3314 					first = false;
3315 				}
3316 				update_event_printk(call, map[i]);
3317 				update_event_fields(call, map[i]);
3318 			}
3319 		}
3320 		cond_resched();
3321 	}
3322 	up_write(&trace_event_sem);
3323 }
3324 
event_in_systems(struct trace_event_call * call,const char * systems)3325 static bool event_in_systems(struct trace_event_call *call,
3326 			     const char *systems)
3327 {
3328 	const char *system;
3329 	const char *p;
3330 
3331 	if (!systems)
3332 		return true;
3333 
3334 	system = call->class->system;
3335 	p = strstr(systems, system);
3336 	if (!p)
3337 		return false;
3338 
3339 	if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
3340 		return false;
3341 
3342 	p += strlen(system);
3343 	return !*p || isspace(*p) || *p == ',';
3344 }
3345 
3346 #ifdef CONFIG_HIST_TRIGGERS
3347 /*
3348  * Wake up waiter on the hist_poll_wq from irq_work because the hist trigger
3349  * may happen in any context.
3350  */
hist_poll_event_irq_work(struct irq_work * work)3351 static void hist_poll_event_irq_work(struct irq_work *work)
3352 {
3353 	wake_up_all(&hist_poll_wq);
3354 }
3355 
3356 DEFINE_IRQ_WORK(hist_poll_work, hist_poll_event_irq_work);
3357 DECLARE_WAIT_QUEUE_HEAD(hist_poll_wq);
3358 #endif
3359 
3360 static struct trace_event_file *
trace_create_new_event(struct trace_event_call * call,struct trace_array * tr)3361 trace_create_new_event(struct trace_event_call *call,
3362 		       struct trace_array *tr)
3363 {
3364 	struct trace_pid_list *no_pid_list;
3365 	struct trace_pid_list *pid_list;
3366 	struct trace_event_file *file;
3367 	unsigned int first;
3368 
3369 	if (!event_in_systems(call, tr->system_names))
3370 		return NULL;
3371 
3372 	file = kmem_cache_alloc(file_cachep, GFP_TRACE);
3373 	if (!file)
3374 		return ERR_PTR(-ENOMEM);
3375 
3376 	pid_list = rcu_dereference_protected(tr->filtered_pids,
3377 					     lockdep_is_held(&event_mutex));
3378 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
3379 					     lockdep_is_held(&event_mutex));
3380 
3381 	if (!trace_pid_list_first(pid_list, &first) ||
3382 	    !trace_pid_list_first(no_pid_list, &first))
3383 		file->flags |= EVENT_FILE_FL_PID_FILTER;
3384 
3385 	file->event_call = call;
3386 	file->tr = tr;
3387 	atomic_set(&file->sm_ref, 0);
3388 	atomic_set(&file->tm_ref, 0);
3389 	INIT_LIST_HEAD(&file->triggers);
3390 	list_add(&file->list, &tr->events);
3391 	refcount_set(&file->ref, 1);
3392 
3393 	return file;
3394 }
3395 
3396 #define MAX_BOOT_TRIGGERS 32
3397 
3398 static struct boot_triggers {
3399 	const char		*event;
3400 	char			*trigger;
3401 } bootup_triggers[MAX_BOOT_TRIGGERS];
3402 
3403 static char bootup_trigger_buf[COMMAND_LINE_SIZE];
3404 static int nr_boot_triggers;
3405 
setup_trace_triggers(char * str)3406 static __init int setup_trace_triggers(char *str)
3407 {
3408 	char *trigger;
3409 	char *buf;
3410 	int i;
3411 
3412 	strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
3413 	trace_set_ring_buffer_expanded(NULL);
3414 	disable_tracing_selftest("running event triggers");
3415 
3416 	buf = bootup_trigger_buf;
3417 	for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
3418 		trigger = strsep(&buf, ",");
3419 		if (!trigger)
3420 			break;
3421 		bootup_triggers[i].event = strsep(&trigger, ".");
3422 		bootup_triggers[i].trigger = trigger;
3423 		if (!bootup_triggers[i].trigger)
3424 			break;
3425 	}
3426 
3427 	nr_boot_triggers = i;
3428 	return 1;
3429 }
3430 __setup("trace_trigger=", setup_trace_triggers);
3431 
3432 /* Add an event to a trace directory */
3433 static int
__trace_add_new_event(struct trace_event_call * call,struct trace_array * tr)3434 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
3435 {
3436 	struct trace_event_file *file;
3437 
3438 	file = trace_create_new_event(call, tr);
3439 	/*
3440 	 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3441 	 * allocation, or NULL if the event is not part of the tr->system_names.
3442 	 * When the event is not part of the tr->system_names, return zero, not
3443 	 * an error.
3444 	 */
3445 	if (!file)
3446 		return 0;
3447 
3448 	if (IS_ERR(file))
3449 		return PTR_ERR(file);
3450 
3451 	if (eventdir_initialized)
3452 		return event_create_dir(tr->event_dir, file);
3453 	else
3454 		return event_define_fields(call);
3455 }
3456 
trace_early_triggers(struct trace_event_file * file,const char * name)3457 static void trace_early_triggers(struct trace_event_file *file, const char *name)
3458 {
3459 	int ret;
3460 	int i;
3461 
3462 	for (i = 0; i < nr_boot_triggers; i++) {
3463 		if (strcmp(name, bootup_triggers[i].event))
3464 			continue;
3465 		mutex_lock(&event_mutex);
3466 		ret = trigger_process_regex(file, bootup_triggers[i].trigger);
3467 		mutex_unlock(&event_mutex);
3468 		if (ret)
3469 			pr_err("Failed to register trigger '%s' on event %s\n",
3470 			       bootup_triggers[i].trigger,
3471 			       bootup_triggers[i].event);
3472 	}
3473 }
3474 
3475 /*
3476  * Just create a descriptor for early init. A descriptor is required
3477  * for enabling events at boot. We want to enable events before
3478  * the filesystem is initialized.
3479  */
3480 static int
__trace_early_add_new_event(struct trace_event_call * call,struct trace_array * tr)3481 __trace_early_add_new_event(struct trace_event_call *call,
3482 			    struct trace_array *tr)
3483 {
3484 	struct trace_event_file *file;
3485 	int ret;
3486 
3487 	file = trace_create_new_event(call, tr);
3488 	/*
3489 	 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3490 	 * allocation, or NULL if the event is not part of the tr->system_names.
3491 	 * When the event is not part of the tr->system_names, return zero, not
3492 	 * an error.
3493 	 */
3494 	if (!file)
3495 		return 0;
3496 
3497 	if (IS_ERR(file))
3498 		return PTR_ERR(file);
3499 
3500 	ret = event_define_fields(call);
3501 	if (ret)
3502 		return ret;
3503 
3504 	trace_early_triggers(file, trace_event_name(call));
3505 
3506 	return 0;
3507 }
3508 
3509 struct ftrace_module_file_ops;
3510 static void __add_event_to_tracers(struct trace_event_call *call);
3511 
3512 /* Add an additional event_call dynamically */
trace_add_event_call(struct trace_event_call * call)3513 int trace_add_event_call(struct trace_event_call *call)
3514 {
3515 	int ret;
3516 	lockdep_assert_held(&event_mutex);
3517 
3518 	guard(mutex)(&trace_types_lock);
3519 
3520 	ret = __register_event(call, NULL);
3521 	if (ret < 0)
3522 		return ret;
3523 
3524 	__add_event_to_tracers(call);
3525 	return ret;
3526 }
3527 EXPORT_SYMBOL_GPL(trace_add_event_call);
3528 
3529 /*
3530  * Must be called under locking of trace_types_lock, event_mutex and
3531  * trace_event_sem.
3532  */
__trace_remove_event_call(struct trace_event_call * call)3533 static void __trace_remove_event_call(struct trace_event_call *call)
3534 {
3535 	event_remove(call);
3536 	trace_destroy_fields(call);
3537 }
3538 
probe_remove_event_call(struct trace_event_call * call)3539 static int probe_remove_event_call(struct trace_event_call *call)
3540 {
3541 	struct trace_array *tr;
3542 	struct trace_event_file *file;
3543 
3544 #ifdef CONFIG_PERF_EVENTS
3545 	if (call->perf_refcount)
3546 		return -EBUSY;
3547 #endif
3548 	do_for_each_event_file(tr, file) {
3549 		if (file->event_call != call)
3550 			continue;
3551 		/*
3552 		 * We can't rely on ftrace_event_enable_disable(enable => 0)
3553 		 * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
3554 		 * TRACE_REG_UNREGISTER.
3555 		 */
3556 		if (file->flags & EVENT_FILE_FL_ENABLED)
3557 			goto busy;
3558 
3559 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3560 			tr->clear_trace = true;
3561 		/*
3562 		 * The do_for_each_event_file_safe() is
3563 		 * a double loop. After finding the call for this
3564 		 * trace_array, we use break to jump to the next
3565 		 * trace_array.
3566 		 */
3567 		break;
3568 	} while_for_each_event_file();
3569 
3570 	__trace_remove_event_call(call);
3571 
3572 	return 0;
3573  busy:
3574 	/* No need to clear the trace now */
3575 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
3576 		tr->clear_trace = false;
3577 	}
3578 	return -EBUSY;
3579 }
3580 
3581 /* Remove an event_call */
trace_remove_event_call(struct trace_event_call * call)3582 int trace_remove_event_call(struct trace_event_call *call)
3583 {
3584 	int ret;
3585 
3586 	lockdep_assert_held(&event_mutex);
3587 
3588 	mutex_lock(&trace_types_lock);
3589 	down_write(&trace_event_sem);
3590 	ret = probe_remove_event_call(call);
3591 	up_write(&trace_event_sem);
3592 	mutex_unlock(&trace_types_lock);
3593 
3594 	return ret;
3595 }
3596 EXPORT_SYMBOL_GPL(trace_remove_event_call);
3597 
3598 #define for_each_event(event, start, end)			\
3599 	for (event = start;					\
3600 	     (unsigned long)event < (unsigned long)end;		\
3601 	     event++)
3602 
3603 #ifdef CONFIG_MODULES
update_mod_cache(struct trace_array * tr,struct module * mod)3604 static void update_mod_cache(struct trace_array *tr, struct module *mod)
3605 {
3606 	struct event_mod_load *event_mod, *n;
3607 
3608 	list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
3609 		if (strcmp(event_mod->module, mod->name) != 0)
3610 			continue;
3611 
3612 		__ftrace_set_clr_event_nolock(tr, event_mod->match,
3613 					      event_mod->system,
3614 					      event_mod->event, 1, mod->name);
3615 		free_event_mod(event_mod);
3616 	}
3617 }
3618 
update_cache_events(struct module * mod)3619 static void update_cache_events(struct module *mod)
3620 {
3621 	struct trace_array *tr;
3622 
3623 	list_for_each_entry(tr, &ftrace_trace_arrays, list)
3624 		update_mod_cache(tr, mod);
3625 }
3626 
trace_module_add_events(struct module * mod)3627 static void trace_module_add_events(struct module *mod)
3628 {
3629 	struct trace_event_call **call, **start, **end;
3630 
3631 	if (!mod->num_trace_events)
3632 		return;
3633 
3634 	/* Don't add infrastructure for mods without tracepoints */
3635 	if (trace_module_has_bad_taint(mod)) {
3636 		pr_err("%s: module has bad taint, not creating trace events\n",
3637 		       mod->name);
3638 		return;
3639 	}
3640 
3641 	start = mod->trace_events;
3642 	end = mod->trace_events + mod->num_trace_events;
3643 
3644 	for_each_event(call, start, end) {
3645 		__register_event(*call, mod);
3646 		__add_event_to_tracers(*call);
3647 	}
3648 
3649 	update_cache_events(mod);
3650 }
3651 
trace_module_remove_events(struct module * mod)3652 static void trace_module_remove_events(struct module *mod)
3653 {
3654 	struct trace_event_call *call, *p;
3655 	struct module_string *modstr, *m;
3656 
3657 	down_write(&trace_event_sem);
3658 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
3659 		if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3660 			continue;
3661 		if (call->module == mod)
3662 			__trace_remove_event_call(call);
3663 	}
3664 	/* Check for any strings allocade for this module */
3665 	list_for_each_entry_safe(modstr, m, &module_strings, next) {
3666 		if (modstr->module != mod)
3667 			continue;
3668 		list_del(&modstr->next);
3669 		kfree(modstr->str);
3670 		kfree(modstr);
3671 	}
3672 	up_write(&trace_event_sem);
3673 
3674 	/*
3675 	 * It is safest to reset the ring buffer if the module being unloaded
3676 	 * registered any events that were used. The only worry is if
3677 	 * a new module gets loaded, and takes on the same id as the events
3678 	 * of this module. When printing out the buffer, traced events left
3679 	 * over from this module may be passed to the new module events and
3680 	 * unexpected results may occur.
3681 	 */
3682 	tracing_reset_all_online_cpus_unlocked();
3683 }
3684 
trace_module_notify(struct notifier_block * self,unsigned long val,void * data)3685 static int trace_module_notify(struct notifier_block *self,
3686 			       unsigned long val, void *data)
3687 {
3688 	struct module *mod = data;
3689 
3690 	mutex_lock(&event_mutex);
3691 	mutex_lock(&trace_types_lock);
3692 	switch (val) {
3693 	case MODULE_STATE_COMING:
3694 		trace_module_add_events(mod);
3695 		break;
3696 	case MODULE_STATE_GOING:
3697 		trace_module_remove_events(mod);
3698 		break;
3699 	}
3700 	mutex_unlock(&trace_types_lock);
3701 	mutex_unlock(&event_mutex);
3702 
3703 	return NOTIFY_OK;
3704 }
3705 
3706 static struct notifier_block trace_module_nb = {
3707 	.notifier_call = trace_module_notify,
3708 	.priority = 1, /* higher than trace.c module notify */
3709 };
3710 #endif /* CONFIG_MODULES */
3711 
3712 /* Create a new event directory structure for a trace directory. */
3713 static void
__trace_add_event_dirs(struct trace_array * tr)3714 __trace_add_event_dirs(struct trace_array *tr)
3715 {
3716 	struct trace_event_call *call;
3717 	int ret;
3718 
3719 	list_for_each_entry(call, &ftrace_events, list) {
3720 		ret = __trace_add_new_event(call, tr);
3721 		if (ret < 0)
3722 			pr_warn("Could not create directory for event %s\n",
3723 				trace_event_name(call));
3724 	}
3725 }
3726 
3727 /* Returns any file that matches the system and event */
3728 struct trace_event_file *
__find_event_file(struct trace_array * tr,const char * system,const char * event)3729 __find_event_file(struct trace_array *tr, const char *system, const char *event)
3730 {
3731 	struct trace_event_file *file;
3732 	struct trace_event_call *call;
3733 	const char *name;
3734 
3735 	list_for_each_entry(file, &tr->events, list) {
3736 
3737 		call = file->event_call;
3738 		name = trace_event_name(call);
3739 
3740 		if (!name || !call->class)
3741 			continue;
3742 
3743 		if (strcmp(event, name) == 0 &&
3744 		    strcmp(system, call->class->system) == 0)
3745 			return file;
3746 	}
3747 	return NULL;
3748 }
3749 
3750 /* Returns valid trace event files that match system and event */
3751 struct trace_event_file *
find_event_file(struct trace_array * tr,const char * system,const char * event)3752 find_event_file(struct trace_array *tr, const char *system, const char *event)
3753 {
3754 	struct trace_event_file *file;
3755 
3756 	file = __find_event_file(tr, system, event);
3757 	if (!file || !file->event_call->class->reg ||
3758 	    file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3759 		return NULL;
3760 
3761 	return file;
3762 }
3763 
3764 /**
3765  * trace_get_event_file - Find and return a trace event file
3766  * @instance: The name of the trace instance containing the event
3767  * @system: The name of the system containing the event
3768  * @event: The name of the event
3769  *
3770  * Return a trace event file given the trace instance name, trace
3771  * system, and trace event name.  If the instance name is NULL, it
3772  * refers to the top-level trace array.
3773  *
3774  * This function will look it up and return it if found, after calling
3775  * trace_array_get() to prevent the instance from going away, and
3776  * increment the event's module refcount to prevent it from being
3777  * removed.
3778  *
3779  * To release the file, call trace_put_event_file(), which will call
3780  * trace_array_put() and decrement the event's module refcount.
3781  *
3782  * Return: The trace event on success, ERR_PTR otherwise.
3783  */
trace_get_event_file(const char * instance,const char * system,const char * event)3784 struct trace_event_file *trace_get_event_file(const char *instance,
3785 					      const char *system,
3786 					      const char *event)
3787 {
3788 	struct trace_array *tr = top_trace_array();
3789 	struct trace_event_file *file = NULL;
3790 	int ret = -EINVAL;
3791 
3792 	if (instance) {
3793 		tr = trace_array_find_get(instance);
3794 		if (!tr)
3795 			return ERR_PTR(-ENOENT);
3796 	} else {
3797 		ret = trace_array_get(tr);
3798 		if (ret)
3799 			return ERR_PTR(ret);
3800 	}
3801 
3802 	guard(mutex)(&event_mutex);
3803 
3804 	file = find_event_file(tr, system, event);
3805 	if (!file) {
3806 		trace_array_put(tr);
3807 		return ERR_PTR(-EINVAL);
3808 	}
3809 
3810 	/* Don't let event modules unload while in use */
3811 	ret = trace_event_try_get_ref(file->event_call);
3812 	if (!ret) {
3813 		trace_array_put(tr);
3814 		return ERR_PTR(-EBUSY);
3815 	}
3816 
3817 	return file;
3818 }
3819 EXPORT_SYMBOL_GPL(trace_get_event_file);
3820 
3821 /**
3822  * trace_put_event_file - Release a file from trace_get_event_file()
3823  * @file: The trace event file
3824  *
3825  * If a file was retrieved using trace_get_event_file(), this should
3826  * be called when it's no longer needed.  It will cancel the previous
3827  * trace_array_get() called by that function, and decrement the
3828  * event's module refcount.
3829  */
trace_put_event_file(struct trace_event_file * file)3830 void trace_put_event_file(struct trace_event_file *file)
3831 {
3832 	mutex_lock(&event_mutex);
3833 	trace_event_put_ref(file->event_call);
3834 	mutex_unlock(&event_mutex);
3835 
3836 	trace_array_put(file->tr);
3837 }
3838 EXPORT_SYMBOL_GPL(trace_put_event_file);
3839 
3840 #ifdef CONFIG_DYNAMIC_FTRACE
3841 
3842 /* Avoid typos */
3843 #define ENABLE_EVENT_STR	"enable_event"
3844 #define DISABLE_EVENT_STR	"disable_event"
3845 
3846 struct event_probe_data {
3847 	struct trace_event_file	*file;
3848 	unsigned long			count;
3849 	int				ref;
3850 	bool				enable;
3851 };
3852 
update_event_probe(struct event_probe_data * data)3853 static void update_event_probe(struct event_probe_data *data)
3854 {
3855 	if (data->enable)
3856 		clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3857 	else
3858 		set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3859 }
3860 
3861 static void
event_enable_probe(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)3862 event_enable_probe(unsigned long ip, unsigned long parent_ip,
3863 		   struct trace_array *tr, struct ftrace_probe_ops *ops,
3864 		   void *data)
3865 {
3866 	struct ftrace_func_mapper *mapper = data;
3867 	struct event_probe_data *edata;
3868 	void **pdata;
3869 
3870 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3871 	if (!pdata || !*pdata)
3872 		return;
3873 
3874 	edata = *pdata;
3875 	update_event_probe(edata);
3876 }
3877 
3878 static void
event_enable_count_probe(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)3879 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
3880 			 struct trace_array *tr, struct ftrace_probe_ops *ops,
3881 			 void *data)
3882 {
3883 	struct ftrace_func_mapper *mapper = data;
3884 	struct event_probe_data *edata;
3885 	void **pdata;
3886 
3887 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3888 	if (!pdata || !*pdata)
3889 		return;
3890 
3891 	edata = *pdata;
3892 
3893 	if (!edata->count)
3894 		return;
3895 
3896 	/* Skip if the event is in a state we want to switch to */
3897 	if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
3898 		return;
3899 
3900 	if (edata->count != -1)
3901 		(edata->count)--;
3902 
3903 	update_event_probe(edata);
3904 }
3905 
3906 static int
event_enable_print(struct seq_file * m,unsigned long ip,struct ftrace_probe_ops * ops,void * data)3907 event_enable_print(struct seq_file *m, unsigned long ip,
3908 		   struct ftrace_probe_ops *ops, void *data)
3909 {
3910 	struct ftrace_func_mapper *mapper = data;
3911 	struct event_probe_data *edata;
3912 	void **pdata;
3913 
3914 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3915 
3916 	if (WARN_ON_ONCE(!pdata || !*pdata))
3917 		return 0;
3918 
3919 	edata = *pdata;
3920 
3921 	seq_printf(m, "%ps:", (void *)ip);
3922 
3923 	seq_printf(m, "%s:%s:%s",
3924 		   edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
3925 		   edata->file->event_call->class->system,
3926 		   trace_event_name(edata->file->event_call));
3927 
3928 	if (edata->count == -1)
3929 		seq_puts(m, ":unlimited\n");
3930 	else
3931 		seq_printf(m, ":count=%ld\n", edata->count);
3932 
3933 	return 0;
3934 }
3935 
3936 static int
event_enable_init(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * init_data,void ** data)3937 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
3938 		  unsigned long ip, void *init_data, void **data)
3939 {
3940 	struct ftrace_func_mapper *mapper = *data;
3941 	struct event_probe_data *edata = init_data;
3942 	int ret;
3943 
3944 	if (!mapper) {
3945 		mapper = allocate_ftrace_func_mapper();
3946 		if (!mapper)
3947 			return -ENODEV;
3948 		*data = mapper;
3949 	}
3950 
3951 	ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
3952 	if (ret < 0)
3953 		return ret;
3954 
3955 	edata->ref++;
3956 
3957 	return 0;
3958 }
3959 
free_probe_data(void * data)3960 static int free_probe_data(void *data)
3961 {
3962 	struct event_probe_data *edata = data;
3963 
3964 	edata->ref--;
3965 	if (!edata->ref) {
3966 		/* Remove the SOFT_MODE flag */
3967 		__ftrace_event_enable_disable(edata->file, 0, 1);
3968 		trace_event_put_ref(edata->file->event_call);
3969 		kfree(edata);
3970 	}
3971 	return 0;
3972 }
3973 
3974 static void
event_enable_free(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * data)3975 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
3976 		  unsigned long ip, void *data)
3977 {
3978 	struct ftrace_func_mapper *mapper = data;
3979 	struct event_probe_data *edata;
3980 
3981 	if (!ip) {
3982 		if (!mapper)
3983 			return;
3984 		free_ftrace_func_mapper(mapper, free_probe_data);
3985 		return;
3986 	}
3987 
3988 	edata = ftrace_func_mapper_remove_ip(mapper, ip);
3989 
3990 	if (WARN_ON_ONCE(!edata))
3991 		return;
3992 
3993 	if (WARN_ON_ONCE(edata->ref <= 0))
3994 		return;
3995 
3996 	free_probe_data(edata);
3997 }
3998 
3999 static struct ftrace_probe_ops event_enable_probe_ops = {
4000 	.func			= event_enable_probe,
4001 	.print			= event_enable_print,
4002 	.init			= event_enable_init,
4003 	.free			= event_enable_free,
4004 };
4005 
4006 static struct ftrace_probe_ops event_enable_count_probe_ops = {
4007 	.func			= event_enable_count_probe,
4008 	.print			= event_enable_print,
4009 	.init			= event_enable_init,
4010 	.free			= event_enable_free,
4011 };
4012 
4013 static struct ftrace_probe_ops event_disable_probe_ops = {
4014 	.func			= event_enable_probe,
4015 	.print			= event_enable_print,
4016 	.init			= event_enable_init,
4017 	.free			= event_enable_free,
4018 };
4019 
4020 static struct ftrace_probe_ops event_disable_count_probe_ops = {
4021 	.func			= event_enable_count_probe,
4022 	.print			= event_enable_print,
4023 	.init			= event_enable_init,
4024 	.free			= event_enable_free,
4025 };
4026 
4027 static int
event_enable_func(struct trace_array * tr,struct ftrace_hash * hash,char * glob,char * cmd,char * param,int enabled)4028 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
4029 		  char *glob, char *cmd, char *param, int enabled)
4030 {
4031 	struct trace_event_file *file;
4032 	struct ftrace_probe_ops *ops;
4033 	struct event_probe_data *data;
4034 	unsigned long count = -1;
4035 	const char *system;
4036 	const char *event;
4037 	char *number;
4038 	bool enable;
4039 	int ret;
4040 
4041 	if (!tr)
4042 		return -ENODEV;
4043 
4044 	/* hash funcs only work with set_ftrace_filter */
4045 	if (!enabled || !param)
4046 		return -EINVAL;
4047 
4048 	system = strsep(&param, ":");
4049 	if (!param)
4050 		return -EINVAL;
4051 
4052 	event = strsep(&param, ":");
4053 
4054 	guard(mutex)(&event_mutex);
4055 
4056 	file = find_event_file(tr, system, event);
4057 	if (!file)
4058 		return -EINVAL;
4059 
4060 	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
4061 
4062 	if (enable)
4063 		ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
4064 	else
4065 		ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
4066 
4067 	if (glob[0] == '!')
4068 		return unregister_ftrace_function_probe_func(glob+1, tr, ops);
4069 
4070 	if (param) {
4071 		number = strsep(&param, ":");
4072 
4073 		if (!strlen(number))
4074 			return -EINVAL;
4075 
4076 		/*
4077 		 * We use the callback data field (which is a pointer)
4078 		 * as our counter.
4079 		 */
4080 		ret = kstrtoul(number, 0, &count);
4081 		if (ret)
4082 			return ret;
4083 	}
4084 
4085 	/* Don't let event modules unload while probe registered */
4086 	ret = trace_event_try_get_ref(file->event_call);
4087 	if (!ret)
4088 		return -EBUSY;
4089 
4090 	ret = __ftrace_event_enable_disable(file, 1, 1);
4091 	if (ret < 0)
4092 		goto out_put;
4093 
4094 	ret = -ENOMEM;
4095 	data = kzalloc(sizeof(*data), GFP_KERNEL);
4096 	if (!data)
4097 		goto out_put;
4098 
4099 	data->enable = enable;
4100 	data->count = count;
4101 	data->file = file;
4102 
4103 	ret = register_ftrace_function_probe(glob, tr, ops, data);
4104 	/*
4105 	 * The above returns on success the # of functions enabled,
4106 	 * but if it didn't find any functions it returns zero.
4107 	 * Consider no functions a failure too.
4108 	 */
4109 
4110 	/* Just return zero, not the number of enabled functions */
4111 	if (ret > 0)
4112 		return 0;
4113 
4114 	kfree(data);
4115 
4116 	if (!ret)
4117 		ret = -ENOENT;
4118 
4119 	__ftrace_event_enable_disable(file, 0, 1);
4120  out_put:
4121 	trace_event_put_ref(file->event_call);
4122 	return ret;
4123 }
4124 
4125 static struct ftrace_func_command event_enable_cmd = {
4126 	.name			= ENABLE_EVENT_STR,
4127 	.func			= event_enable_func,
4128 };
4129 
4130 static struct ftrace_func_command event_disable_cmd = {
4131 	.name			= DISABLE_EVENT_STR,
4132 	.func			= event_enable_func,
4133 };
4134 
register_event_cmds(void)4135 static __init int register_event_cmds(void)
4136 {
4137 	int ret;
4138 
4139 	ret = register_ftrace_command(&event_enable_cmd);
4140 	if (WARN_ON(ret < 0))
4141 		return ret;
4142 	ret = register_ftrace_command(&event_disable_cmd);
4143 	if (WARN_ON(ret < 0))
4144 		unregister_ftrace_command(&event_enable_cmd);
4145 	return ret;
4146 }
4147 #else
register_event_cmds(void)4148 static inline int register_event_cmds(void) { return 0; }
4149 #endif /* CONFIG_DYNAMIC_FTRACE */
4150 
4151 /*
4152  * The top level array and trace arrays created by boot-time tracing
4153  * have already had its trace_event_file descriptors created in order
4154  * to allow for early events to be recorded.
4155  * This function is called after the tracefs has been initialized,
4156  * and we now have to create the files associated to the events.
4157  */
__trace_early_add_event_dirs(struct trace_array * tr)4158 static void __trace_early_add_event_dirs(struct trace_array *tr)
4159 {
4160 	struct trace_event_file *file;
4161 	int ret;
4162 
4163 
4164 	list_for_each_entry(file, &tr->events, list) {
4165 		ret = event_create_dir(tr->event_dir, file);
4166 		if (ret < 0)
4167 			pr_warn("Could not create directory for event %s\n",
4168 				trace_event_name(file->event_call));
4169 	}
4170 }
4171 
4172 /*
4173  * For early boot up, the top trace array and the trace arrays created
4174  * by boot-time tracing require to have a list of events that can be
4175  * enabled. This must be done before the filesystem is set up in order
4176  * to allow events to be traced early.
4177  */
__trace_early_add_events(struct trace_array * tr)4178 void __trace_early_add_events(struct trace_array *tr)
4179 {
4180 	struct trace_event_call *call;
4181 	int ret;
4182 
4183 	list_for_each_entry(call, &ftrace_events, list) {
4184 		/* Early boot up should not have any modules loaded */
4185 		if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
4186 		    WARN_ON_ONCE(call->module))
4187 			continue;
4188 
4189 		ret = __trace_early_add_new_event(call, tr);
4190 		if (ret < 0)
4191 			pr_warn("Could not create early event %s\n",
4192 				trace_event_name(call));
4193 	}
4194 }
4195 
4196 /* Remove the event directory structure for a trace directory. */
4197 static void
__trace_remove_event_dirs(struct trace_array * tr)4198 __trace_remove_event_dirs(struct trace_array *tr)
4199 {
4200 	struct trace_event_file *file, *next;
4201 
4202 	list_for_each_entry_safe(file, next, &tr->events, list)
4203 		remove_event_file_dir(file);
4204 }
4205 
__add_event_to_tracers(struct trace_event_call * call)4206 static void __add_event_to_tracers(struct trace_event_call *call)
4207 {
4208 	struct trace_array *tr;
4209 
4210 	list_for_each_entry(tr, &ftrace_trace_arrays, list)
4211 		__trace_add_new_event(call, tr);
4212 }
4213 
4214 extern struct trace_event_call *__start_ftrace_events[];
4215 extern struct trace_event_call *__stop_ftrace_events[];
4216 
4217 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
4218 
setup_trace_event(char * str)4219 static __init int setup_trace_event(char *str)
4220 {
4221 	strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
4222 	trace_set_ring_buffer_expanded(NULL);
4223 	disable_tracing_selftest("running event tracing");
4224 
4225 	return 1;
4226 }
4227 __setup("trace_event=", setup_trace_event);
4228 
events_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)4229 static int events_callback(const char *name, umode_t *mode, void **data,
4230 			   const struct file_operations **fops)
4231 {
4232 	if (strcmp(name, "enable") == 0) {
4233 		*mode = TRACE_MODE_WRITE;
4234 		*fops = &ftrace_tr_enable_fops;
4235 		return 1;
4236 	}
4237 
4238 	if (strcmp(name, "header_page") == 0) {
4239 		*mode = TRACE_MODE_READ;
4240 		*fops = &ftrace_show_header_page_fops;
4241 
4242 	} else if (strcmp(name, "header_event") == 0) {
4243 		*mode = TRACE_MODE_READ;
4244 		*fops = &ftrace_show_header_event_fops;
4245 	} else
4246 		return 0;
4247 
4248 	return 1;
4249 }
4250 
4251 /* Expects to have event_mutex held when called */
4252 static int
create_event_toplevel_files(struct dentry * parent,struct trace_array * tr)4253 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
4254 {
4255 	struct eventfs_inode *e_events;
4256 	struct dentry *entry;
4257 	int nr_entries;
4258 	static struct eventfs_entry events_entries[] = {
4259 		{
4260 			.name		= "enable",
4261 			.callback	= events_callback,
4262 		},
4263 		{
4264 			.name		= "header_page",
4265 			.callback	= events_callback,
4266 		},
4267 		{
4268 			.name		= "header_event",
4269 			.callback	= events_callback,
4270 		},
4271 	};
4272 
4273 	entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
4274 				  tr, &ftrace_set_event_fops);
4275 	if (!entry)
4276 		return -ENOMEM;
4277 
4278 	nr_entries = ARRAY_SIZE(events_entries);
4279 
4280 	e_events = eventfs_create_events_dir("events", parent, events_entries,
4281 					     nr_entries, tr);
4282 	if (IS_ERR(e_events)) {
4283 		pr_warn("Could not create tracefs 'events' directory\n");
4284 		return -ENOMEM;
4285 	}
4286 
4287 	/* There are not as crucial, just warn if they are not created */
4288 
4289 	trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
4290 			  tr, &ftrace_set_event_pid_fops);
4291 
4292 	trace_create_file("set_event_notrace_pid",
4293 			  TRACE_MODE_WRITE, parent, tr,
4294 			  &ftrace_set_event_notrace_pid_fops);
4295 
4296 	tr->event_dir = e_events;
4297 
4298 	return 0;
4299 }
4300 
4301 /**
4302  * event_trace_add_tracer - add a instance of a trace_array to events
4303  * @parent: The parent dentry to place the files/directories for events in
4304  * @tr: The trace array associated with these events
4305  *
4306  * When a new instance is created, it needs to set up its events
4307  * directory, as well as other files associated with events. It also
4308  * creates the event hierarchy in the @parent/events directory.
4309  *
4310  * Returns 0 on success.
4311  *
4312  * Must be called with event_mutex held.
4313  */
event_trace_add_tracer(struct dentry * parent,struct trace_array * tr)4314 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
4315 {
4316 	int ret;
4317 
4318 	lockdep_assert_held(&event_mutex);
4319 
4320 	ret = create_event_toplevel_files(parent, tr);
4321 	if (ret)
4322 		goto out;
4323 
4324 	down_write(&trace_event_sem);
4325 	/* If tr already has the event list, it is initialized in early boot. */
4326 	if (unlikely(!list_empty(&tr->events)))
4327 		__trace_early_add_event_dirs(tr);
4328 	else
4329 		__trace_add_event_dirs(tr);
4330 	up_write(&trace_event_sem);
4331 
4332  out:
4333 	return ret;
4334 }
4335 
4336 /*
4337  * The top trace array already had its file descriptors created.
4338  * Now the files themselves need to be created.
4339  */
4340 static __init int
early_event_add_tracer(struct dentry * parent,struct trace_array * tr)4341 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
4342 {
4343 	int ret;
4344 
4345 	guard(mutex)(&event_mutex);
4346 
4347 	ret = create_event_toplevel_files(parent, tr);
4348 	if (ret)
4349 		return ret;
4350 
4351 	down_write(&trace_event_sem);
4352 	__trace_early_add_event_dirs(tr);
4353 	up_write(&trace_event_sem);
4354 
4355 	return 0;
4356 }
4357 
4358 /* Must be called with event_mutex held */
event_trace_del_tracer(struct trace_array * tr)4359 int event_trace_del_tracer(struct trace_array *tr)
4360 {
4361 	lockdep_assert_held(&event_mutex);
4362 
4363 	/* Disable any event triggers and associated soft-disabled events */
4364 	clear_event_triggers(tr);
4365 
4366 	/* Clear the pid list */
4367 	__ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
4368 
4369 	/* Disable any running events */
4370 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0, NULL);
4371 
4372 	/* Make sure no more events are being executed */
4373 	tracepoint_synchronize_unregister();
4374 
4375 	down_write(&trace_event_sem);
4376 	__trace_remove_event_dirs(tr);
4377 	eventfs_remove_events_dir(tr->event_dir);
4378 	up_write(&trace_event_sem);
4379 
4380 	tr->event_dir = NULL;
4381 
4382 	return 0;
4383 }
4384 
event_trace_memsetup(void)4385 static __init int event_trace_memsetup(void)
4386 {
4387 	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
4388 	file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
4389 	return 0;
4390 }
4391 
4392 __init void
early_enable_events(struct trace_array * tr,char * buf,bool disable_first)4393 early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
4394 {
4395 	char *token;
4396 	int ret;
4397 
4398 	while (true) {
4399 		token = strsep(&buf, ",");
4400 
4401 		if (!token)
4402 			break;
4403 
4404 		if (*token) {
4405 			/* Restarting syscalls requires that we stop them first */
4406 			if (disable_first)
4407 				ftrace_set_clr_event(tr, token, 0);
4408 
4409 			ret = ftrace_set_clr_event(tr, token, 1);
4410 			if (ret)
4411 				pr_warn("Failed to enable trace event: %s\n", token);
4412 		}
4413 
4414 		/* Put back the comma to allow this to be called again */
4415 		if (buf)
4416 			*(buf - 1) = ',';
4417 	}
4418 }
4419 
event_trace_enable(void)4420 static __init int event_trace_enable(void)
4421 {
4422 	struct trace_array *tr = top_trace_array();
4423 	struct trace_event_call **iter, *call;
4424 	int ret;
4425 
4426 	if (!tr)
4427 		return -ENODEV;
4428 
4429 	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
4430 
4431 		call = *iter;
4432 		ret = event_init(call);
4433 		if (!ret)
4434 			list_add(&call->list, &ftrace_events);
4435 	}
4436 
4437 	register_trigger_cmds();
4438 
4439 	/*
4440 	 * We need the top trace array to have a working set of trace
4441 	 * points at early init, before the debug files and directories
4442 	 * are created. Create the file entries now, and attach them
4443 	 * to the actual file dentries later.
4444 	 */
4445 	__trace_early_add_events(tr);
4446 
4447 	early_enable_events(tr, bootup_event_buf, false);
4448 
4449 	trace_printk_start_comm();
4450 
4451 	register_event_cmds();
4452 
4453 
4454 	return 0;
4455 }
4456 
4457 /*
4458  * event_trace_enable() is called from trace_event_init() first to
4459  * initialize events and perhaps start any events that are on the
4460  * command line. Unfortunately, there are some events that will not
4461  * start this early, like the system call tracepoints that need
4462  * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
4463  * event_trace_enable() is called before pid 1 starts, and this flag
4464  * is never set, making the syscall tracepoint never get reached, but
4465  * the event is enabled regardless (and not doing anything).
4466  */
event_trace_enable_again(void)4467 static __init int event_trace_enable_again(void)
4468 {
4469 	struct trace_array *tr;
4470 
4471 	tr = top_trace_array();
4472 	if (!tr)
4473 		return -ENODEV;
4474 
4475 	early_enable_events(tr, bootup_event_buf, true);
4476 
4477 	return 0;
4478 }
4479 
4480 early_initcall(event_trace_enable_again);
4481 
4482 /* Init fields which doesn't related to the tracefs */
event_trace_init_fields(void)4483 static __init int event_trace_init_fields(void)
4484 {
4485 	if (trace_define_generic_fields())
4486 		pr_warn("tracing: Failed to allocated generic fields");
4487 
4488 	if (trace_define_common_fields())
4489 		pr_warn("tracing: Failed to allocate common fields");
4490 
4491 	return 0;
4492 }
4493 
event_trace_init(void)4494 __init int event_trace_init(void)
4495 {
4496 	struct trace_array *tr;
4497 	int ret;
4498 
4499 	tr = top_trace_array();
4500 	if (!tr)
4501 		return -ENODEV;
4502 
4503 	trace_create_file("available_events", TRACE_MODE_READ,
4504 			  NULL, tr, &ftrace_avail_fops);
4505 
4506 	ret = early_event_add_tracer(NULL, tr);
4507 	if (ret)
4508 		return ret;
4509 
4510 #ifdef CONFIG_MODULES
4511 	ret = register_module_notifier(&trace_module_nb);
4512 	if (ret)
4513 		pr_warn("Failed to register trace events module notifier\n");
4514 #endif
4515 
4516 	eventdir_initialized = true;
4517 
4518 	return 0;
4519 }
4520 
trace_event_init(void)4521 void __init trace_event_init(void)
4522 {
4523 	event_trace_memsetup();
4524 	init_ftrace_syscalls();
4525 	event_trace_enable();
4526 	event_trace_init_fields();
4527 }
4528 
4529 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4530 
4531 static DEFINE_SPINLOCK(test_spinlock);
4532 static DEFINE_SPINLOCK(test_spinlock_irq);
4533 static DEFINE_MUTEX(test_mutex);
4534 
test_work(struct work_struct * dummy)4535 static __init void test_work(struct work_struct *dummy)
4536 {
4537 	spin_lock(&test_spinlock);
4538 	spin_lock_irq(&test_spinlock_irq);
4539 	udelay(1);
4540 	spin_unlock_irq(&test_spinlock_irq);
4541 	spin_unlock(&test_spinlock);
4542 
4543 	mutex_lock(&test_mutex);
4544 	msleep(1);
4545 	mutex_unlock(&test_mutex);
4546 }
4547 
event_test_thread(void * unused)4548 static __init int event_test_thread(void *unused)
4549 {
4550 	void *test_malloc;
4551 
4552 	test_malloc = kmalloc(1234, GFP_KERNEL);
4553 	if (!test_malloc)
4554 		pr_info("failed to kmalloc\n");
4555 
4556 	schedule_on_each_cpu(test_work);
4557 
4558 	kfree(test_malloc);
4559 
4560 	set_current_state(TASK_INTERRUPTIBLE);
4561 	while (!kthread_should_stop()) {
4562 		schedule();
4563 		set_current_state(TASK_INTERRUPTIBLE);
4564 	}
4565 	__set_current_state(TASK_RUNNING);
4566 
4567 	return 0;
4568 }
4569 
4570 /*
4571  * Do various things that may trigger events.
4572  */
event_test_stuff(void)4573 static __init void event_test_stuff(void)
4574 {
4575 	struct task_struct *test_thread;
4576 
4577 	test_thread = kthread_run(event_test_thread, NULL, "test-events");
4578 	msleep(1);
4579 	kthread_stop(test_thread);
4580 }
4581 
4582 /*
4583  * For every trace event defined, we will test each trace point separately,
4584  * and then by groups, and finally all trace points.
4585  */
event_trace_self_tests(void)4586 static __init void event_trace_self_tests(void)
4587 {
4588 	struct trace_subsystem_dir *dir;
4589 	struct trace_event_file *file;
4590 	struct trace_event_call *call;
4591 	struct event_subsystem *system;
4592 	struct trace_array *tr;
4593 	int ret;
4594 
4595 	tr = top_trace_array();
4596 	if (!tr)
4597 		return;
4598 
4599 	pr_info("Running tests on trace events:\n");
4600 
4601 	list_for_each_entry(file, &tr->events, list) {
4602 
4603 		call = file->event_call;
4604 
4605 		/* Only test those that have a probe */
4606 		if (!call->class || !call->class->probe)
4607 			continue;
4608 
4609 /*
4610  * Testing syscall events here is pretty useless, but
4611  * we still do it if configured. But this is time consuming.
4612  * What we really need is a user thread to perform the
4613  * syscalls as we test.
4614  */
4615 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
4616 		if (call->class->system &&
4617 		    strcmp(call->class->system, "syscalls") == 0)
4618 			continue;
4619 #endif
4620 
4621 		pr_info("Testing event %s: ", trace_event_name(call));
4622 
4623 		/*
4624 		 * If an event is already enabled, someone is using
4625 		 * it and the self test should not be on.
4626 		 */
4627 		if (file->flags & EVENT_FILE_FL_ENABLED) {
4628 			pr_warn("Enabled event during self test!\n");
4629 			WARN_ON_ONCE(1);
4630 			continue;
4631 		}
4632 
4633 		ftrace_event_enable_disable(file, 1);
4634 		event_test_stuff();
4635 		ftrace_event_enable_disable(file, 0);
4636 
4637 		pr_cont("OK\n");
4638 	}
4639 
4640 	/* Now test at the sub system level */
4641 
4642 	pr_info("Running tests on trace event systems:\n");
4643 
4644 	list_for_each_entry(dir, &tr->systems, list) {
4645 
4646 		system = dir->subsystem;
4647 
4648 		/* the ftrace system is special, skip it */
4649 		if (strcmp(system->name, "ftrace") == 0)
4650 			continue;
4651 
4652 		pr_info("Testing event system %s: ", system->name);
4653 
4654 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1, NULL);
4655 		if (WARN_ON_ONCE(ret)) {
4656 			pr_warn("error enabling system %s\n",
4657 				system->name);
4658 			continue;
4659 		}
4660 
4661 		event_test_stuff();
4662 
4663 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0, NULL);
4664 		if (WARN_ON_ONCE(ret)) {
4665 			pr_warn("error disabling system %s\n",
4666 				system->name);
4667 			continue;
4668 		}
4669 
4670 		pr_cont("OK\n");
4671 	}
4672 
4673 	/* Test with all events enabled */
4674 
4675 	pr_info("Running tests on all trace events:\n");
4676 	pr_info("Testing all events: ");
4677 
4678 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1, NULL);
4679 	if (WARN_ON_ONCE(ret)) {
4680 		pr_warn("error enabling all events\n");
4681 		return;
4682 	}
4683 
4684 	event_test_stuff();
4685 
4686 	/* reset sysname */
4687 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0, NULL);
4688 	if (WARN_ON_ONCE(ret)) {
4689 		pr_warn("error disabling all events\n");
4690 		return;
4691 	}
4692 
4693 	pr_cont("OK\n");
4694 }
4695 
4696 #ifdef CONFIG_FUNCTION_TRACER
4697 
4698 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4699 
4700 static struct trace_event_file event_trace_file __initdata;
4701 
4702 static void __init
function_test_events_call(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * regs)4703 function_test_events_call(unsigned long ip, unsigned long parent_ip,
4704 			  struct ftrace_ops *op, struct ftrace_regs *regs)
4705 {
4706 	struct trace_buffer *buffer;
4707 	struct ring_buffer_event *event;
4708 	struct ftrace_entry *entry;
4709 	unsigned int trace_ctx;
4710 	long disabled;
4711 	int cpu;
4712 
4713 	trace_ctx = tracing_gen_ctx();
4714 	preempt_disable_notrace();
4715 	cpu = raw_smp_processor_id();
4716 	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4717 
4718 	if (disabled != 1)
4719 		goto out;
4720 
4721 	event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4722 						TRACE_FN, sizeof(*entry),
4723 						trace_ctx);
4724 	if (!event)
4725 		goto out;
4726 	entry	= ring_buffer_event_data(event);
4727 	entry->ip			= ip;
4728 	entry->parent_ip		= parent_ip;
4729 
4730 	event_trigger_unlock_commit(&event_trace_file, buffer, event,
4731 				    entry, trace_ctx);
4732  out:
4733 	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4734 	preempt_enable_notrace();
4735 }
4736 
4737 static struct ftrace_ops trace_ops __initdata  =
4738 {
4739 	.func = function_test_events_call,
4740 };
4741 
event_trace_self_test_with_function(void)4742 static __init void event_trace_self_test_with_function(void)
4743 {
4744 	int ret;
4745 
4746 	event_trace_file.tr = top_trace_array();
4747 	if (WARN_ON(!event_trace_file.tr))
4748 		return;
4749 
4750 	ret = register_ftrace_function(&trace_ops);
4751 	if (WARN_ON(ret < 0)) {
4752 		pr_info("Failed to enable function tracer for event tests\n");
4753 		return;
4754 	}
4755 	pr_info("Running tests again, along with the function tracer\n");
4756 	event_trace_self_tests();
4757 	unregister_ftrace_function(&trace_ops);
4758 }
4759 #else
event_trace_self_test_with_function(void)4760 static __init void event_trace_self_test_with_function(void)
4761 {
4762 }
4763 #endif
4764 
event_trace_self_tests_init(void)4765 static __init int event_trace_self_tests_init(void)
4766 {
4767 	if (!tracing_selftest_disabled) {
4768 		event_trace_self_tests();
4769 		event_trace_self_test_with_function();
4770 	}
4771 
4772 	return 0;
4773 }
4774 
4775 late_initcall(event_trace_self_tests_init);
4776 
4777 #endif
4778