xref: /linux/kernel/trace/trace_events.c (revision 9466b6ae6befa87b27cc13ee7263e01b9c48efc0)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * event tracer
4  *
5  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
6  *
7  *  - Added format output of fields of the trace point.
8  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
9  *
10  */
11 
12 #define pr_fmt(fmt) fmt
13 
14 #include <linux/workqueue.h>
15 #include <linux/security.h>
16 #include <linux/spinlock.h>
17 #include <linux/kthread.h>
18 #include <linux/tracefs.h>
19 #include <linux/uaccess.h>
20 #include <linux/module.h>
21 #include <linux/ctype.h>
22 #include <linux/sort.h>
23 #include <linux/slab.h>
24 #include <linux/delay.h>
25 
26 #include <trace/events/sched.h>
27 #include <trace/syscall.h>
28 
29 #include <asm/setup.h>
30 
31 #include "trace_output.h"
32 
33 #undef TRACE_SYSTEM
34 #define TRACE_SYSTEM "TRACE_SYSTEM"
35 
36 DEFINE_MUTEX(event_mutex);
37 
38 LIST_HEAD(ftrace_events);
39 static LIST_HEAD(ftrace_generic_fields);
40 static LIST_HEAD(ftrace_common_fields);
41 static bool eventdir_initialized;
42 
43 static LIST_HEAD(module_strings);
44 
45 struct module_string {
46 	struct list_head	next;
47 	struct module		*module;
48 	char			*str;
49 };
50 
51 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
52 
53 static struct kmem_cache *field_cachep;
54 static struct kmem_cache *file_cachep;
55 
system_refcount(struct event_subsystem * system)56 static inline int system_refcount(struct event_subsystem *system)
57 {
58 	return system->ref_count;
59 }
60 
system_refcount_inc(struct event_subsystem * system)61 static int system_refcount_inc(struct event_subsystem *system)
62 {
63 	return system->ref_count++;
64 }
65 
system_refcount_dec(struct event_subsystem * system)66 static int system_refcount_dec(struct event_subsystem *system)
67 {
68 	return --system->ref_count;
69 }
70 
71 /* Double loops, do not use break, only goto's work */
72 #define do_for_each_event_file(tr, file)			\
73 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
74 		list_for_each_entry(file, &tr->events, list)
75 
76 #define do_for_each_event_file_safe(tr, file)			\
77 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
78 		struct trace_event_file *___n;				\
79 		list_for_each_entry_safe(file, ___n, &tr->events, list)
80 
81 #define while_for_each_event_file()		\
82 	}
83 
84 static struct ftrace_event_field *
__find_event_field(struct list_head * head,char * name)85 __find_event_field(struct list_head *head, char *name)
86 {
87 	struct ftrace_event_field *field;
88 
89 	list_for_each_entry(field, head, link) {
90 		if (!strcmp(field->name, name))
91 			return field;
92 	}
93 
94 	return NULL;
95 }
96 
97 struct ftrace_event_field *
trace_find_event_field(struct trace_event_call * call,char * name)98 trace_find_event_field(struct trace_event_call *call, char *name)
99 {
100 	struct ftrace_event_field *field;
101 	struct list_head *head;
102 
103 	head = trace_get_fields(call);
104 	field = __find_event_field(head, name);
105 	if (field)
106 		return field;
107 
108 	field = __find_event_field(&ftrace_generic_fields, name);
109 	if (field)
110 		return field;
111 
112 	return __find_event_field(&ftrace_common_fields, name);
113 }
114 
__trace_define_field(struct list_head * head,const char * type,const char * name,int offset,int size,int is_signed,int filter_type,int len)115 static int __trace_define_field(struct list_head *head, const char *type,
116 				const char *name, int offset, int size,
117 				int is_signed, int filter_type, int len)
118 {
119 	struct ftrace_event_field *field;
120 
121 	field = kmem_cache_alloc(field_cachep, GFP_TRACE);
122 	if (!field)
123 		return -ENOMEM;
124 
125 	field->name = name;
126 	field->type = type;
127 
128 	if (filter_type == FILTER_OTHER)
129 		field->filter_type = filter_assign_type(type);
130 	else
131 		field->filter_type = filter_type;
132 
133 	field->offset = offset;
134 	field->size = size;
135 	field->is_signed = is_signed;
136 	field->len = len;
137 
138 	list_add(&field->link, head);
139 
140 	return 0;
141 }
142 
trace_define_field(struct trace_event_call * call,const char * type,const char * name,int offset,int size,int is_signed,int filter_type)143 int trace_define_field(struct trace_event_call *call, const char *type,
144 		       const char *name, int offset, int size, int is_signed,
145 		       int filter_type)
146 {
147 	struct list_head *head;
148 
149 	if (WARN_ON(!call->class))
150 		return 0;
151 
152 	head = trace_get_fields(call);
153 	return __trace_define_field(head, type, name, offset, size,
154 				    is_signed, filter_type, 0);
155 }
156 EXPORT_SYMBOL_GPL(trace_define_field);
157 
trace_define_field_ext(struct trace_event_call * call,const char * type,const char * name,int offset,int size,int is_signed,int filter_type,int len)158 static int trace_define_field_ext(struct trace_event_call *call, const char *type,
159 		       const char *name, int offset, int size, int is_signed,
160 		       int filter_type, int len)
161 {
162 	struct list_head *head;
163 
164 	if (WARN_ON(!call->class))
165 		return 0;
166 
167 	head = trace_get_fields(call);
168 	return __trace_define_field(head, type, name, offset, size,
169 				    is_signed, filter_type, len);
170 }
171 
172 #define __generic_field(type, item, filter_type)			\
173 	ret = __trace_define_field(&ftrace_generic_fields, #type,	\
174 				   #item, 0, 0, is_signed_type(type),	\
175 				   filter_type, 0);			\
176 	if (ret)							\
177 		return ret;
178 
179 #define __common_field(type, item)					\
180 	ret = __trace_define_field(&ftrace_common_fields, #type,	\
181 				   "common_" #item,			\
182 				   offsetof(typeof(ent), item),		\
183 				   sizeof(ent.item),			\
184 				   is_signed_type(type), FILTER_OTHER, 0);	\
185 	if (ret)							\
186 		return ret;
187 
trace_define_generic_fields(void)188 static int trace_define_generic_fields(void)
189 {
190 	int ret;
191 
192 	__generic_field(int, CPU, FILTER_CPU);
193 	__generic_field(int, cpu, FILTER_CPU);
194 	__generic_field(int, common_cpu, FILTER_CPU);
195 	__generic_field(char *, COMM, FILTER_COMM);
196 	__generic_field(char *, comm, FILTER_COMM);
197 	__generic_field(char *, stacktrace, FILTER_STACKTRACE);
198 	__generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
199 
200 	return ret;
201 }
202 
trace_define_common_fields(void)203 static int trace_define_common_fields(void)
204 {
205 	int ret;
206 	struct trace_entry ent;
207 
208 	__common_field(unsigned short, type);
209 	__common_field(unsigned char, flags);
210 	/* Holds both preempt_count and migrate_disable */
211 	__common_field(unsigned char, preempt_count);
212 	__common_field(int, pid);
213 
214 	return ret;
215 }
216 
trace_destroy_fields(struct trace_event_call * call)217 static void trace_destroy_fields(struct trace_event_call *call)
218 {
219 	struct ftrace_event_field *field, *next;
220 	struct list_head *head;
221 
222 	head = trace_get_fields(call);
223 	list_for_each_entry_safe(field, next, head, link) {
224 		list_del(&field->link);
225 		kmem_cache_free(field_cachep, field);
226 	}
227 }
228 
229 /*
230  * run-time version of trace_event_get_offsets_<call>() that returns the last
231  * accessible offset of trace fields excluding __dynamic_array bytes
232  */
trace_event_get_offsets(struct trace_event_call * call)233 int trace_event_get_offsets(struct trace_event_call *call)
234 {
235 	struct ftrace_event_field *tail;
236 	struct list_head *head;
237 
238 	head = trace_get_fields(call);
239 	/*
240 	 * head->next points to the last field with the largest offset,
241 	 * since it was added last by trace_define_field()
242 	 */
243 	tail = list_first_entry(head, struct ftrace_event_field, link);
244 	return tail->offset + tail->size;
245 }
246 
247 /*
248  * Check if the referenced field is an array and return true,
249  * as arrays are OK to dereference.
250  */
test_field(const char * fmt,struct trace_event_call * call)251 static bool test_field(const char *fmt, struct trace_event_call *call)
252 {
253 	struct trace_event_fields *field = call->class->fields_array;
254 	const char *array_descriptor;
255 	const char *p = fmt;
256 	int len;
257 
258 	if (!(len = str_has_prefix(fmt, "REC->")))
259 		return false;
260 	fmt += len;
261 	for (p = fmt; *p; p++) {
262 		if (!isalnum(*p) && *p != '_')
263 			break;
264 	}
265 	len = p - fmt;
266 
267 	for (; field->type; field++) {
268 		if (strncmp(field->name, fmt, len) ||
269 		    field->name[len])
270 			continue;
271 		array_descriptor = strchr(field->type, '[');
272 		/* This is an array and is OK to dereference. */
273 		return array_descriptor != NULL;
274 	}
275 	return false;
276 }
277 
278 /*
279  * Examine the print fmt of the event looking for unsafe dereference
280  * pointers using %p* that could be recorded in the trace event and
281  * much later referenced after the pointer was freed. Dereferencing
282  * pointers are OK, if it is dereferenced into the event itself.
283  */
test_event_printk(struct trace_event_call * call)284 static void test_event_printk(struct trace_event_call *call)
285 {
286 	u64 dereference_flags = 0;
287 	bool first = true;
288 	const char *fmt, *c, *r, *a;
289 	int parens = 0;
290 	char in_quote = 0;
291 	int start_arg = 0;
292 	int arg = 0;
293 	int i;
294 
295 	fmt = call->print_fmt;
296 
297 	if (!fmt)
298 		return;
299 
300 	for (i = 0; fmt[i]; i++) {
301 		switch (fmt[i]) {
302 		case '\\':
303 			i++;
304 			if (!fmt[i])
305 				return;
306 			continue;
307 		case '"':
308 		case '\'':
309 			/*
310 			 * The print fmt starts with a string that
311 			 * is processed first to find %p* usage,
312 			 * then after the first string, the print fmt
313 			 * contains arguments that are used to check
314 			 * if the dereferenced %p* usage is safe.
315 			 */
316 			if (first) {
317 				if (fmt[i] == '\'')
318 					continue;
319 				if (in_quote) {
320 					arg = 0;
321 					first = false;
322 					/*
323 					 * If there was no %p* uses
324 					 * the fmt is OK.
325 					 */
326 					if (!dereference_flags)
327 						return;
328 				}
329 			}
330 			if (in_quote) {
331 				if (in_quote == fmt[i])
332 					in_quote = 0;
333 			} else {
334 				in_quote = fmt[i];
335 			}
336 			continue;
337 		case '%':
338 			if (!first || !in_quote)
339 				continue;
340 			i++;
341 			if (!fmt[i])
342 				return;
343 			switch (fmt[i]) {
344 			case '%':
345 				continue;
346 			case 'p':
347 				/* Find dereferencing fields */
348 				switch (fmt[i + 1]) {
349 				case 'B': case 'R': case 'r':
350 				case 'b': case 'M': case 'm':
351 				case 'I': case 'i': case 'E':
352 				case 'U': case 'V': case 'N':
353 				case 'a': case 'd': case 'D':
354 				case 'g': case 't': case 'C':
355 				case 'O': case 'f':
356 					if (WARN_ONCE(arg == 63,
357 						      "Too many args for event: %s",
358 						      trace_event_name(call)))
359 						return;
360 					dereference_flags |= 1ULL << arg;
361 				}
362 				break;
363 			default:
364 			{
365 				bool star = false;
366 				int j;
367 
368 				/* Increment arg if %*s exists. */
369 				for (j = 0; fmt[i + j]; j++) {
370 					if (isdigit(fmt[i + j]) ||
371 					    fmt[i + j] == '.')
372 						continue;
373 					if (fmt[i + j] == '*') {
374 						star = true;
375 						continue;
376 					}
377 					if ((fmt[i + j] == 's') && star)
378 						arg++;
379 					break;
380 				}
381 				break;
382 			} /* default */
383 
384 			} /* switch */
385 			arg++;
386 			continue;
387 		case '(':
388 			if (in_quote)
389 				continue;
390 			parens++;
391 			continue;
392 		case ')':
393 			if (in_quote)
394 				continue;
395 			parens--;
396 			if (WARN_ONCE(parens < 0,
397 				      "Paren mismatch for event: %s\narg='%s'\n%*s",
398 				      trace_event_name(call),
399 				      fmt + start_arg,
400 				      (i - start_arg) + 5, "^"))
401 				return;
402 			continue;
403 		case ',':
404 			if (in_quote || parens)
405 				continue;
406 			i++;
407 			while (isspace(fmt[i]))
408 				i++;
409 			start_arg = i;
410 			if (!(dereference_flags & (1ULL << arg)))
411 				goto next_arg;
412 
413 			/* Find the REC-> in the argument */
414 			c = strchr(fmt + i, ',');
415 			r = strstr(fmt + i, "REC->");
416 			if (r && (!c || r < c)) {
417 				/*
418 				 * Addresses of events on the buffer,
419 				 * or an array on the buffer is
420 				 * OK to dereference.
421 				 * There's ways to fool this, but
422 				 * this is to catch common mistakes,
423 				 * not malicious code.
424 				 */
425 				a = strchr(fmt + i, '&');
426 				if ((a && (a < r)) || test_field(r, call))
427 					dereference_flags &= ~(1ULL << arg);
428 			} else if ((r = strstr(fmt + i, "__get_dynamic_array(")) &&
429 				   (!c || r < c)) {
430 				dereference_flags &= ~(1ULL << arg);
431 			} else if ((r = strstr(fmt + i, "__get_sockaddr(")) &&
432 				   (!c || r < c)) {
433 				dereference_flags &= ~(1ULL << arg);
434 			}
435 
436 		next_arg:
437 			i--;
438 			arg++;
439 		}
440 	}
441 
442 	/*
443 	 * If you triggered the below warning, the trace event reported
444 	 * uses an unsafe dereference pointer %p*. As the data stored
445 	 * at the trace event time may no longer exist when the trace
446 	 * event is printed, dereferencing to the original source is
447 	 * unsafe. The source of the dereference must be copied into the
448 	 * event itself, and the dereference must access the copy instead.
449 	 */
450 	if (WARN_ON_ONCE(dereference_flags)) {
451 		arg = 1;
452 		while (!(dereference_flags & 1)) {
453 			dereference_flags >>= 1;
454 			arg++;
455 		}
456 		pr_warn("event %s has unsafe dereference of argument %d\n",
457 			trace_event_name(call), arg);
458 		pr_warn("print_fmt: %s\n", fmt);
459 	}
460 }
461 
trace_event_raw_init(struct trace_event_call * call)462 int trace_event_raw_init(struct trace_event_call *call)
463 {
464 	int id;
465 
466 	id = register_trace_event(&call->event);
467 	if (!id)
468 		return -ENODEV;
469 
470 	test_event_printk(call);
471 
472 	return 0;
473 }
474 EXPORT_SYMBOL_GPL(trace_event_raw_init);
475 
trace_event_ignore_this_pid(struct trace_event_file * trace_file)476 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
477 {
478 	struct trace_array *tr = trace_file->tr;
479 	struct trace_array_cpu *data;
480 	struct trace_pid_list *no_pid_list;
481 	struct trace_pid_list *pid_list;
482 
483 	pid_list = rcu_dereference_raw(tr->filtered_pids);
484 	no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
485 
486 	if (!pid_list && !no_pid_list)
487 		return false;
488 
489 	data = this_cpu_ptr(tr->array_buffer.data);
490 
491 	return data->ignore_pid;
492 }
493 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
494 
trace_event_buffer_reserve(struct trace_event_buffer * fbuffer,struct trace_event_file * trace_file,unsigned long len)495 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
496 				 struct trace_event_file *trace_file,
497 				 unsigned long len)
498 {
499 	struct trace_event_call *event_call = trace_file->event_call;
500 
501 	if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
502 	    trace_event_ignore_this_pid(trace_file))
503 		return NULL;
504 
505 	/*
506 	 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
507 	 * preemption (adding one to the preempt_count). Since we are
508 	 * interested in the preempt_count at the time the tracepoint was
509 	 * hit, we need to subtract one to offset the increment.
510 	 */
511 	fbuffer->trace_ctx = tracing_gen_ctx_dec();
512 	fbuffer->trace_file = trace_file;
513 
514 	fbuffer->event =
515 		trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
516 						event_call->event.type, len,
517 						fbuffer->trace_ctx);
518 	if (!fbuffer->event)
519 		return NULL;
520 
521 	fbuffer->regs = NULL;
522 	fbuffer->entry = ring_buffer_event_data(fbuffer->event);
523 	return fbuffer->entry;
524 }
525 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
526 
trace_event_reg(struct trace_event_call * call,enum trace_reg type,void * data)527 int trace_event_reg(struct trace_event_call *call,
528 		    enum trace_reg type, void *data)
529 {
530 	struct trace_event_file *file = data;
531 
532 	WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
533 	switch (type) {
534 	case TRACE_REG_REGISTER:
535 		return tracepoint_probe_register(call->tp,
536 						 call->class->probe,
537 						 file);
538 	case TRACE_REG_UNREGISTER:
539 		tracepoint_probe_unregister(call->tp,
540 					    call->class->probe,
541 					    file);
542 		return 0;
543 
544 #ifdef CONFIG_PERF_EVENTS
545 	case TRACE_REG_PERF_REGISTER:
546 		return tracepoint_probe_register(call->tp,
547 						 call->class->perf_probe,
548 						 call);
549 	case TRACE_REG_PERF_UNREGISTER:
550 		tracepoint_probe_unregister(call->tp,
551 					    call->class->perf_probe,
552 					    call);
553 		return 0;
554 	case TRACE_REG_PERF_OPEN:
555 	case TRACE_REG_PERF_CLOSE:
556 	case TRACE_REG_PERF_ADD:
557 	case TRACE_REG_PERF_DEL:
558 		return 0;
559 #endif
560 	}
561 	return 0;
562 }
563 EXPORT_SYMBOL_GPL(trace_event_reg);
564 
trace_event_enable_cmd_record(bool enable)565 void trace_event_enable_cmd_record(bool enable)
566 {
567 	struct trace_event_file *file;
568 	struct trace_array *tr;
569 
570 	lockdep_assert_held(&event_mutex);
571 
572 	do_for_each_event_file(tr, file) {
573 
574 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
575 			continue;
576 
577 		if (enable) {
578 			tracing_start_cmdline_record();
579 			set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
580 		} else {
581 			tracing_stop_cmdline_record();
582 			clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
583 		}
584 	} while_for_each_event_file();
585 }
586 
trace_event_enable_tgid_record(bool enable)587 void trace_event_enable_tgid_record(bool enable)
588 {
589 	struct trace_event_file *file;
590 	struct trace_array *tr;
591 
592 	lockdep_assert_held(&event_mutex);
593 
594 	do_for_each_event_file(tr, file) {
595 		if (!(file->flags & EVENT_FILE_FL_ENABLED))
596 			continue;
597 
598 		if (enable) {
599 			tracing_start_tgid_record();
600 			set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
601 		} else {
602 			tracing_stop_tgid_record();
603 			clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
604 				  &file->flags);
605 		}
606 	} while_for_each_event_file();
607 }
608 
__ftrace_event_enable_disable(struct trace_event_file * file,int enable,int soft_disable)609 static int __ftrace_event_enable_disable(struct trace_event_file *file,
610 					 int enable, int soft_disable)
611 {
612 	struct trace_event_call *call = file->event_call;
613 	struct trace_array *tr = file->tr;
614 	int ret = 0;
615 	int disable;
616 
617 	switch (enable) {
618 	case 0:
619 		/*
620 		 * When soft_disable is set and enable is cleared, the sm_ref
621 		 * reference counter is decremented. If it reaches 0, we want
622 		 * to clear the SOFT_DISABLED flag but leave the event in the
623 		 * state that it was. That is, if the event was enabled and
624 		 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
625 		 * is set we do not want the event to be enabled before we
626 		 * clear the bit.
627 		 *
628 		 * When soft_disable is not set but the SOFT_MODE flag is,
629 		 * we do nothing. Do not disable the tracepoint, otherwise
630 		 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
631 		 */
632 		if (soft_disable) {
633 			if (atomic_dec_return(&file->sm_ref) > 0)
634 				break;
635 			disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
636 			clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
637 			/* Disable use of trace_buffered_event */
638 			trace_buffered_event_disable();
639 		} else
640 			disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
641 
642 		if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
643 			clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
644 			if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
645 				tracing_stop_cmdline_record();
646 				clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
647 			}
648 
649 			if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
650 				tracing_stop_tgid_record();
651 				clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
652 			}
653 
654 			call->class->reg(call, TRACE_REG_UNREGISTER, file);
655 		}
656 		/* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
657 		if (file->flags & EVENT_FILE_FL_SOFT_MODE)
658 			set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
659 		else
660 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
661 		break;
662 	case 1:
663 		/*
664 		 * When soft_disable is set and enable is set, we want to
665 		 * register the tracepoint for the event, but leave the event
666 		 * as is. That means, if the event was already enabled, we do
667 		 * nothing (but set SOFT_MODE). If the event is disabled, we
668 		 * set SOFT_DISABLED before enabling the event tracepoint, so
669 		 * it still seems to be disabled.
670 		 */
671 		if (!soft_disable)
672 			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
673 		else {
674 			if (atomic_inc_return(&file->sm_ref) > 1)
675 				break;
676 			set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
677 			/* Enable use of trace_buffered_event */
678 			trace_buffered_event_enable();
679 		}
680 
681 		if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
682 			bool cmd = false, tgid = false;
683 
684 			/* Keep the event disabled, when going to SOFT_MODE. */
685 			if (soft_disable)
686 				set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
687 
688 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
689 				cmd = true;
690 				tracing_start_cmdline_record();
691 				set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
692 			}
693 
694 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
695 				tgid = true;
696 				tracing_start_tgid_record();
697 				set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
698 			}
699 
700 			ret = call->class->reg(call, TRACE_REG_REGISTER, file);
701 			if (ret) {
702 				if (cmd)
703 					tracing_stop_cmdline_record();
704 				if (tgid)
705 					tracing_stop_tgid_record();
706 				pr_info("event trace: Could not enable event "
707 					"%s\n", trace_event_name(call));
708 				break;
709 			}
710 			set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
711 
712 			/* WAS_ENABLED gets set but never cleared. */
713 			set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
714 		}
715 		break;
716 	}
717 
718 	return ret;
719 }
720 
trace_event_enable_disable(struct trace_event_file * file,int enable,int soft_disable)721 int trace_event_enable_disable(struct trace_event_file *file,
722 			       int enable, int soft_disable)
723 {
724 	return __ftrace_event_enable_disable(file, enable, soft_disable);
725 }
726 
ftrace_event_enable_disable(struct trace_event_file * file,int enable)727 static int ftrace_event_enable_disable(struct trace_event_file *file,
728 				       int enable)
729 {
730 	return __ftrace_event_enable_disable(file, enable, 0);
731 }
732 
ftrace_clear_events(struct trace_array * tr)733 static void ftrace_clear_events(struct trace_array *tr)
734 {
735 	struct trace_event_file *file;
736 
737 	mutex_lock(&event_mutex);
738 	list_for_each_entry(file, &tr->events, list) {
739 		ftrace_event_enable_disable(file, 0);
740 	}
741 	mutex_unlock(&event_mutex);
742 }
743 
744 static void
event_filter_pid_sched_process_exit(void * data,struct task_struct * task)745 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
746 {
747 	struct trace_pid_list *pid_list;
748 	struct trace_array *tr = data;
749 
750 	pid_list = rcu_dereference_raw(tr->filtered_pids);
751 	trace_filter_add_remove_task(pid_list, NULL, task);
752 
753 	pid_list = rcu_dereference_raw(tr->filtered_no_pids);
754 	trace_filter_add_remove_task(pid_list, NULL, task);
755 }
756 
757 static void
event_filter_pid_sched_process_fork(void * data,struct task_struct * self,struct task_struct * task)758 event_filter_pid_sched_process_fork(void *data,
759 				    struct task_struct *self,
760 				    struct task_struct *task)
761 {
762 	struct trace_pid_list *pid_list;
763 	struct trace_array *tr = data;
764 
765 	pid_list = rcu_dereference_sched(tr->filtered_pids);
766 	trace_filter_add_remove_task(pid_list, self, task);
767 
768 	pid_list = rcu_dereference_sched(tr->filtered_no_pids);
769 	trace_filter_add_remove_task(pid_list, self, task);
770 }
771 
trace_event_follow_fork(struct trace_array * tr,bool enable)772 void trace_event_follow_fork(struct trace_array *tr, bool enable)
773 {
774 	if (enable) {
775 		register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
776 						       tr, INT_MIN);
777 		register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
778 						       tr, INT_MAX);
779 	} else {
780 		unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
781 						    tr);
782 		unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
783 						    tr);
784 	}
785 }
786 
787 static void
event_filter_pid_sched_switch_probe_pre(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)788 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
789 					struct task_struct *prev,
790 					struct task_struct *next,
791 					unsigned int prev_state)
792 {
793 	struct trace_array *tr = data;
794 	struct trace_pid_list *no_pid_list;
795 	struct trace_pid_list *pid_list;
796 	bool ret;
797 
798 	pid_list = rcu_dereference_sched(tr->filtered_pids);
799 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
800 
801 	/*
802 	 * Sched switch is funny, as we only want to ignore it
803 	 * in the notrace case if both prev and next should be ignored.
804 	 */
805 	ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
806 		trace_ignore_this_task(NULL, no_pid_list, next);
807 
808 	this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
809 		       (trace_ignore_this_task(pid_list, NULL, prev) &&
810 			trace_ignore_this_task(pid_list, NULL, next)));
811 }
812 
813 static void
event_filter_pid_sched_switch_probe_post(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)814 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
815 					 struct task_struct *prev,
816 					 struct task_struct *next,
817 					 unsigned int prev_state)
818 {
819 	struct trace_array *tr = data;
820 	struct trace_pid_list *no_pid_list;
821 	struct trace_pid_list *pid_list;
822 
823 	pid_list = rcu_dereference_sched(tr->filtered_pids);
824 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
825 
826 	this_cpu_write(tr->array_buffer.data->ignore_pid,
827 		       trace_ignore_this_task(pid_list, no_pid_list, next));
828 }
829 
830 static void
event_filter_pid_sched_wakeup_probe_pre(void * data,struct task_struct * task)831 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
832 {
833 	struct trace_array *tr = data;
834 	struct trace_pid_list *no_pid_list;
835 	struct trace_pid_list *pid_list;
836 
837 	/* Nothing to do if we are already tracing */
838 	if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
839 		return;
840 
841 	pid_list = rcu_dereference_sched(tr->filtered_pids);
842 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
843 
844 	this_cpu_write(tr->array_buffer.data->ignore_pid,
845 		       trace_ignore_this_task(pid_list, no_pid_list, task));
846 }
847 
848 static void
event_filter_pid_sched_wakeup_probe_post(void * data,struct task_struct * task)849 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
850 {
851 	struct trace_array *tr = data;
852 	struct trace_pid_list *no_pid_list;
853 	struct trace_pid_list *pid_list;
854 
855 	/* Nothing to do if we are not tracing */
856 	if (this_cpu_read(tr->array_buffer.data->ignore_pid))
857 		return;
858 
859 	pid_list = rcu_dereference_sched(tr->filtered_pids);
860 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
861 
862 	/* Set tracing if current is enabled */
863 	this_cpu_write(tr->array_buffer.data->ignore_pid,
864 		       trace_ignore_this_task(pid_list, no_pid_list, current));
865 }
866 
unregister_pid_events(struct trace_array * tr)867 static void unregister_pid_events(struct trace_array *tr)
868 {
869 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
870 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
871 
872 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
873 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
874 
875 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
876 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
877 
878 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
879 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
880 }
881 
__ftrace_clear_event_pids(struct trace_array * tr,int type)882 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
883 {
884 	struct trace_pid_list *pid_list;
885 	struct trace_pid_list *no_pid_list;
886 	struct trace_event_file *file;
887 	int cpu;
888 
889 	pid_list = rcu_dereference_protected(tr->filtered_pids,
890 					     lockdep_is_held(&event_mutex));
891 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
892 					     lockdep_is_held(&event_mutex));
893 
894 	/* Make sure there's something to do */
895 	if (!pid_type_enabled(type, pid_list, no_pid_list))
896 		return;
897 
898 	if (!still_need_pid_events(type, pid_list, no_pid_list)) {
899 		unregister_pid_events(tr);
900 
901 		list_for_each_entry(file, &tr->events, list) {
902 			clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
903 		}
904 
905 		for_each_possible_cpu(cpu)
906 			per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
907 	}
908 
909 	if (type & TRACE_PIDS)
910 		rcu_assign_pointer(tr->filtered_pids, NULL);
911 
912 	if (type & TRACE_NO_PIDS)
913 		rcu_assign_pointer(tr->filtered_no_pids, NULL);
914 
915 	/* Wait till all users are no longer using pid filtering */
916 	tracepoint_synchronize_unregister();
917 
918 	if ((type & TRACE_PIDS) && pid_list)
919 		trace_pid_list_free(pid_list);
920 
921 	if ((type & TRACE_NO_PIDS) && no_pid_list)
922 		trace_pid_list_free(no_pid_list);
923 }
924 
ftrace_clear_event_pids(struct trace_array * tr,int type)925 static void ftrace_clear_event_pids(struct trace_array *tr, int type)
926 {
927 	mutex_lock(&event_mutex);
928 	__ftrace_clear_event_pids(tr, type);
929 	mutex_unlock(&event_mutex);
930 }
931 
__put_system(struct event_subsystem * system)932 static void __put_system(struct event_subsystem *system)
933 {
934 	struct event_filter *filter = system->filter;
935 
936 	WARN_ON_ONCE(system_refcount(system) == 0);
937 	if (system_refcount_dec(system))
938 		return;
939 
940 	list_del(&system->list);
941 
942 	if (filter) {
943 		kfree(filter->filter_string);
944 		kfree(filter);
945 	}
946 	kfree_const(system->name);
947 	kfree(system);
948 }
949 
__get_system(struct event_subsystem * system)950 static void __get_system(struct event_subsystem *system)
951 {
952 	WARN_ON_ONCE(system_refcount(system) == 0);
953 	system_refcount_inc(system);
954 }
955 
__get_system_dir(struct trace_subsystem_dir * dir)956 static void __get_system_dir(struct trace_subsystem_dir *dir)
957 {
958 	WARN_ON_ONCE(dir->ref_count == 0);
959 	dir->ref_count++;
960 	__get_system(dir->subsystem);
961 }
962 
__put_system_dir(struct trace_subsystem_dir * dir)963 static void __put_system_dir(struct trace_subsystem_dir *dir)
964 {
965 	WARN_ON_ONCE(dir->ref_count == 0);
966 	/* If the subsystem is about to be freed, the dir must be too */
967 	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
968 
969 	__put_system(dir->subsystem);
970 	if (!--dir->ref_count)
971 		kfree(dir);
972 }
973 
put_system(struct trace_subsystem_dir * dir)974 static void put_system(struct trace_subsystem_dir *dir)
975 {
976 	mutex_lock(&event_mutex);
977 	__put_system_dir(dir);
978 	mutex_unlock(&event_mutex);
979 }
980 
remove_subsystem(struct trace_subsystem_dir * dir)981 static void remove_subsystem(struct trace_subsystem_dir *dir)
982 {
983 	if (!dir)
984 		return;
985 
986 	if (!--dir->nr_events) {
987 		eventfs_remove_dir(dir->ei);
988 		list_del(&dir->list);
989 		__put_system_dir(dir);
990 	}
991 }
992 
event_file_get(struct trace_event_file * file)993 void event_file_get(struct trace_event_file *file)
994 {
995 	refcount_inc(&file->ref);
996 }
997 
event_file_put(struct trace_event_file * file)998 void event_file_put(struct trace_event_file *file)
999 {
1000 	if (WARN_ON_ONCE(!refcount_read(&file->ref))) {
1001 		if (file->flags & EVENT_FILE_FL_FREED)
1002 			kmem_cache_free(file_cachep, file);
1003 		return;
1004 	}
1005 
1006 	if (refcount_dec_and_test(&file->ref)) {
1007 		/* Count should only go to zero when it is freed */
1008 		if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1009 			return;
1010 		kmem_cache_free(file_cachep, file);
1011 	}
1012 }
1013 
remove_event_file_dir(struct trace_event_file * file)1014 static void remove_event_file_dir(struct trace_event_file *file)
1015 {
1016 	eventfs_remove_dir(file->ei);
1017 	list_del(&file->list);
1018 	remove_subsystem(file->system);
1019 	free_event_filter(file->filter);
1020 	file->flags |= EVENT_FILE_FL_FREED;
1021 	event_file_put(file);
1022 }
1023 
1024 /*
1025  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1026  */
1027 static int
__ftrace_set_clr_event_nolock(struct trace_array * tr,const char * match,const char * sub,const char * event,int set)1028 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1029 			      const char *sub, const char *event, int set)
1030 {
1031 	struct trace_event_file *file;
1032 	struct trace_event_call *call;
1033 	const char *name;
1034 	int ret = -EINVAL;
1035 	int eret = 0;
1036 
1037 	list_for_each_entry(file, &tr->events, list) {
1038 
1039 		call = file->event_call;
1040 		name = trace_event_name(call);
1041 
1042 		if (!name || !call->class || !call->class->reg)
1043 			continue;
1044 
1045 		if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1046 			continue;
1047 
1048 		if (match &&
1049 		    strcmp(match, name) != 0 &&
1050 		    strcmp(match, call->class->system) != 0)
1051 			continue;
1052 
1053 		if (sub && strcmp(sub, call->class->system) != 0)
1054 			continue;
1055 
1056 		if (event && strcmp(event, name) != 0)
1057 			continue;
1058 
1059 		ret = ftrace_event_enable_disable(file, set);
1060 
1061 		/*
1062 		 * Save the first error and return that. Some events
1063 		 * may still have been enabled, but let the user
1064 		 * know that something went wrong.
1065 		 */
1066 		if (ret && !eret)
1067 			eret = ret;
1068 
1069 		ret = eret;
1070 	}
1071 
1072 	return ret;
1073 }
1074 
__ftrace_set_clr_event(struct trace_array * tr,const char * match,const char * sub,const char * event,int set)1075 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1076 				  const char *sub, const char *event, int set)
1077 {
1078 	int ret;
1079 
1080 	mutex_lock(&event_mutex);
1081 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
1082 	mutex_unlock(&event_mutex);
1083 
1084 	return ret;
1085 }
1086 
ftrace_set_clr_event(struct trace_array * tr,char * buf,int set)1087 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1088 {
1089 	char *event = NULL, *sub = NULL, *match;
1090 	int ret;
1091 
1092 	if (!tr)
1093 		return -ENOENT;
1094 	/*
1095 	 * The buf format can be <subsystem>:<event-name>
1096 	 *  *:<event-name> means any event by that name.
1097 	 *  :<event-name> is the same.
1098 	 *
1099 	 *  <subsystem>:* means all events in that subsystem
1100 	 *  <subsystem>: means the same.
1101 	 *
1102 	 *  <name> (no ':') means all events in a subsystem with
1103 	 *  the name <name> or any event that matches <name>
1104 	 */
1105 
1106 	match = strsep(&buf, ":");
1107 	if (buf) {
1108 		sub = match;
1109 		event = buf;
1110 		match = NULL;
1111 
1112 		if (!strlen(sub) || strcmp(sub, "*") == 0)
1113 			sub = NULL;
1114 		if (!strlen(event) || strcmp(event, "*") == 0)
1115 			event = NULL;
1116 	}
1117 
1118 	ret = __ftrace_set_clr_event(tr, match, sub, event, set);
1119 
1120 	/* Put back the colon to allow this to be called again */
1121 	if (buf)
1122 		*(buf - 1) = ':';
1123 
1124 	return ret;
1125 }
1126 
1127 /**
1128  * trace_set_clr_event - enable or disable an event
1129  * @system: system name to match (NULL for any system)
1130  * @event: event name to match (NULL for all events, within system)
1131  * @set: 1 to enable, 0 to disable
1132  *
1133  * This is a way for other parts of the kernel to enable or disable
1134  * event recording.
1135  *
1136  * Returns 0 on success, -EINVAL if the parameters do not match any
1137  * registered events.
1138  */
trace_set_clr_event(const char * system,const char * event,int set)1139 int trace_set_clr_event(const char *system, const char *event, int set)
1140 {
1141 	struct trace_array *tr = top_trace_array();
1142 
1143 	if (!tr)
1144 		return -ENODEV;
1145 
1146 	return __ftrace_set_clr_event(tr, NULL, system, event, set);
1147 }
1148 EXPORT_SYMBOL_GPL(trace_set_clr_event);
1149 
1150 /**
1151  * trace_array_set_clr_event - enable or disable an event for a trace array.
1152  * @tr: concerned trace array.
1153  * @system: system name to match (NULL for any system)
1154  * @event: event name to match (NULL for all events, within system)
1155  * @enable: true to enable, false to disable
1156  *
1157  * This is a way for other parts of the kernel to enable or disable
1158  * event recording.
1159  *
1160  * Returns 0 on success, -EINVAL if the parameters do not match any
1161  * registered events.
1162  */
trace_array_set_clr_event(struct trace_array * tr,const char * system,const char * event,bool enable)1163 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1164 		const char *event, bool enable)
1165 {
1166 	int set;
1167 
1168 	if (!tr)
1169 		return -ENOENT;
1170 
1171 	set = (enable == true) ? 1 : 0;
1172 	return __ftrace_set_clr_event(tr, NULL, system, event, set);
1173 }
1174 EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1175 
1176 /* 128 should be much more than enough */
1177 #define EVENT_BUF_SIZE		127
1178 
1179 static ssize_t
ftrace_event_write(struct file * file,const char __user * ubuf,size_t cnt,loff_t * ppos)1180 ftrace_event_write(struct file *file, const char __user *ubuf,
1181 		   size_t cnt, loff_t *ppos)
1182 {
1183 	struct trace_parser parser;
1184 	struct seq_file *m = file->private_data;
1185 	struct trace_array *tr = m->private;
1186 	ssize_t read, ret;
1187 
1188 	if (!cnt)
1189 		return 0;
1190 
1191 	ret = tracing_update_buffers(tr);
1192 	if (ret < 0)
1193 		return ret;
1194 
1195 	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1196 		return -ENOMEM;
1197 
1198 	read = trace_get_user(&parser, ubuf, cnt, ppos);
1199 
1200 	if (read >= 0 && trace_parser_loaded((&parser))) {
1201 		int set = 1;
1202 
1203 		if (*parser.buffer == '!')
1204 			set = 0;
1205 
1206 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1207 		if (ret)
1208 			goto out_put;
1209 	}
1210 
1211 	ret = read;
1212 
1213  out_put:
1214 	trace_parser_put(&parser);
1215 
1216 	return ret;
1217 }
1218 
1219 static void *
t_next(struct seq_file * m,void * v,loff_t * pos)1220 t_next(struct seq_file *m, void *v, loff_t *pos)
1221 {
1222 	struct trace_event_file *file = v;
1223 	struct trace_event_call *call;
1224 	struct trace_array *tr = m->private;
1225 
1226 	(*pos)++;
1227 
1228 	list_for_each_entry_continue(file, &tr->events, list) {
1229 		call = file->event_call;
1230 		/*
1231 		 * The ftrace subsystem is for showing formats only.
1232 		 * They can not be enabled or disabled via the event files.
1233 		 */
1234 		if (call->class && call->class->reg &&
1235 		    !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1236 			return file;
1237 	}
1238 
1239 	return NULL;
1240 }
1241 
t_start(struct seq_file * m,loff_t * pos)1242 static void *t_start(struct seq_file *m, loff_t *pos)
1243 {
1244 	struct trace_event_file *file;
1245 	struct trace_array *tr = m->private;
1246 	loff_t l;
1247 
1248 	mutex_lock(&event_mutex);
1249 
1250 	file = list_entry(&tr->events, struct trace_event_file, list);
1251 	for (l = 0; l <= *pos; ) {
1252 		file = t_next(m, file, &l);
1253 		if (!file)
1254 			break;
1255 	}
1256 	return file;
1257 }
1258 
1259 static void *
s_next(struct seq_file * m,void * v,loff_t * pos)1260 s_next(struct seq_file *m, void *v, loff_t *pos)
1261 {
1262 	struct trace_event_file *file = v;
1263 	struct trace_array *tr = m->private;
1264 
1265 	(*pos)++;
1266 
1267 	list_for_each_entry_continue(file, &tr->events, list) {
1268 		if (file->flags & EVENT_FILE_FL_ENABLED)
1269 			return file;
1270 	}
1271 
1272 	return NULL;
1273 }
1274 
s_start(struct seq_file * m,loff_t * pos)1275 static void *s_start(struct seq_file *m, loff_t *pos)
1276 {
1277 	struct trace_event_file *file;
1278 	struct trace_array *tr = m->private;
1279 	loff_t l;
1280 
1281 	mutex_lock(&event_mutex);
1282 
1283 	file = list_entry(&tr->events, struct trace_event_file, list);
1284 	for (l = 0; l <= *pos; ) {
1285 		file = s_next(m, file, &l);
1286 		if (!file)
1287 			break;
1288 	}
1289 	return file;
1290 }
1291 
t_show(struct seq_file * m,void * v)1292 static int t_show(struct seq_file *m, void *v)
1293 {
1294 	struct trace_event_file *file = v;
1295 	struct trace_event_call *call = file->event_call;
1296 
1297 	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1298 		seq_printf(m, "%s:", call->class->system);
1299 	seq_printf(m, "%s\n", trace_event_name(call));
1300 
1301 	return 0;
1302 }
1303 
t_stop(struct seq_file * m,void * p)1304 static void t_stop(struct seq_file *m, void *p)
1305 {
1306 	mutex_unlock(&event_mutex);
1307 }
1308 
1309 static void *
__next(struct seq_file * m,void * v,loff_t * pos,int type)1310 __next(struct seq_file *m, void *v, loff_t *pos, int type)
1311 {
1312 	struct trace_array *tr = m->private;
1313 	struct trace_pid_list *pid_list;
1314 
1315 	if (type == TRACE_PIDS)
1316 		pid_list = rcu_dereference_sched(tr->filtered_pids);
1317 	else
1318 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1319 
1320 	return trace_pid_next(pid_list, v, pos);
1321 }
1322 
1323 static void *
p_next(struct seq_file * m,void * v,loff_t * pos)1324 p_next(struct seq_file *m, void *v, loff_t *pos)
1325 {
1326 	return __next(m, v, pos, TRACE_PIDS);
1327 }
1328 
1329 static void *
np_next(struct seq_file * m,void * v,loff_t * pos)1330 np_next(struct seq_file *m, void *v, loff_t *pos)
1331 {
1332 	return __next(m, v, pos, TRACE_NO_PIDS);
1333 }
1334 
__start(struct seq_file * m,loff_t * pos,int type)1335 static void *__start(struct seq_file *m, loff_t *pos, int type)
1336 	__acquires(RCU)
1337 {
1338 	struct trace_pid_list *pid_list;
1339 	struct trace_array *tr = m->private;
1340 
1341 	/*
1342 	 * Grab the mutex, to keep calls to p_next() having the same
1343 	 * tr->filtered_pids as p_start() has.
1344 	 * If we just passed the tr->filtered_pids around, then RCU would
1345 	 * have been enough, but doing that makes things more complex.
1346 	 */
1347 	mutex_lock(&event_mutex);
1348 	rcu_read_lock_sched();
1349 
1350 	if (type == TRACE_PIDS)
1351 		pid_list = rcu_dereference_sched(tr->filtered_pids);
1352 	else
1353 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1354 
1355 	if (!pid_list)
1356 		return NULL;
1357 
1358 	return trace_pid_start(pid_list, pos);
1359 }
1360 
p_start(struct seq_file * m,loff_t * pos)1361 static void *p_start(struct seq_file *m, loff_t *pos)
1362 	__acquires(RCU)
1363 {
1364 	return __start(m, pos, TRACE_PIDS);
1365 }
1366 
np_start(struct seq_file * m,loff_t * pos)1367 static void *np_start(struct seq_file *m, loff_t *pos)
1368 	__acquires(RCU)
1369 {
1370 	return __start(m, pos, TRACE_NO_PIDS);
1371 }
1372 
p_stop(struct seq_file * m,void * p)1373 static void p_stop(struct seq_file *m, void *p)
1374 	__releases(RCU)
1375 {
1376 	rcu_read_unlock_sched();
1377 	mutex_unlock(&event_mutex);
1378 }
1379 
1380 static ssize_t
event_enable_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1381 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1382 		  loff_t *ppos)
1383 {
1384 	struct trace_event_file *file;
1385 	unsigned long flags;
1386 	char buf[4] = "0";
1387 
1388 	mutex_lock(&event_mutex);
1389 	file = event_file_file(filp);
1390 	if (likely(file))
1391 		flags = file->flags;
1392 	mutex_unlock(&event_mutex);
1393 
1394 	if (!file)
1395 		return -ENODEV;
1396 
1397 	if (flags & EVENT_FILE_FL_ENABLED &&
1398 	    !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1399 		strcpy(buf, "1");
1400 
1401 	if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1402 	    flags & EVENT_FILE_FL_SOFT_MODE)
1403 		strcat(buf, "*");
1404 
1405 	strcat(buf, "\n");
1406 
1407 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1408 }
1409 
1410 static ssize_t
event_enable_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1411 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1412 		   loff_t *ppos)
1413 {
1414 	struct trace_event_file *file;
1415 	unsigned long val;
1416 	int ret;
1417 
1418 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1419 	if (ret)
1420 		return ret;
1421 
1422 	switch (val) {
1423 	case 0:
1424 	case 1:
1425 		ret = -ENODEV;
1426 		mutex_lock(&event_mutex);
1427 		file = event_file_file(filp);
1428 		if (likely(file)) {
1429 			ret = tracing_update_buffers(file->tr);
1430 			if (ret < 0) {
1431 				mutex_unlock(&event_mutex);
1432 				return ret;
1433 			}
1434 			ret = ftrace_event_enable_disable(file, val);
1435 		}
1436 		mutex_unlock(&event_mutex);
1437 		break;
1438 
1439 	default:
1440 		return -EINVAL;
1441 	}
1442 
1443 	*ppos += cnt;
1444 
1445 	return ret ? ret : cnt;
1446 }
1447 
1448 static ssize_t
system_enable_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1449 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1450 		   loff_t *ppos)
1451 {
1452 	const char set_to_char[4] = { '?', '0', '1', 'X' };
1453 	struct trace_subsystem_dir *dir = filp->private_data;
1454 	struct event_subsystem *system = dir->subsystem;
1455 	struct trace_event_call *call;
1456 	struct trace_event_file *file;
1457 	struct trace_array *tr = dir->tr;
1458 	char buf[2];
1459 	int set = 0;
1460 	int ret;
1461 
1462 	mutex_lock(&event_mutex);
1463 	list_for_each_entry(file, &tr->events, list) {
1464 		call = file->event_call;
1465 		if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1466 		    !trace_event_name(call) || !call->class || !call->class->reg)
1467 			continue;
1468 
1469 		if (system && strcmp(call->class->system, system->name) != 0)
1470 			continue;
1471 
1472 		/*
1473 		 * We need to find out if all the events are set
1474 		 * or if all events or cleared, or if we have
1475 		 * a mixture.
1476 		 */
1477 		set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1478 
1479 		/*
1480 		 * If we have a mixture, no need to look further.
1481 		 */
1482 		if (set == 3)
1483 			break;
1484 	}
1485 	mutex_unlock(&event_mutex);
1486 
1487 	buf[0] = set_to_char[set];
1488 	buf[1] = '\n';
1489 
1490 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1491 
1492 	return ret;
1493 }
1494 
1495 static ssize_t
system_enable_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1496 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1497 		    loff_t *ppos)
1498 {
1499 	struct trace_subsystem_dir *dir = filp->private_data;
1500 	struct event_subsystem *system = dir->subsystem;
1501 	const char *name = NULL;
1502 	unsigned long val;
1503 	ssize_t ret;
1504 
1505 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1506 	if (ret)
1507 		return ret;
1508 
1509 	ret = tracing_update_buffers(dir->tr);
1510 	if (ret < 0)
1511 		return ret;
1512 
1513 	if (val != 0 && val != 1)
1514 		return -EINVAL;
1515 
1516 	/*
1517 	 * Opening of "enable" adds a ref count to system,
1518 	 * so the name is safe to use.
1519 	 */
1520 	if (system)
1521 		name = system->name;
1522 
1523 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
1524 	if (ret)
1525 		goto out;
1526 
1527 	ret = cnt;
1528 
1529 out:
1530 	*ppos += cnt;
1531 
1532 	return ret;
1533 }
1534 
1535 enum {
1536 	FORMAT_HEADER		= 1,
1537 	FORMAT_FIELD_SEPERATOR	= 2,
1538 	FORMAT_PRINTFMT		= 3,
1539 };
1540 
f_next(struct seq_file * m,void * v,loff_t * pos)1541 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1542 {
1543 	struct trace_event_file *file = event_file_data(m->private);
1544 	struct trace_event_call *call = file->event_call;
1545 	struct list_head *common_head = &ftrace_common_fields;
1546 	struct list_head *head = trace_get_fields(call);
1547 	struct list_head *node = v;
1548 
1549 	(*pos)++;
1550 
1551 	switch ((unsigned long)v) {
1552 	case FORMAT_HEADER:
1553 		node = common_head;
1554 		break;
1555 
1556 	case FORMAT_FIELD_SEPERATOR:
1557 		node = head;
1558 		break;
1559 
1560 	case FORMAT_PRINTFMT:
1561 		/* all done */
1562 		return NULL;
1563 	}
1564 
1565 	node = node->prev;
1566 	if (node == common_head)
1567 		return (void *)FORMAT_FIELD_SEPERATOR;
1568 	else if (node == head)
1569 		return (void *)FORMAT_PRINTFMT;
1570 	else
1571 		return node;
1572 }
1573 
f_show(struct seq_file * m,void * v)1574 static int f_show(struct seq_file *m, void *v)
1575 {
1576 	struct trace_event_file *file = event_file_data(m->private);
1577 	struct trace_event_call *call = file->event_call;
1578 	struct ftrace_event_field *field;
1579 	const char *array_descriptor;
1580 
1581 	switch ((unsigned long)v) {
1582 	case FORMAT_HEADER:
1583 		seq_printf(m, "name: %s\n", trace_event_name(call));
1584 		seq_printf(m, "ID: %d\n", call->event.type);
1585 		seq_puts(m, "format:\n");
1586 		return 0;
1587 
1588 	case FORMAT_FIELD_SEPERATOR:
1589 		seq_putc(m, '\n');
1590 		return 0;
1591 
1592 	case FORMAT_PRINTFMT:
1593 		seq_printf(m, "\nprint fmt: %s\n",
1594 			   call->print_fmt);
1595 		return 0;
1596 	}
1597 
1598 	field = list_entry(v, struct ftrace_event_field, link);
1599 	/*
1600 	 * Smartly shows the array type(except dynamic array).
1601 	 * Normal:
1602 	 *	field:TYPE VAR
1603 	 * If TYPE := TYPE[LEN], it is shown:
1604 	 *	field:TYPE VAR[LEN]
1605 	 */
1606 	array_descriptor = strchr(field->type, '[');
1607 
1608 	if (str_has_prefix(field->type, "__data_loc"))
1609 		array_descriptor = NULL;
1610 
1611 	if (!array_descriptor)
1612 		seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1613 			   field->type, field->name, field->offset,
1614 			   field->size, !!field->is_signed);
1615 	else if (field->len)
1616 		seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1617 			   (int)(array_descriptor - field->type),
1618 			   field->type, field->name,
1619 			   field->len, field->offset,
1620 			   field->size, !!field->is_signed);
1621 	else
1622 		seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1623 				(int)(array_descriptor - field->type),
1624 				field->type, field->name,
1625 				field->offset, field->size, !!field->is_signed);
1626 
1627 	return 0;
1628 }
1629 
f_start(struct seq_file * m,loff_t * pos)1630 static void *f_start(struct seq_file *m, loff_t *pos)
1631 {
1632 	struct trace_event_file *file;
1633 	void *p = (void *)FORMAT_HEADER;
1634 	loff_t l = 0;
1635 
1636 	/* ->stop() is called even if ->start() fails */
1637 	mutex_lock(&event_mutex);
1638 	file = event_file_file(m->private);
1639 	if (!file)
1640 		return ERR_PTR(-ENODEV);
1641 
1642 	while (l < *pos && p)
1643 		p = f_next(m, p, &l);
1644 
1645 	return p;
1646 }
1647 
f_stop(struct seq_file * m,void * p)1648 static void f_stop(struct seq_file *m, void *p)
1649 {
1650 	mutex_unlock(&event_mutex);
1651 }
1652 
1653 static const struct seq_operations trace_format_seq_ops = {
1654 	.start		= f_start,
1655 	.next		= f_next,
1656 	.stop		= f_stop,
1657 	.show		= f_show,
1658 };
1659 
trace_format_open(struct inode * inode,struct file * file)1660 static int trace_format_open(struct inode *inode, struct file *file)
1661 {
1662 	struct seq_file *m;
1663 	int ret;
1664 
1665 	/* Do we want to hide event format files on tracefs lockdown? */
1666 
1667 	ret = seq_open(file, &trace_format_seq_ops);
1668 	if (ret < 0)
1669 		return ret;
1670 
1671 	m = file->private_data;
1672 	m->private = file;
1673 
1674 	return 0;
1675 }
1676 
1677 #ifdef CONFIG_PERF_EVENTS
1678 static ssize_t
event_id_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1679 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1680 {
1681 	int id = (long)event_file_data(filp);
1682 	char buf[32];
1683 	int len;
1684 
1685 	if (unlikely(!id))
1686 		return -ENODEV;
1687 
1688 	len = sprintf(buf, "%d\n", id);
1689 
1690 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
1691 }
1692 #endif
1693 
1694 static ssize_t
event_filter_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1695 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1696 		  loff_t *ppos)
1697 {
1698 	struct trace_event_file *file;
1699 	struct trace_seq *s;
1700 	int r = -ENODEV;
1701 
1702 	if (*ppos)
1703 		return 0;
1704 
1705 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1706 
1707 	if (!s)
1708 		return -ENOMEM;
1709 
1710 	trace_seq_init(s);
1711 
1712 	mutex_lock(&event_mutex);
1713 	file = event_file_file(filp);
1714 	if (file)
1715 		print_event_filter(file, s);
1716 	mutex_unlock(&event_mutex);
1717 
1718 	if (file)
1719 		r = simple_read_from_buffer(ubuf, cnt, ppos,
1720 					    s->buffer, trace_seq_used(s));
1721 
1722 	kfree(s);
1723 
1724 	return r;
1725 }
1726 
1727 static ssize_t
event_filter_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1728 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1729 		   loff_t *ppos)
1730 {
1731 	struct trace_event_file *file;
1732 	char *buf;
1733 	int err = -ENODEV;
1734 
1735 	if (cnt >= PAGE_SIZE)
1736 		return -EINVAL;
1737 
1738 	buf = memdup_user_nul(ubuf, cnt);
1739 	if (IS_ERR(buf))
1740 		return PTR_ERR(buf);
1741 
1742 	mutex_lock(&event_mutex);
1743 	file = event_file_file(filp);
1744 	if (file) {
1745 		if (file->flags & EVENT_FILE_FL_FREED)
1746 			err = -ENODEV;
1747 		else
1748 			err = apply_event_filter(file, buf);
1749 	}
1750 	mutex_unlock(&event_mutex);
1751 
1752 	kfree(buf);
1753 	if (err < 0)
1754 		return err;
1755 
1756 	*ppos += cnt;
1757 
1758 	return cnt;
1759 }
1760 
1761 static LIST_HEAD(event_subsystems);
1762 
subsystem_open(struct inode * inode,struct file * filp)1763 static int subsystem_open(struct inode *inode, struct file *filp)
1764 {
1765 	struct trace_subsystem_dir *dir = NULL, *iter_dir;
1766 	struct trace_array *tr = NULL, *iter_tr;
1767 	struct event_subsystem *system = NULL;
1768 	int ret;
1769 
1770 	if (tracing_is_disabled())
1771 		return -ENODEV;
1772 
1773 	/* Make sure the system still exists */
1774 	mutex_lock(&event_mutex);
1775 	mutex_lock(&trace_types_lock);
1776 	list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
1777 		list_for_each_entry(iter_dir, &iter_tr->systems, list) {
1778 			if (iter_dir == inode->i_private) {
1779 				/* Don't open systems with no events */
1780 				tr = iter_tr;
1781 				dir = iter_dir;
1782 				if (dir->nr_events) {
1783 					__get_system_dir(dir);
1784 					system = dir->subsystem;
1785 				}
1786 				goto exit_loop;
1787 			}
1788 		}
1789 	}
1790  exit_loop:
1791 	mutex_unlock(&trace_types_lock);
1792 	mutex_unlock(&event_mutex);
1793 
1794 	if (!system)
1795 		return -ENODEV;
1796 
1797 	/* Still need to increment the ref count of the system */
1798 	if (trace_array_get(tr) < 0) {
1799 		put_system(dir);
1800 		return -ENODEV;
1801 	}
1802 
1803 	ret = tracing_open_generic(inode, filp);
1804 	if (ret < 0) {
1805 		trace_array_put(tr);
1806 		put_system(dir);
1807 	}
1808 
1809 	return ret;
1810 }
1811 
system_tr_open(struct inode * inode,struct file * filp)1812 static int system_tr_open(struct inode *inode, struct file *filp)
1813 {
1814 	struct trace_subsystem_dir *dir;
1815 	struct trace_array *tr = inode->i_private;
1816 	int ret;
1817 
1818 	/* Make a temporary dir that has no system but points to tr */
1819 	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1820 	if (!dir)
1821 		return -ENOMEM;
1822 
1823 	ret = tracing_open_generic_tr(inode, filp);
1824 	if (ret < 0) {
1825 		kfree(dir);
1826 		return ret;
1827 	}
1828 	dir->tr = tr;
1829 	filp->private_data = dir;
1830 
1831 	return 0;
1832 }
1833 
subsystem_release(struct inode * inode,struct file * file)1834 static int subsystem_release(struct inode *inode, struct file *file)
1835 {
1836 	struct trace_subsystem_dir *dir = file->private_data;
1837 
1838 	trace_array_put(dir->tr);
1839 
1840 	/*
1841 	 * If dir->subsystem is NULL, then this is a temporary
1842 	 * descriptor that was made for a trace_array to enable
1843 	 * all subsystems.
1844 	 */
1845 	if (dir->subsystem)
1846 		put_system(dir);
1847 	else
1848 		kfree(dir);
1849 
1850 	return 0;
1851 }
1852 
1853 static ssize_t
subsystem_filter_read(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1854 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1855 		      loff_t *ppos)
1856 {
1857 	struct trace_subsystem_dir *dir = filp->private_data;
1858 	struct event_subsystem *system = dir->subsystem;
1859 	struct trace_seq *s;
1860 	int r;
1861 
1862 	if (*ppos)
1863 		return 0;
1864 
1865 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1866 	if (!s)
1867 		return -ENOMEM;
1868 
1869 	trace_seq_init(s);
1870 
1871 	print_subsystem_event_filter(system, s);
1872 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1873 				    s->buffer, trace_seq_used(s));
1874 
1875 	kfree(s);
1876 
1877 	return r;
1878 }
1879 
1880 static ssize_t
subsystem_filter_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)1881 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1882 		       loff_t *ppos)
1883 {
1884 	struct trace_subsystem_dir *dir = filp->private_data;
1885 	char *buf;
1886 	int err;
1887 
1888 	if (cnt >= PAGE_SIZE)
1889 		return -EINVAL;
1890 
1891 	buf = memdup_user_nul(ubuf, cnt);
1892 	if (IS_ERR(buf))
1893 		return PTR_ERR(buf);
1894 
1895 	err = apply_subsystem_event_filter(dir, buf);
1896 	kfree(buf);
1897 	if (err < 0)
1898 		return err;
1899 
1900 	*ppos += cnt;
1901 
1902 	return cnt;
1903 }
1904 
1905 static ssize_t
show_header_page_file(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1906 show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1907 {
1908 	struct trace_array *tr = filp->private_data;
1909 	struct trace_seq *s;
1910 	int r;
1911 
1912 	if (*ppos)
1913 		return 0;
1914 
1915 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1916 	if (!s)
1917 		return -ENOMEM;
1918 
1919 	trace_seq_init(s);
1920 
1921 	ring_buffer_print_page_header(tr->array_buffer.buffer, s);
1922 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1923 				    s->buffer, trace_seq_used(s));
1924 
1925 	kfree(s);
1926 
1927 	return r;
1928 }
1929 
1930 static ssize_t
show_header_event_file(struct file * filp,char __user * ubuf,size_t cnt,loff_t * ppos)1931 show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1932 {
1933 	struct trace_seq *s;
1934 	int r;
1935 
1936 	if (*ppos)
1937 		return 0;
1938 
1939 	s = kmalloc(sizeof(*s), GFP_KERNEL);
1940 	if (!s)
1941 		return -ENOMEM;
1942 
1943 	trace_seq_init(s);
1944 
1945 	ring_buffer_print_entry_header(s);
1946 	r = simple_read_from_buffer(ubuf, cnt, ppos,
1947 				    s->buffer, trace_seq_used(s));
1948 
1949 	kfree(s);
1950 
1951 	return r;
1952 }
1953 
ignore_task_cpu(void * data)1954 static void ignore_task_cpu(void *data)
1955 {
1956 	struct trace_array *tr = data;
1957 	struct trace_pid_list *pid_list;
1958 	struct trace_pid_list *no_pid_list;
1959 
1960 	/*
1961 	 * This function is called by on_each_cpu() while the
1962 	 * event_mutex is held.
1963 	 */
1964 	pid_list = rcu_dereference_protected(tr->filtered_pids,
1965 					     mutex_is_locked(&event_mutex));
1966 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1967 					     mutex_is_locked(&event_mutex));
1968 
1969 	this_cpu_write(tr->array_buffer.data->ignore_pid,
1970 		       trace_ignore_this_task(pid_list, no_pid_list, current));
1971 }
1972 
register_pid_events(struct trace_array * tr)1973 static void register_pid_events(struct trace_array *tr)
1974 {
1975 	/*
1976 	 * Register a probe that is called before all other probes
1977 	 * to set ignore_pid if next or prev do not match.
1978 	 * Register a probe this is called after all other probes
1979 	 * to only keep ignore_pid set if next pid matches.
1980 	 */
1981 	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
1982 					 tr, INT_MAX);
1983 	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
1984 					 tr, 0);
1985 
1986 	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
1987 					 tr, INT_MAX);
1988 	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
1989 					 tr, 0);
1990 
1991 	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
1992 					     tr, INT_MAX);
1993 	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
1994 					     tr, 0);
1995 
1996 	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
1997 					 tr, INT_MAX);
1998 	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
1999 					 tr, 0);
2000 }
2001 
2002 static ssize_t
event_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos,int type)2003 event_pid_write(struct file *filp, const char __user *ubuf,
2004 		size_t cnt, loff_t *ppos, int type)
2005 {
2006 	struct seq_file *m = filp->private_data;
2007 	struct trace_array *tr = m->private;
2008 	struct trace_pid_list *filtered_pids = NULL;
2009 	struct trace_pid_list *other_pids = NULL;
2010 	struct trace_pid_list *pid_list;
2011 	struct trace_event_file *file;
2012 	ssize_t ret;
2013 
2014 	if (!cnt)
2015 		return 0;
2016 
2017 	ret = tracing_update_buffers(tr);
2018 	if (ret < 0)
2019 		return ret;
2020 
2021 	mutex_lock(&event_mutex);
2022 
2023 	if (type == TRACE_PIDS) {
2024 		filtered_pids = rcu_dereference_protected(tr->filtered_pids,
2025 							  lockdep_is_held(&event_mutex));
2026 		other_pids = rcu_dereference_protected(tr->filtered_no_pids,
2027 							  lockdep_is_held(&event_mutex));
2028 	} else {
2029 		filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
2030 							  lockdep_is_held(&event_mutex));
2031 		other_pids = rcu_dereference_protected(tr->filtered_pids,
2032 							  lockdep_is_held(&event_mutex));
2033 	}
2034 
2035 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
2036 	if (ret < 0)
2037 		goto out;
2038 
2039 	if (type == TRACE_PIDS)
2040 		rcu_assign_pointer(tr->filtered_pids, pid_list);
2041 	else
2042 		rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2043 
2044 	list_for_each_entry(file, &tr->events, list) {
2045 		set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2046 	}
2047 
2048 	if (filtered_pids) {
2049 		tracepoint_synchronize_unregister();
2050 		trace_pid_list_free(filtered_pids);
2051 	} else if (pid_list && !other_pids) {
2052 		register_pid_events(tr);
2053 	}
2054 
2055 	/*
2056 	 * Ignoring of pids is done at task switch. But we have to
2057 	 * check for those tasks that are currently running.
2058 	 * Always do this in case a pid was appended or removed.
2059 	 */
2060 	on_each_cpu(ignore_task_cpu, tr, 1);
2061 
2062  out:
2063 	mutex_unlock(&event_mutex);
2064 
2065 	if (ret > 0)
2066 		*ppos += ret;
2067 
2068 	return ret;
2069 }
2070 
2071 static ssize_t
ftrace_event_pid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2072 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2073 		       size_t cnt, loff_t *ppos)
2074 {
2075 	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2076 }
2077 
2078 static ssize_t
ftrace_event_npid_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2079 ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2080 			size_t cnt, loff_t *ppos)
2081 {
2082 	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2083 }
2084 
2085 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2086 static int ftrace_event_set_open(struct inode *inode, struct file *file);
2087 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2088 static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2089 static int ftrace_event_release(struct inode *inode, struct file *file);
2090 
2091 static const struct seq_operations show_event_seq_ops = {
2092 	.start = t_start,
2093 	.next = t_next,
2094 	.show = t_show,
2095 	.stop = t_stop,
2096 };
2097 
2098 static const struct seq_operations show_set_event_seq_ops = {
2099 	.start = s_start,
2100 	.next = s_next,
2101 	.show = t_show,
2102 	.stop = t_stop,
2103 };
2104 
2105 static const struct seq_operations show_set_pid_seq_ops = {
2106 	.start = p_start,
2107 	.next = p_next,
2108 	.show = trace_pid_show,
2109 	.stop = p_stop,
2110 };
2111 
2112 static const struct seq_operations show_set_no_pid_seq_ops = {
2113 	.start = np_start,
2114 	.next = np_next,
2115 	.show = trace_pid_show,
2116 	.stop = p_stop,
2117 };
2118 
2119 static const struct file_operations ftrace_avail_fops = {
2120 	.open = ftrace_event_avail_open,
2121 	.read = seq_read,
2122 	.llseek = seq_lseek,
2123 	.release = seq_release,
2124 };
2125 
2126 static const struct file_operations ftrace_set_event_fops = {
2127 	.open = ftrace_event_set_open,
2128 	.read = seq_read,
2129 	.write = ftrace_event_write,
2130 	.llseek = seq_lseek,
2131 	.release = ftrace_event_release,
2132 };
2133 
2134 static const struct file_operations ftrace_set_event_pid_fops = {
2135 	.open = ftrace_event_set_pid_open,
2136 	.read = seq_read,
2137 	.write = ftrace_event_pid_write,
2138 	.llseek = seq_lseek,
2139 	.release = ftrace_event_release,
2140 };
2141 
2142 static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2143 	.open = ftrace_event_set_npid_open,
2144 	.read = seq_read,
2145 	.write = ftrace_event_npid_write,
2146 	.llseek = seq_lseek,
2147 	.release = ftrace_event_release,
2148 };
2149 
2150 static const struct file_operations ftrace_enable_fops = {
2151 	.open = tracing_open_file_tr,
2152 	.read = event_enable_read,
2153 	.write = event_enable_write,
2154 	.release = tracing_release_file_tr,
2155 	.llseek = default_llseek,
2156 };
2157 
2158 static const struct file_operations ftrace_event_format_fops = {
2159 	.open = trace_format_open,
2160 	.read = seq_read,
2161 	.llseek = seq_lseek,
2162 	.release = seq_release,
2163 };
2164 
2165 #ifdef CONFIG_PERF_EVENTS
2166 static const struct file_operations ftrace_event_id_fops = {
2167 	.read = event_id_read,
2168 	.llseek = default_llseek,
2169 };
2170 #endif
2171 
2172 static const struct file_operations ftrace_event_filter_fops = {
2173 	.open = tracing_open_file_tr,
2174 	.read = event_filter_read,
2175 	.write = event_filter_write,
2176 	.release = tracing_release_file_tr,
2177 	.llseek = default_llseek,
2178 };
2179 
2180 static const struct file_operations ftrace_subsystem_filter_fops = {
2181 	.open = subsystem_open,
2182 	.read = subsystem_filter_read,
2183 	.write = subsystem_filter_write,
2184 	.llseek = default_llseek,
2185 	.release = subsystem_release,
2186 };
2187 
2188 static const struct file_operations ftrace_system_enable_fops = {
2189 	.open = subsystem_open,
2190 	.read = system_enable_read,
2191 	.write = system_enable_write,
2192 	.llseek = default_llseek,
2193 	.release = subsystem_release,
2194 };
2195 
2196 static const struct file_operations ftrace_tr_enable_fops = {
2197 	.open = system_tr_open,
2198 	.read = system_enable_read,
2199 	.write = system_enable_write,
2200 	.llseek = default_llseek,
2201 	.release = subsystem_release,
2202 };
2203 
2204 static const struct file_operations ftrace_show_header_page_fops = {
2205 	.open = tracing_open_generic_tr,
2206 	.read = show_header_page_file,
2207 	.llseek = default_llseek,
2208 	.release = tracing_release_generic_tr,
2209 };
2210 
2211 static const struct file_operations ftrace_show_header_event_fops = {
2212 	.open = tracing_open_generic_tr,
2213 	.read = show_header_event_file,
2214 	.llseek = default_llseek,
2215 	.release = tracing_release_generic_tr,
2216 };
2217 
2218 static int
ftrace_event_open(struct inode * inode,struct file * file,const struct seq_operations * seq_ops)2219 ftrace_event_open(struct inode *inode, struct file *file,
2220 		  const struct seq_operations *seq_ops)
2221 {
2222 	struct seq_file *m;
2223 	int ret;
2224 
2225 	ret = security_locked_down(LOCKDOWN_TRACEFS);
2226 	if (ret)
2227 		return ret;
2228 
2229 	ret = seq_open(file, seq_ops);
2230 	if (ret < 0)
2231 		return ret;
2232 	m = file->private_data;
2233 	/* copy tr over to seq ops */
2234 	m->private = inode->i_private;
2235 
2236 	return ret;
2237 }
2238 
ftrace_event_release(struct inode * inode,struct file * file)2239 static int ftrace_event_release(struct inode *inode, struct file *file)
2240 {
2241 	struct trace_array *tr = inode->i_private;
2242 
2243 	trace_array_put(tr);
2244 
2245 	return seq_release(inode, file);
2246 }
2247 
2248 static int
ftrace_event_avail_open(struct inode * inode,struct file * file)2249 ftrace_event_avail_open(struct inode *inode, struct file *file)
2250 {
2251 	const struct seq_operations *seq_ops = &show_event_seq_ops;
2252 
2253 	/* Checks for tracefs lockdown */
2254 	return ftrace_event_open(inode, file, seq_ops);
2255 }
2256 
2257 static int
ftrace_event_set_open(struct inode * inode,struct file * file)2258 ftrace_event_set_open(struct inode *inode, struct file *file)
2259 {
2260 	const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2261 	struct trace_array *tr = inode->i_private;
2262 	int ret;
2263 
2264 	ret = tracing_check_open_get_tr(tr);
2265 	if (ret)
2266 		return ret;
2267 
2268 	if ((file->f_mode & FMODE_WRITE) &&
2269 	    (file->f_flags & O_TRUNC))
2270 		ftrace_clear_events(tr);
2271 
2272 	ret = ftrace_event_open(inode, file, seq_ops);
2273 	if (ret < 0)
2274 		trace_array_put(tr);
2275 	return ret;
2276 }
2277 
2278 static int
ftrace_event_set_pid_open(struct inode * inode,struct file * file)2279 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2280 {
2281 	const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2282 	struct trace_array *tr = inode->i_private;
2283 	int ret;
2284 
2285 	ret = tracing_check_open_get_tr(tr);
2286 	if (ret)
2287 		return ret;
2288 
2289 	if ((file->f_mode & FMODE_WRITE) &&
2290 	    (file->f_flags & O_TRUNC))
2291 		ftrace_clear_event_pids(tr, TRACE_PIDS);
2292 
2293 	ret = ftrace_event_open(inode, file, seq_ops);
2294 	if (ret < 0)
2295 		trace_array_put(tr);
2296 	return ret;
2297 }
2298 
2299 static int
ftrace_event_set_npid_open(struct inode * inode,struct file * file)2300 ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2301 {
2302 	const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2303 	struct trace_array *tr = inode->i_private;
2304 	int ret;
2305 
2306 	ret = tracing_check_open_get_tr(tr);
2307 	if (ret)
2308 		return ret;
2309 
2310 	if ((file->f_mode & FMODE_WRITE) &&
2311 	    (file->f_flags & O_TRUNC))
2312 		ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2313 
2314 	ret = ftrace_event_open(inode, file, seq_ops);
2315 	if (ret < 0)
2316 		trace_array_put(tr);
2317 	return ret;
2318 }
2319 
2320 static struct event_subsystem *
create_new_subsystem(const char * name)2321 create_new_subsystem(const char *name)
2322 {
2323 	struct event_subsystem *system;
2324 
2325 	/* need to create new entry */
2326 	system = kmalloc(sizeof(*system), GFP_KERNEL);
2327 	if (!system)
2328 		return NULL;
2329 
2330 	system->ref_count = 1;
2331 
2332 	/* Only allocate if dynamic (kprobes and modules) */
2333 	system->name = kstrdup_const(name, GFP_KERNEL);
2334 	if (!system->name)
2335 		goto out_free;
2336 
2337 	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2338 	if (!system->filter)
2339 		goto out_free;
2340 
2341 	list_add(&system->list, &event_subsystems);
2342 
2343 	return system;
2344 
2345  out_free:
2346 	kfree_const(system->name);
2347 	kfree(system);
2348 	return NULL;
2349 }
2350 
system_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)2351 static int system_callback(const char *name, umode_t *mode, void **data,
2352 		    const struct file_operations **fops)
2353 {
2354 	if (strcmp(name, "filter") == 0)
2355 		*fops = &ftrace_subsystem_filter_fops;
2356 
2357 	else if (strcmp(name, "enable") == 0)
2358 		*fops = &ftrace_system_enable_fops;
2359 
2360 	else
2361 		return 0;
2362 
2363 	*mode = TRACE_MODE_WRITE;
2364 	return 1;
2365 }
2366 
2367 static struct eventfs_inode *
event_subsystem_dir(struct trace_array * tr,const char * name,struct trace_event_file * file,struct eventfs_inode * parent)2368 event_subsystem_dir(struct trace_array *tr, const char *name,
2369 		    struct trace_event_file *file, struct eventfs_inode *parent)
2370 {
2371 	struct event_subsystem *system, *iter;
2372 	struct trace_subsystem_dir *dir;
2373 	struct eventfs_inode *ei;
2374 	int nr_entries;
2375 	static struct eventfs_entry system_entries[] = {
2376 		{
2377 			.name		= "filter",
2378 			.callback	= system_callback,
2379 		},
2380 		{
2381 			.name		= "enable",
2382 			.callback	= system_callback,
2383 		}
2384 	};
2385 
2386 	/* First see if we did not already create this dir */
2387 	list_for_each_entry(dir, &tr->systems, list) {
2388 		system = dir->subsystem;
2389 		if (strcmp(system->name, name) == 0) {
2390 			dir->nr_events++;
2391 			file->system = dir;
2392 			return dir->ei;
2393 		}
2394 	}
2395 
2396 	/* Now see if the system itself exists. */
2397 	system = NULL;
2398 	list_for_each_entry(iter, &event_subsystems, list) {
2399 		if (strcmp(iter->name, name) == 0) {
2400 			system = iter;
2401 			break;
2402 		}
2403 	}
2404 
2405 	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2406 	if (!dir)
2407 		goto out_fail;
2408 
2409 	if (!system) {
2410 		system = create_new_subsystem(name);
2411 		if (!system)
2412 			goto out_free;
2413 	} else
2414 		__get_system(system);
2415 
2416 	/* ftrace only has directories no files */
2417 	if (strcmp(name, "ftrace") == 0)
2418 		nr_entries = 0;
2419 	else
2420 		nr_entries = ARRAY_SIZE(system_entries);
2421 
2422 	ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir);
2423 	if (IS_ERR(ei)) {
2424 		pr_warn("Failed to create system directory %s\n", name);
2425 		__put_system(system);
2426 		goto out_free;
2427 	}
2428 
2429 	dir->ei = ei;
2430 	dir->tr = tr;
2431 	dir->ref_count = 1;
2432 	dir->nr_events = 1;
2433 	dir->subsystem = system;
2434 	file->system = dir;
2435 
2436 	list_add(&dir->list, &tr->systems);
2437 
2438 	return dir->ei;
2439 
2440  out_free:
2441 	kfree(dir);
2442  out_fail:
2443 	/* Only print this message if failed on memory allocation */
2444 	if (!dir || !system)
2445 		pr_warn("No memory to create event subsystem %s\n", name);
2446 	return NULL;
2447 }
2448 
2449 static int
event_define_fields(struct trace_event_call * call)2450 event_define_fields(struct trace_event_call *call)
2451 {
2452 	struct list_head *head;
2453 	int ret = 0;
2454 
2455 	/*
2456 	 * Other events may have the same class. Only update
2457 	 * the fields if they are not already defined.
2458 	 */
2459 	head = trace_get_fields(call);
2460 	if (list_empty(head)) {
2461 		struct trace_event_fields *field = call->class->fields_array;
2462 		unsigned int offset = sizeof(struct trace_entry);
2463 
2464 		for (; field->type; field++) {
2465 			if (field->type == TRACE_FUNCTION_TYPE) {
2466 				field->define_fields(call);
2467 				break;
2468 			}
2469 
2470 			offset = ALIGN(offset, field->align);
2471 			ret = trace_define_field_ext(call, field->type, field->name,
2472 						 offset, field->size,
2473 						 field->is_signed, field->filter_type,
2474 						 field->len);
2475 			if (WARN_ON_ONCE(ret)) {
2476 				pr_err("error code is %d\n", ret);
2477 				break;
2478 			}
2479 
2480 			offset += field->size;
2481 		}
2482 	}
2483 
2484 	return ret;
2485 }
2486 
event_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)2487 static int event_callback(const char *name, umode_t *mode, void **data,
2488 			  const struct file_operations **fops)
2489 {
2490 	struct trace_event_file *file = *data;
2491 	struct trace_event_call *call = file->event_call;
2492 
2493 	if (strcmp(name, "format") == 0) {
2494 		*mode = TRACE_MODE_READ;
2495 		*fops = &ftrace_event_format_fops;
2496 		return 1;
2497 	}
2498 
2499 	/*
2500 	 * Only event directories that can be enabled should have
2501 	 * triggers or filters, with the exception of the "print"
2502 	 * event that can have a "trigger" file.
2503 	 */
2504 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2505 		if (call->class->reg && strcmp(name, "enable") == 0) {
2506 			*mode = TRACE_MODE_WRITE;
2507 			*fops = &ftrace_enable_fops;
2508 			return 1;
2509 		}
2510 
2511 		if (strcmp(name, "filter") == 0) {
2512 			*mode = TRACE_MODE_WRITE;
2513 			*fops = &ftrace_event_filter_fops;
2514 			return 1;
2515 		}
2516 	}
2517 
2518 	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
2519 	    strcmp(trace_event_name(call), "print") == 0) {
2520 		if (strcmp(name, "trigger") == 0) {
2521 			*mode = TRACE_MODE_WRITE;
2522 			*fops = &event_trigger_fops;
2523 			return 1;
2524 		}
2525 	}
2526 
2527 #ifdef CONFIG_PERF_EVENTS
2528 	if (call->event.type && call->class->reg &&
2529 	    strcmp(name, "id") == 0) {
2530 		*mode = TRACE_MODE_READ;
2531 		*data = (void *)(long)call->event.type;
2532 		*fops = &ftrace_event_id_fops;
2533 		return 1;
2534 	}
2535 #endif
2536 
2537 #ifdef CONFIG_HIST_TRIGGERS
2538 	if (strcmp(name, "hist") == 0) {
2539 		*mode = TRACE_MODE_READ;
2540 		*fops = &event_hist_fops;
2541 		return 1;
2542 	}
2543 #endif
2544 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2545 	if (strcmp(name, "hist_debug") == 0) {
2546 		*mode = TRACE_MODE_READ;
2547 		*fops = &event_hist_debug_fops;
2548 		return 1;
2549 	}
2550 #endif
2551 #ifdef CONFIG_TRACE_EVENT_INJECT
2552 	if (call->event.type && call->class->reg &&
2553 	    strcmp(name, "inject") == 0) {
2554 		*mode = 0200;
2555 		*fops = &event_inject_fops;
2556 		return 1;
2557 	}
2558 #endif
2559 	return 0;
2560 }
2561 
2562 /* The file is incremented on creation and freeing the enable file decrements it */
event_release(const char * name,void * data)2563 static void event_release(const char *name, void *data)
2564 {
2565 	struct trace_event_file *file = data;
2566 
2567 	event_file_put(file);
2568 }
2569 
2570 static int
event_create_dir(struct eventfs_inode * parent,struct trace_event_file * file)2571 event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
2572 {
2573 	struct trace_event_call *call = file->event_call;
2574 	struct trace_array *tr = file->tr;
2575 	struct eventfs_inode *e_events;
2576 	struct eventfs_inode *ei;
2577 	const char *name;
2578 	int nr_entries;
2579 	int ret;
2580 	static struct eventfs_entry event_entries[] = {
2581 		{
2582 			.name		= "enable",
2583 			.callback	= event_callback,
2584 			.release	= event_release,
2585 		},
2586 		{
2587 			.name		= "filter",
2588 			.callback	= event_callback,
2589 		},
2590 		{
2591 			.name		= "trigger",
2592 			.callback	= event_callback,
2593 		},
2594 		{
2595 			.name		= "format",
2596 			.callback	= event_callback,
2597 		},
2598 #ifdef CONFIG_PERF_EVENTS
2599 		{
2600 			.name		= "id",
2601 			.callback	= event_callback,
2602 		},
2603 #endif
2604 #ifdef CONFIG_HIST_TRIGGERS
2605 		{
2606 			.name		= "hist",
2607 			.callback	= event_callback,
2608 		},
2609 #endif
2610 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2611 		{
2612 			.name		= "hist_debug",
2613 			.callback	= event_callback,
2614 		},
2615 #endif
2616 #ifdef CONFIG_TRACE_EVENT_INJECT
2617 		{
2618 			.name		= "inject",
2619 			.callback	= event_callback,
2620 		},
2621 #endif
2622 	};
2623 
2624 	/*
2625 	 * If the trace point header did not define TRACE_SYSTEM
2626 	 * then the system would be called "TRACE_SYSTEM". This should
2627 	 * never happen.
2628 	 */
2629 	if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
2630 		return -ENODEV;
2631 
2632 	e_events = event_subsystem_dir(tr, call->class->system, file, parent);
2633 	if (!e_events)
2634 		return -ENOMEM;
2635 
2636 	nr_entries = ARRAY_SIZE(event_entries);
2637 
2638 	name = trace_event_name(call);
2639 	ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file);
2640 	if (IS_ERR(ei)) {
2641 		pr_warn("Could not create tracefs '%s' directory\n", name);
2642 		return -1;
2643 	}
2644 
2645 	file->ei = ei;
2646 
2647 	ret = event_define_fields(call);
2648 	if (ret < 0) {
2649 		pr_warn("Could not initialize trace point events/%s\n", name);
2650 		return ret;
2651 	}
2652 
2653 	/* Gets decremented on freeing of the "enable" file */
2654 	event_file_get(file);
2655 
2656 	return 0;
2657 }
2658 
remove_event_from_tracers(struct trace_event_call * call)2659 static void remove_event_from_tracers(struct trace_event_call *call)
2660 {
2661 	struct trace_event_file *file;
2662 	struct trace_array *tr;
2663 
2664 	do_for_each_event_file_safe(tr, file) {
2665 		if (file->event_call != call)
2666 			continue;
2667 
2668 		remove_event_file_dir(file);
2669 		/*
2670 		 * The do_for_each_event_file_safe() is
2671 		 * a double loop. After finding the call for this
2672 		 * trace_array, we use break to jump to the next
2673 		 * trace_array.
2674 		 */
2675 		break;
2676 	} while_for_each_event_file();
2677 }
2678 
event_remove(struct trace_event_call * call)2679 static void event_remove(struct trace_event_call *call)
2680 {
2681 	struct trace_array *tr;
2682 	struct trace_event_file *file;
2683 
2684 	do_for_each_event_file(tr, file) {
2685 		if (file->event_call != call)
2686 			continue;
2687 
2688 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
2689 			tr->clear_trace = true;
2690 
2691 		ftrace_event_enable_disable(file, 0);
2692 		/*
2693 		 * The do_for_each_event_file() is
2694 		 * a double loop. After finding the call for this
2695 		 * trace_array, we use break to jump to the next
2696 		 * trace_array.
2697 		 */
2698 		break;
2699 	} while_for_each_event_file();
2700 
2701 	if (call->event.funcs)
2702 		__unregister_trace_event(&call->event);
2703 	remove_event_from_tracers(call);
2704 	list_del(&call->list);
2705 }
2706 
event_init(struct trace_event_call * call)2707 static int event_init(struct trace_event_call *call)
2708 {
2709 	int ret = 0;
2710 	const char *name;
2711 
2712 	name = trace_event_name(call);
2713 	if (WARN_ON(!name))
2714 		return -EINVAL;
2715 
2716 	if (call->class->raw_init) {
2717 		ret = call->class->raw_init(call);
2718 		if (ret < 0 && ret != -ENOSYS)
2719 			pr_warn("Could not initialize trace events/%s\n", name);
2720 	}
2721 
2722 	return ret;
2723 }
2724 
2725 static int
__register_event(struct trace_event_call * call,struct module * mod)2726 __register_event(struct trace_event_call *call, struct module *mod)
2727 {
2728 	int ret;
2729 
2730 	ret = event_init(call);
2731 	if (ret < 0)
2732 		return ret;
2733 
2734 	list_add(&call->list, &ftrace_events);
2735 	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
2736 		atomic_set(&call->refcnt, 0);
2737 	else
2738 		call->module = mod;
2739 
2740 	return 0;
2741 }
2742 
eval_replace(char * ptr,struct trace_eval_map * map,int len)2743 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
2744 {
2745 	int rlen;
2746 	int elen;
2747 
2748 	/* Find the length of the eval value as a string */
2749 	elen = snprintf(ptr, 0, "%ld", map->eval_value);
2750 	/* Make sure there's enough room to replace the string with the value */
2751 	if (len < elen)
2752 		return NULL;
2753 
2754 	snprintf(ptr, elen + 1, "%ld", map->eval_value);
2755 
2756 	/* Get the rest of the string of ptr */
2757 	rlen = strlen(ptr + len);
2758 	memmove(ptr + elen, ptr + len, rlen);
2759 	/* Make sure we end the new string */
2760 	ptr[elen + rlen] = 0;
2761 
2762 	return ptr + elen;
2763 }
2764 
update_event_printk(struct trace_event_call * call,struct trace_eval_map * map)2765 static void update_event_printk(struct trace_event_call *call,
2766 				struct trace_eval_map *map)
2767 {
2768 	char *ptr;
2769 	int quote = 0;
2770 	int len = strlen(map->eval_string);
2771 
2772 	for (ptr = call->print_fmt; *ptr; ptr++) {
2773 		if (*ptr == '\\') {
2774 			ptr++;
2775 			/* paranoid */
2776 			if (!*ptr)
2777 				break;
2778 			continue;
2779 		}
2780 		if (*ptr == '"') {
2781 			quote ^= 1;
2782 			continue;
2783 		}
2784 		if (quote)
2785 			continue;
2786 		if (isdigit(*ptr)) {
2787 			/* skip numbers */
2788 			do {
2789 				ptr++;
2790 				/* Check for alpha chars like ULL */
2791 			} while (isalnum(*ptr));
2792 			if (!*ptr)
2793 				break;
2794 			/*
2795 			 * A number must have some kind of delimiter after
2796 			 * it, and we can ignore that too.
2797 			 */
2798 			continue;
2799 		}
2800 		if (isalpha(*ptr) || *ptr == '_') {
2801 			if (strncmp(map->eval_string, ptr, len) == 0 &&
2802 			    !isalnum(ptr[len]) && ptr[len] != '_') {
2803 				ptr = eval_replace(ptr, map, len);
2804 				/* enum/sizeof string smaller than value */
2805 				if (WARN_ON_ONCE(!ptr))
2806 					return;
2807 				/*
2808 				 * No need to decrement here, as eval_replace()
2809 				 * returns the pointer to the character passed
2810 				 * the eval, and two evals can not be placed
2811 				 * back to back without something in between.
2812 				 * We can skip that something in between.
2813 				 */
2814 				continue;
2815 			}
2816 		skip_more:
2817 			do {
2818 				ptr++;
2819 			} while (isalnum(*ptr) || *ptr == '_');
2820 			if (!*ptr)
2821 				break;
2822 			/*
2823 			 * If what comes after this variable is a '.' or
2824 			 * '->' then we can continue to ignore that string.
2825 			 */
2826 			if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
2827 				ptr += *ptr == '.' ? 1 : 2;
2828 				if (!*ptr)
2829 					break;
2830 				goto skip_more;
2831 			}
2832 			/*
2833 			 * Once again, we can skip the delimiter that came
2834 			 * after the string.
2835 			 */
2836 			continue;
2837 		}
2838 	}
2839 }
2840 
add_str_to_module(struct module * module,char * str)2841 static void add_str_to_module(struct module *module, char *str)
2842 {
2843 	struct module_string *modstr;
2844 
2845 	modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
2846 
2847 	/*
2848 	 * If we failed to allocate memory here, then we'll just
2849 	 * let the str memory leak when the module is removed.
2850 	 * If this fails to allocate, there's worse problems than
2851 	 * a leaked string on module removal.
2852 	 */
2853 	if (WARN_ON_ONCE(!modstr))
2854 		return;
2855 
2856 	modstr->module = module;
2857 	modstr->str = str;
2858 
2859 	list_add(&modstr->next, &module_strings);
2860 }
2861 
update_event_fields(struct trace_event_call * call,struct trace_eval_map * map)2862 static void update_event_fields(struct trace_event_call *call,
2863 				struct trace_eval_map *map)
2864 {
2865 	struct ftrace_event_field *field;
2866 	struct list_head *head;
2867 	char *ptr;
2868 	char *str;
2869 	int len = strlen(map->eval_string);
2870 
2871 	/* Dynamic events should never have field maps */
2872 	if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
2873 		return;
2874 
2875 	head = trace_get_fields(call);
2876 	list_for_each_entry(field, head, link) {
2877 		ptr = strchr(field->type, '[');
2878 		if (!ptr)
2879 			continue;
2880 		ptr++;
2881 
2882 		if (!isalpha(*ptr) && *ptr != '_')
2883 			continue;
2884 
2885 		if (strncmp(map->eval_string, ptr, len) != 0)
2886 			continue;
2887 
2888 		str = kstrdup(field->type, GFP_KERNEL);
2889 		if (WARN_ON_ONCE(!str))
2890 			return;
2891 		ptr = str + (ptr - field->type);
2892 		ptr = eval_replace(ptr, map, len);
2893 		/* enum/sizeof string smaller than value */
2894 		if (WARN_ON_ONCE(!ptr)) {
2895 			kfree(str);
2896 			continue;
2897 		}
2898 
2899 		/*
2900 		 * If the event is part of a module, then we need to free the string
2901 		 * when the module is removed. Otherwise, it will stay allocated
2902 		 * until a reboot.
2903 		 */
2904 		if (call->module)
2905 			add_str_to_module(call->module, str);
2906 
2907 		field->type = str;
2908 	}
2909 }
2910 
trace_event_eval_update(struct trace_eval_map ** map,int len)2911 void trace_event_eval_update(struct trace_eval_map **map, int len)
2912 {
2913 	struct trace_event_call *call, *p;
2914 	const char *last_system = NULL;
2915 	bool first = false;
2916 	int last_i;
2917 	int i;
2918 
2919 	down_write(&trace_event_sem);
2920 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
2921 		/* events are usually grouped together with systems */
2922 		if (!last_system || call->class->system != last_system) {
2923 			first = true;
2924 			last_i = 0;
2925 			last_system = call->class->system;
2926 		}
2927 
2928 		/*
2929 		 * Since calls are grouped by systems, the likelihood that the
2930 		 * next call in the iteration belongs to the same system as the
2931 		 * previous call is high. As an optimization, we skip searching
2932 		 * for a map[] that matches the call's system if the last call
2933 		 * was from the same system. That's what last_i is for. If the
2934 		 * call has the same system as the previous call, then last_i
2935 		 * will be the index of the first map[] that has a matching
2936 		 * system.
2937 		 */
2938 		for (i = last_i; i < len; i++) {
2939 			if (call->class->system == map[i]->system) {
2940 				/* Save the first system if need be */
2941 				if (first) {
2942 					last_i = i;
2943 					first = false;
2944 				}
2945 				update_event_printk(call, map[i]);
2946 				update_event_fields(call, map[i]);
2947 			}
2948 		}
2949 		cond_resched();
2950 	}
2951 	up_write(&trace_event_sem);
2952 }
2953 
event_in_systems(struct trace_event_call * call,const char * systems)2954 static bool event_in_systems(struct trace_event_call *call,
2955 			     const char *systems)
2956 {
2957 	const char *system;
2958 	const char *p;
2959 
2960 	if (!systems)
2961 		return true;
2962 
2963 	system = call->class->system;
2964 	p = strstr(systems, system);
2965 	if (!p)
2966 		return false;
2967 
2968 	if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
2969 		return false;
2970 
2971 	p += strlen(system);
2972 	return !*p || isspace(*p) || *p == ',';
2973 }
2974 
2975 static struct trace_event_file *
trace_create_new_event(struct trace_event_call * call,struct trace_array * tr)2976 trace_create_new_event(struct trace_event_call *call,
2977 		       struct trace_array *tr)
2978 {
2979 	struct trace_pid_list *no_pid_list;
2980 	struct trace_pid_list *pid_list;
2981 	struct trace_event_file *file;
2982 	unsigned int first;
2983 
2984 	if (!event_in_systems(call, tr->system_names))
2985 		return NULL;
2986 
2987 	file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2988 	if (!file)
2989 		return ERR_PTR(-ENOMEM);
2990 
2991 	pid_list = rcu_dereference_protected(tr->filtered_pids,
2992 					     lockdep_is_held(&event_mutex));
2993 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2994 					     lockdep_is_held(&event_mutex));
2995 
2996 	if (!trace_pid_list_first(pid_list, &first) ||
2997 	    !trace_pid_list_first(no_pid_list, &first))
2998 		file->flags |= EVENT_FILE_FL_PID_FILTER;
2999 
3000 	file->event_call = call;
3001 	file->tr = tr;
3002 	atomic_set(&file->sm_ref, 0);
3003 	atomic_set(&file->tm_ref, 0);
3004 	INIT_LIST_HEAD(&file->triggers);
3005 	list_add(&file->list, &tr->events);
3006 	refcount_set(&file->ref, 1);
3007 
3008 	return file;
3009 }
3010 
3011 #define MAX_BOOT_TRIGGERS 32
3012 
3013 static struct boot_triggers {
3014 	const char		*event;
3015 	char			*trigger;
3016 } bootup_triggers[MAX_BOOT_TRIGGERS];
3017 
3018 static char bootup_trigger_buf[COMMAND_LINE_SIZE];
3019 static int nr_boot_triggers;
3020 
setup_trace_triggers(char * str)3021 static __init int setup_trace_triggers(char *str)
3022 {
3023 	char *trigger;
3024 	char *buf;
3025 	int i;
3026 
3027 	strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
3028 	trace_set_ring_buffer_expanded(NULL);
3029 	disable_tracing_selftest("running event triggers");
3030 
3031 	buf = bootup_trigger_buf;
3032 	for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
3033 		trigger = strsep(&buf, ",");
3034 		if (!trigger)
3035 			break;
3036 		bootup_triggers[i].event = strsep(&trigger, ".");
3037 		bootup_triggers[i].trigger = trigger;
3038 		if (!bootup_triggers[i].trigger)
3039 			break;
3040 	}
3041 
3042 	nr_boot_triggers = i;
3043 	return 1;
3044 }
3045 __setup("trace_trigger=", setup_trace_triggers);
3046 
3047 /* Add an event to a trace directory */
3048 static int
__trace_add_new_event(struct trace_event_call * call,struct trace_array * tr)3049 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
3050 {
3051 	struct trace_event_file *file;
3052 
3053 	file = trace_create_new_event(call, tr);
3054 	/*
3055 	 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3056 	 * allocation, or NULL if the event is not part of the tr->system_names.
3057 	 * When the event is not part of the tr->system_names, return zero, not
3058 	 * an error.
3059 	 */
3060 	if (!file)
3061 		return 0;
3062 
3063 	if (IS_ERR(file))
3064 		return PTR_ERR(file);
3065 
3066 	if (eventdir_initialized)
3067 		return event_create_dir(tr->event_dir, file);
3068 	else
3069 		return event_define_fields(call);
3070 }
3071 
trace_early_triggers(struct trace_event_file * file,const char * name)3072 static void trace_early_triggers(struct trace_event_file *file, const char *name)
3073 {
3074 	int ret;
3075 	int i;
3076 
3077 	for (i = 0; i < nr_boot_triggers; i++) {
3078 		if (strcmp(name, bootup_triggers[i].event))
3079 			continue;
3080 		mutex_lock(&event_mutex);
3081 		ret = trigger_process_regex(file, bootup_triggers[i].trigger);
3082 		mutex_unlock(&event_mutex);
3083 		if (ret)
3084 			pr_err("Failed to register trigger '%s' on event %s\n",
3085 			       bootup_triggers[i].trigger,
3086 			       bootup_triggers[i].event);
3087 	}
3088 }
3089 
3090 /*
3091  * Just create a descriptor for early init. A descriptor is required
3092  * for enabling events at boot. We want to enable events before
3093  * the filesystem is initialized.
3094  */
3095 static int
__trace_early_add_new_event(struct trace_event_call * call,struct trace_array * tr)3096 __trace_early_add_new_event(struct trace_event_call *call,
3097 			    struct trace_array *tr)
3098 {
3099 	struct trace_event_file *file;
3100 	int ret;
3101 
3102 	file = trace_create_new_event(call, tr);
3103 	/*
3104 	 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3105 	 * allocation, or NULL if the event is not part of the tr->system_names.
3106 	 * When the event is not part of the tr->system_names, return zero, not
3107 	 * an error.
3108 	 */
3109 	if (!file)
3110 		return 0;
3111 
3112 	if (IS_ERR(file))
3113 		return PTR_ERR(file);
3114 
3115 	ret = event_define_fields(call);
3116 	if (ret)
3117 		return ret;
3118 
3119 	trace_early_triggers(file, trace_event_name(call));
3120 
3121 	return 0;
3122 }
3123 
3124 struct ftrace_module_file_ops;
3125 static void __add_event_to_tracers(struct trace_event_call *call);
3126 
3127 /* Add an additional event_call dynamically */
trace_add_event_call(struct trace_event_call * call)3128 int trace_add_event_call(struct trace_event_call *call)
3129 {
3130 	int ret;
3131 	lockdep_assert_held(&event_mutex);
3132 
3133 	mutex_lock(&trace_types_lock);
3134 
3135 	ret = __register_event(call, NULL);
3136 	if (ret >= 0)
3137 		__add_event_to_tracers(call);
3138 
3139 	mutex_unlock(&trace_types_lock);
3140 	return ret;
3141 }
3142 EXPORT_SYMBOL_GPL(trace_add_event_call);
3143 
3144 /*
3145  * Must be called under locking of trace_types_lock, event_mutex and
3146  * trace_event_sem.
3147  */
__trace_remove_event_call(struct trace_event_call * call)3148 static void __trace_remove_event_call(struct trace_event_call *call)
3149 {
3150 	event_remove(call);
3151 	trace_destroy_fields(call);
3152 	free_event_filter(call->filter);
3153 	call->filter = NULL;
3154 }
3155 
probe_remove_event_call(struct trace_event_call * call)3156 static int probe_remove_event_call(struct trace_event_call *call)
3157 {
3158 	struct trace_array *tr;
3159 	struct trace_event_file *file;
3160 
3161 #ifdef CONFIG_PERF_EVENTS
3162 	if (call->perf_refcount)
3163 		return -EBUSY;
3164 #endif
3165 	do_for_each_event_file(tr, file) {
3166 		if (file->event_call != call)
3167 			continue;
3168 		/*
3169 		 * We can't rely on ftrace_event_enable_disable(enable => 0)
3170 		 * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
3171 		 * TRACE_REG_UNREGISTER.
3172 		 */
3173 		if (file->flags & EVENT_FILE_FL_ENABLED)
3174 			goto busy;
3175 
3176 		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3177 			tr->clear_trace = true;
3178 		/*
3179 		 * The do_for_each_event_file_safe() is
3180 		 * a double loop. After finding the call for this
3181 		 * trace_array, we use break to jump to the next
3182 		 * trace_array.
3183 		 */
3184 		break;
3185 	} while_for_each_event_file();
3186 
3187 	__trace_remove_event_call(call);
3188 
3189 	return 0;
3190  busy:
3191 	/* No need to clear the trace now */
3192 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
3193 		tr->clear_trace = false;
3194 	}
3195 	return -EBUSY;
3196 }
3197 
3198 /* Remove an event_call */
trace_remove_event_call(struct trace_event_call * call)3199 int trace_remove_event_call(struct trace_event_call *call)
3200 {
3201 	int ret;
3202 
3203 	lockdep_assert_held(&event_mutex);
3204 
3205 	mutex_lock(&trace_types_lock);
3206 	down_write(&trace_event_sem);
3207 	ret = probe_remove_event_call(call);
3208 	up_write(&trace_event_sem);
3209 	mutex_unlock(&trace_types_lock);
3210 
3211 	return ret;
3212 }
3213 EXPORT_SYMBOL_GPL(trace_remove_event_call);
3214 
3215 #define for_each_event(event, start, end)			\
3216 	for (event = start;					\
3217 	     (unsigned long)event < (unsigned long)end;		\
3218 	     event++)
3219 
3220 #ifdef CONFIG_MODULES
3221 
trace_module_add_events(struct module * mod)3222 static void trace_module_add_events(struct module *mod)
3223 {
3224 	struct trace_event_call **call, **start, **end;
3225 
3226 	if (!mod->num_trace_events)
3227 		return;
3228 
3229 	/* Don't add infrastructure for mods without tracepoints */
3230 	if (trace_module_has_bad_taint(mod)) {
3231 		pr_err("%s: module has bad taint, not creating trace events\n",
3232 		       mod->name);
3233 		return;
3234 	}
3235 
3236 	start = mod->trace_events;
3237 	end = mod->trace_events + mod->num_trace_events;
3238 
3239 	for_each_event(call, start, end) {
3240 		__register_event(*call, mod);
3241 		__add_event_to_tracers(*call);
3242 	}
3243 }
3244 
trace_module_remove_events(struct module * mod)3245 static void trace_module_remove_events(struct module *mod)
3246 {
3247 	struct trace_event_call *call, *p;
3248 	struct module_string *modstr, *m;
3249 
3250 	down_write(&trace_event_sem);
3251 	list_for_each_entry_safe(call, p, &ftrace_events, list) {
3252 		if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3253 			continue;
3254 		if (call->module == mod)
3255 			__trace_remove_event_call(call);
3256 	}
3257 	/* Check for any strings allocade for this module */
3258 	list_for_each_entry_safe(modstr, m, &module_strings, next) {
3259 		if (modstr->module != mod)
3260 			continue;
3261 		list_del(&modstr->next);
3262 		kfree(modstr->str);
3263 		kfree(modstr);
3264 	}
3265 	up_write(&trace_event_sem);
3266 
3267 	/*
3268 	 * It is safest to reset the ring buffer if the module being unloaded
3269 	 * registered any events that were used. The only worry is if
3270 	 * a new module gets loaded, and takes on the same id as the events
3271 	 * of this module. When printing out the buffer, traced events left
3272 	 * over from this module may be passed to the new module events and
3273 	 * unexpected results may occur.
3274 	 */
3275 	tracing_reset_all_online_cpus_unlocked();
3276 }
3277 
trace_module_notify(struct notifier_block * self,unsigned long val,void * data)3278 static int trace_module_notify(struct notifier_block *self,
3279 			       unsigned long val, void *data)
3280 {
3281 	struct module *mod = data;
3282 
3283 	mutex_lock(&event_mutex);
3284 	mutex_lock(&trace_types_lock);
3285 	switch (val) {
3286 	case MODULE_STATE_COMING:
3287 		trace_module_add_events(mod);
3288 		break;
3289 	case MODULE_STATE_GOING:
3290 		trace_module_remove_events(mod);
3291 		break;
3292 	}
3293 	mutex_unlock(&trace_types_lock);
3294 	mutex_unlock(&event_mutex);
3295 
3296 	return NOTIFY_OK;
3297 }
3298 
3299 static struct notifier_block trace_module_nb = {
3300 	.notifier_call = trace_module_notify,
3301 	.priority = 1, /* higher than trace.c module notify */
3302 };
3303 #endif /* CONFIG_MODULES */
3304 
3305 /* Create a new event directory structure for a trace directory. */
3306 static void
__trace_add_event_dirs(struct trace_array * tr)3307 __trace_add_event_dirs(struct trace_array *tr)
3308 {
3309 	struct trace_event_call *call;
3310 	int ret;
3311 
3312 	list_for_each_entry(call, &ftrace_events, list) {
3313 		ret = __trace_add_new_event(call, tr);
3314 		if (ret < 0)
3315 			pr_warn("Could not create directory for event %s\n",
3316 				trace_event_name(call));
3317 	}
3318 }
3319 
3320 /* Returns any file that matches the system and event */
3321 struct trace_event_file *
__find_event_file(struct trace_array * tr,const char * system,const char * event)3322 __find_event_file(struct trace_array *tr, const char *system, const char *event)
3323 {
3324 	struct trace_event_file *file;
3325 	struct trace_event_call *call;
3326 	const char *name;
3327 
3328 	list_for_each_entry(file, &tr->events, list) {
3329 
3330 		call = file->event_call;
3331 		name = trace_event_name(call);
3332 
3333 		if (!name || !call->class)
3334 			continue;
3335 
3336 		if (strcmp(event, name) == 0 &&
3337 		    strcmp(system, call->class->system) == 0)
3338 			return file;
3339 	}
3340 	return NULL;
3341 }
3342 
3343 /* Returns valid trace event files that match system and event */
3344 struct trace_event_file *
find_event_file(struct trace_array * tr,const char * system,const char * event)3345 find_event_file(struct trace_array *tr, const char *system, const char *event)
3346 {
3347 	struct trace_event_file *file;
3348 
3349 	file = __find_event_file(tr, system, event);
3350 	if (!file || !file->event_call->class->reg ||
3351 	    file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3352 		return NULL;
3353 
3354 	return file;
3355 }
3356 
3357 /**
3358  * trace_get_event_file - Find and return a trace event file
3359  * @instance: The name of the trace instance containing the event
3360  * @system: The name of the system containing the event
3361  * @event: The name of the event
3362  *
3363  * Return a trace event file given the trace instance name, trace
3364  * system, and trace event name.  If the instance name is NULL, it
3365  * refers to the top-level trace array.
3366  *
3367  * This function will look it up and return it if found, after calling
3368  * trace_array_get() to prevent the instance from going away, and
3369  * increment the event's module refcount to prevent it from being
3370  * removed.
3371  *
3372  * To release the file, call trace_put_event_file(), which will call
3373  * trace_array_put() and decrement the event's module refcount.
3374  *
3375  * Return: The trace event on success, ERR_PTR otherwise.
3376  */
trace_get_event_file(const char * instance,const char * system,const char * event)3377 struct trace_event_file *trace_get_event_file(const char *instance,
3378 					      const char *system,
3379 					      const char *event)
3380 {
3381 	struct trace_array *tr = top_trace_array();
3382 	struct trace_event_file *file = NULL;
3383 	int ret = -EINVAL;
3384 
3385 	if (instance) {
3386 		tr = trace_array_find_get(instance);
3387 		if (!tr)
3388 			return ERR_PTR(-ENOENT);
3389 	} else {
3390 		ret = trace_array_get(tr);
3391 		if (ret)
3392 			return ERR_PTR(ret);
3393 	}
3394 
3395 	mutex_lock(&event_mutex);
3396 
3397 	file = find_event_file(tr, system, event);
3398 	if (!file) {
3399 		trace_array_put(tr);
3400 		ret = -EINVAL;
3401 		goto out;
3402 	}
3403 
3404 	/* Don't let event modules unload while in use */
3405 	ret = trace_event_try_get_ref(file->event_call);
3406 	if (!ret) {
3407 		trace_array_put(tr);
3408 		ret = -EBUSY;
3409 		goto out;
3410 	}
3411 
3412 	ret = 0;
3413  out:
3414 	mutex_unlock(&event_mutex);
3415 
3416 	if (ret)
3417 		file = ERR_PTR(ret);
3418 
3419 	return file;
3420 }
3421 EXPORT_SYMBOL_GPL(trace_get_event_file);
3422 
3423 /**
3424  * trace_put_event_file - Release a file from trace_get_event_file()
3425  * @file: The trace event file
3426  *
3427  * If a file was retrieved using trace_get_event_file(), this should
3428  * be called when it's no longer needed.  It will cancel the previous
3429  * trace_array_get() called by that function, and decrement the
3430  * event's module refcount.
3431  */
trace_put_event_file(struct trace_event_file * file)3432 void trace_put_event_file(struct trace_event_file *file)
3433 {
3434 	mutex_lock(&event_mutex);
3435 	trace_event_put_ref(file->event_call);
3436 	mutex_unlock(&event_mutex);
3437 
3438 	trace_array_put(file->tr);
3439 }
3440 EXPORT_SYMBOL_GPL(trace_put_event_file);
3441 
3442 #ifdef CONFIG_DYNAMIC_FTRACE
3443 
3444 /* Avoid typos */
3445 #define ENABLE_EVENT_STR	"enable_event"
3446 #define DISABLE_EVENT_STR	"disable_event"
3447 
3448 struct event_probe_data {
3449 	struct trace_event_file	*file;
3450 	unsigned long			count;
3451 	int				ref;
3452 	bool				enable;
3453 };
3454 
update_event_probe(struct event_probe_data * data)3455 static void update_event_probe(struct event_probe_data *data)
3456 {
3457 	if (data->enable)
3458 		clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3459 	else
3460 		set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3461 }
3462 
3463 static void
event_enable_probe(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)3464 event_enable_probe(unsigned long ip, unsigned long parent_ip,
3465 		   struct trace_array *tr, struct ftrace_probe_ops *ops,
3466 		   void *data)
3467 {
3468 	struct ftrace_func_mapper *mapper = data;
3469 	struct event_probe_data *edata;
3470 	void **pdata;
3471 
3472 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3473 	if (!pdata || !*pdata)
3474 		return;
3475 
3476 	edata = *pdata;
3477 	update_event_probe(edata);
3478 }
3479 
3480 static void
event_enable_count_probe(unsigned long ip,unsigned long parent_ip,struct trace_array * tr,struct ftrace_probe_ops * ops,void * data)3481 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
3482 			 struct trace_array *tr, struct ftrace_probe_ops *ops,
3483 			 void *data)
3484 {
3485 	struct ftrace_func_mapper *mapper = data;
3486 	struct event_probe_data *edata;
3487 	void **pdata;
3488 
3489 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3490 	if (!pdata || !*pdata)
3491 		return;
3492 
3493 	edata = *pdata;
3494 
3495 	if (!edata->count)
3496 		return;
3497 
3498 	/* Skip if the event is in a state we want to switch to */
3499 	if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
3500 		return;
3501 
3502 	if (edata->count != -1)
3503 		(edata->count)--;
3504 
3505 	update_event_probe(edata);
3506 }
3507 
3508 static int
event_enable_print(struct seq_file * m,unsigned long ip,struct ftrace_probe_ops * ops,void * data)3509 event_enable_print(struct seq_file *m, unsigned long ip,
3510 		   struct ftrace_probe_ops *ops, void *data)
3511 {
3512 	struct ftrace_func_mapper *mapper = data;
3513 	struct event_probe_data *edata;
3514 	void **pdata;
3515 
3516 	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3517 
3518 	if (WARN_ON_ONCE(!pdata || !*pdata))
3519 		return 0;
3520 
3521 	edata = *pdata;
3522 
3523 	seq_printf(m, "%ps:", (void *)ip);
3524 
3525 	seq_printf(m, "%s:%s:%s",
3526 		   edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
3527 		   edata->file->event_call->class->system,
3528 		   trace_event_name(edata->file->event_call));
3529 
3530 	if (edata->count == -1)
3531 		seq_puts(m, ":unlimited\n");
3532 	else
3533 		seq_printf(m, ":count=%ld\n", edata->count);
3534 
3535 	return 0;
3536 }
3537 
3538 static int
event_enable_init(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * init_data,void ** data)3539 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
3540 		  unsigned long ip, void *init_data, void **data)
3541 {
3542 	struct ftrace_func_mapper *mapper = *data;
3543 	struct event_probe_data *edata = init_data;
3544 	int ret;
3545 
3546 	if (!mapper) {
3547 		mapper = allocate_ftrace_func_mapper();
3548 		if (!mapper)
3549 			return -ENODEV;
3550 		*data = mapper;
3551 	}
3552 
3553 	ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
3554 	if (ret < 0)
3555 		return ret;
3556 
3557 	edata->ref++;
3558 
3559 	return 0;
3560 }
3561 
free_probe_data(void * data)3562 static int free_probe_data(void *data)
3563 {
3564 	struct event_probe_data *edata = data;
3565 
3566 	edata->ref--;
3567 	if (!edata->ref) {
3568 		/* Remove the SOFT_MODE flag */
3569 		__ftrace_event_enable_disable(edata->file, 0, 1);
3570 		trace_event_put_ref(edata->file->event_call);
3571 		kfree(edata);
3572 	}
3573 	return 0;
3574 }
3575 
3576 static void
event_enable_free(struct ftrace_probe_ops * ops,struct trace_array * tr,unsigned long ip,void * data)3577 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
3578 		  unsigned long ip, void *data)
3579 {
3580 	struct ftrace_func_mapper *mapper = data;
3581 	struct event_probe_data *edata;
3582 
3583 	if (!ip) {
3584 		if (!mapper)
3585 			return;
3586 		free_ftrace_func_mapper(mapper, free_probe_data);
3587 		return;
3588 	}
3589 
3590 	edata = ftrace_func_mapper_remove_ip(mapper, ip);
3591 
3592 	if (WARN_ON_ONCE(!edata))
3593 		return;
3594 
3595 	if (WARN_ON_ONCE(edata->ref <= 0))
3596 		return;
3597 
3598 	free_probe_data(edata);
3599 }
3600 
3601 static struct ftrace_probe_ops event_enable_probe_ops = {
3602 	.func			= event_enable_probe,
3603 	.print			= event_enable_print,
3604 	.init			= event_enable_init,
3605 	.free			= event_enable_free,
3606 };
3607 
3608 static struct ftrace_probe_ops event_enable_count_probe_ops = {
3609 	.func			= event_enable_count_probe,
3610 	.print			= event_enable_print,
3611 	.init			= event_enable_init,
3612 	.free			= event_enable_free,
3613 };
3614 
3615 static struct ftrace_probe_ops event_disable_probe_ops = {
3616 	.func			= event_enable_probe,
3617 	.print			= event_enable_print,
3618 	.init			= event_enable_init,
3619 	.free			= event_enable_free,
3620 };
3621 
3622 static struct ftrace_probe_ops event_disable_count_probe_ops = {
3623 	.func			= event_enable_count_probe,
3624 	.print			= event_enable_print,
3625 	.init			= event_enable_init,
3626 	.free			= event_enable_free,
3627 };
3628 
3629 static int
event_enable_func(struct trace_array * tr,struct ftrace_hash * hash,char * glob,char * cmd,char * param,int enabled)3630 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
3631 		  char *glob, char *cmd, char *param, int enabled)
3632 {
3633 	struct trace_event_file *file;
3634 	struct ftrace_probe_ops *ops;
3635 	struct event_probe_data *data;
3636 	const char *system;
3637 	const char *event;
3638 	char *number;
3639 	bool enable;
3640 	int ret;
3641 
3642 	if (!tr)
3643 		return -ENODEV;
3644 
3645 	/* hash funcs only work with set_ftrace_filter */
3646 	if (!enabled || !param)
3647 		return -EINVAL;
3648 
3649 	system = strsep(&param, ":");
3650 	if (!param)
3651 		return -EINVAL;
3652 
3653 	event = strsep(&param, ":");
3654 
3655 	mutex_lock(&event_mutex);
3656 
3657 	ret = -EINVAL;
3658 	file = find_event_file(tr, system, event);
3659 	if (!file)
3660 		goto out;
3661 
3662 	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
3663 
3664 	if (enable)
3665 		ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
3666 	else
3667 		ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
3668 
3669 	if (glob[0] == '!') {
3670 		ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);
3671 		goto out;
3672 	}
3673 
3674 	ret = -ENOMEM;
3675 
3676 	data = kzalloc(sizeof(*data), GFP_KERNEL);
3677 	if (!data)
3678 		goto out;
3679 
3680 	data->enable = enable;
3681 	data->count = -1;
3682 	data->file = file;
3683 
3684 	if (!param)
3685 		goto out_reg;
3686 
3687 	number = strsep(&param, ":");
3688 
3689 	ret = -EINVAL;
3690 	if (!strlen(number))
3691 		goto out_free;
3692 
3693 	/*
3694 	 * We use the callback data field (which is a pointer)
3695 	 * as our counter.
3696 	 */
3697 	ret = kstrtoul(number, 0, &data->count);
3698 	if (ret)
3699 		goto out_free;
3700 
3701  out_reg:
3702 	/* Don't let event modules unload while probe registered */
3703 	ret = trace_event_try_get_ref(file->event_call);
3704 	if (!ret) {
3705 		ret = -EBUSY;
3706 		goto out_free;
3707 	}
3708 
3709 	ret = __ftrace_event_enable_disable(file, 1, 1);
3710 	if (ret < 0)
3711 		goto out_put;
3712 
3713 	ret = register_ftrace_function_probe(glob, tr, ops, data);
3714 	/*
3715 	 * The above returns on success the # of functions enabled,
3716 	 * but if it didn't find any functions it returns zero.
3717 	 * Consider no functions a failure too.
3718 	 */
3719 	if (!ret) {
3720 		ret = -ENOENT;
3721 		goto out_disable;
3722 	} else if (ret < 0)
3723 		goto out_disable;
3724 	/* Just return zero, not the number of enabled functions */
3725 	ret = 0;
3726  out:
3727 	mutex_unlock(&event_mutex);
3728 	return ret;
3729 
3730  out_disable:
3731 	__ftrace_event_enable_disable(file, 0, 1);
3732  out_put:
3733 	trace_event_put_ref(file->event_call);
3734  out_free:
3735 	kfree(data);
3736 	goto out;
3737 }
3738 
3739 static struct ftrace_func_command event_enable_cmd = {
3740 	.name			= ENABLE_EVENT_STR,
3741 	.func			= event_enable_func,
3742 };
3743 
3744 static struct ftrace_func_command event_disable_cmd = {
3745 	.name			= DISABLE_EVENT_STR,
3746 	.func			= event_enable_func,
3747 };
3748 
register_event_cmds(void)3749 static __init int register_event_cmds(void)
3750 {
3751 	int ret;
3752 
3753 	ret = register_ftrace_command(&event_enable_cmd);
3754 	if (WARN_ON(ret < 0))
3755 		return ret;
3756 	ret = register_ftrace_command(&event_disable_cmd);
3757 	if (WARN_ON(ret < 0))
3758 		unregister_ftrace_command(&event_enable_cmd);
3759 	return ret;
3760 }
3761 #else
register_event_cmds(void)3762 static inline int register_event_cmds(void) { return 0; }
3763 #endif /* CONFIG_DYNAMIC_FTRACE */
3764 
3765 /*
3766  * The top level array and trace arrays created by boot-time tracing
3767  * have already had its trace_event_file descriptors created in order
3768  * to allow for early events to be recorded.
3769  * This function is called after the tracefs has been initialized,
3770  * and we now have to create the files associated to the events.
3771  */
__trace_early_add_event_dirs(struct trace_array * tr)3772 static void __trace_early_add_event_dirs(struct trace_array *tr)
3773 {
3774 	struct trace_event_file *file;
3775 	int ret;
3776 
3777 
3778 	list_for_each_entry(file, &tr->events, list) {
3779 		ret = event_create_dir(tr->event_dir, file);
3780 		if (ret < 0)
3781 			pr_warn("Could not create directory for event %s\n",
3782 				trace_event_name(file->event_call));
3783 	}
3784 }
3785 
3786 /*
3787  * For early boot up, the top trace array and the trace arrays created
3788  * by boot-time tracing require to have a list of events that can be
3789  * enabled. This must be done before the filesystem is set up in order
3790  * to allow events to be traced early.
3791  */
__trace_early_add_events(struct trace_array * tr)3792 void __trace_early_add_events(struct trace_array *tr)
3793 {
3794 	struct trace_event_call *call;
3795 	int ret;
3796 
3797 	list_for_each_entry(call, &ftrace_events, list) {
3798 		/* Early boot up should not have any modules loaded */
3799 		if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
3800 		    WARN_ON_ONCE(call->module))
3801 			continue;
3802 
3803 		ret = __trace_early_add_new_event(call, tr);
3804 		if (ret < 0)
3805 			pr_warn("Could not create early event %s\n",
3806 				trace_event_name(call));
3807 	}
3808 }
3809 
3810 /* Remove the event directory structure for a trace directory. */
3811 static void
__trace_remove_event_dirs(struct trace_array * tr)3812 __trace_remove_event_dirs(struct trace_array *tr)
3813 {
3814 	struct trace_event_file *file, *next;
3815 
3816 	list_for_each_entry_safe(file, next, &tr->events, list)
3817 		remove_event_file_dir(file);
3818 }
3819 
__add_event_to_tracers(struct trace_event_call * call)3820 static void __add_event_to_tracers(struct trace_event_call *call)
3821 {
3822 	struct trace_array *tr;
3823 
3824 	list_for_each_entry(tr, &ftrace_trace_arrays, list)
3825 		__trace_add_new_event(call, tr);
3826 }
3827 
3828 extern struct trace_event_call *__start_ftrace_events[];
3829 extern struct trace_event_call *__stop_ftrace_events[];
3830 
3831 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
3832 
setup_trace_event(char * str)3833 static __init int setup_trace_event(char *str)
3834 {
3835 	strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
3836 	trace_set_ring_buffer_expanded(NULL);
3837 	disable_tracing_selftest("running event tracing");
3838 
3839 	return 1;
3840 }
3841 __setup("trace_event=", setup_trace_event);
3842 
events_callback(const char * name,umode_t * mode,void ** data,const struct file_operations ** fops)3843 static int events_callback(const char *name, umode_t *mode, void **data,
3844 			   const struct file_operations **fops)
3845 {
3846 	if (strcmp(name, "enable") == 0) {
3847 		*mode = TRACE_MODE_WRITE;
3848 		*fops = &ftrace_tr_enable_fops;
3849 		return 1;
3850 	}
3851 
3852 	if (strcmp(name, "header_page") == 0) {
3853 		*mode = TRACE_MODE_READ;
3854 		*fops = &ftrace_show_header_page_fops;
3855 
3856 	} else if (strcmp(name, "header_event") == 0) {
3857 		*mode = TRACE_MODE_READ;
3858 		*fops = &ftrace_show_header_event_fops;
3859 	} else
3860 		return 0;
3861 
3862 	return 1;
3863 }
3864 
3865 /* Expects to have event_mutex held when called */
3866 static int
create_event_toplevel_files(struct dentry * parent,struct trace_array * tr)3867 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
3868 {
3869 	struct eventfs_inode *e_events;
3870 	struct dentry *entry;
3871 	int nr_entries;
3872 	static struct eventfs_entry events_entries[] = {
3873 		{
3874 			.name		= "enable",
3875 			.callback	= events_callback,
3876 		},
3877 		{
3878 			.name		= "header_page",
3879 			.callback	= events_callback,
3880 		},
3881 		{
3882 			.name		= "header_event",
3883 			.callback	= events_callback,
3884 		},
3885 	};
3886 
3887 	entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
3888 				  tr, &ftrace_set_event_fops);
3889 	if (!entry)
3890 		return -ENOMEM;
3891 
3892 	nr_entries = ARRAY_SIZE(events_entries);
3893 
3894 	e_events = eventfs_create_events_dir("events", parent, events_entries,
3895 					     nr_entries, tr);
3896 	if (IS_ERR(e_events)) {
3897 		pr_warn("Could not create tracefs 'events' directory\n");
3898 		return -ENOMEM;
3899 	}
3900 
3901 	/* There are not as crucial, just warn if they are not created */
3902 
3903 	trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
3904 			  tr, &ftrace_set_event_pid_fops);
3905 
3906 	trace_create_file("set_event_notrace_pid",
3907 			  TRACE_MODE_WRITE, parent, tr,
3908 			  &ftrace_set_event_notrace_pid_fops);
3909 
3910 	tr->event_dir = e_events;
3911 
3912 	return 0;
3913 }
3914 
3915 /**
3916  * event_trace_add_tracer - add a instance of a trace_array to events
3917  * @parent: The parent dentry to place the files/directories for events in
3918  * @tr: The trace array associated with these events
3919  *
3920  * When a new instance is created, it needs to set up its events
3921  * directory, as well as other files associated with events. It also
3922  * creates the event hierarchy in the @parent/events directory.
3923  *
3924  * Returns 0 on success.
3925  *
3926  * Must be called with event_mutex held.
3927  */
event_trace_add_tracer(struct dentry * parent,struct trace_array * tr)3928 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
3929 {
3930 	int ret;
3931 
3932 	lockdep_assert_held(&event_mutex);
3933 
3934 	ret = create_event_toplevel_files(parent, tr);
3935 	if (ret)
3936 		goto out;
3937 
3938 	down_write(&trace_event_sem);
3939 	/* If tr already has the event list, it is initialized in early boot. */
3940 	if (unlikely(!list_empty(&tr->events)))
3941 		__trace_early_add_event_dirs(tr);
3942 	else
3943 		__trace_add_event_dirs(tr);
3944 	up_write(&trace_event_sem);
3945 
3946  out:
3947 	return ret;
3948 }
3949 
3950 /*
3951  * The top trace array already had its file descriptors created.
3952  * Now the files themselves need to be created.
3953  */
3954 static __init int
early_event_add_tracer(struct dentry * parent,struct trace_array * tr)3955 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
3956 {
3957 	int ret;
3958 
3959 	mutex_lock(&event_mutex);
3960 
3961 	ret = create_event_toplevel_files(parent, tr);
3962 	if (ret)
3963 		goto out_unlock;
3964 
3965 	down_write(&trace_event_sem);
3966 	__trace_early_add_event_dirs(tr);
3967 	up_write(&trace_event_sem);
3968 
3969  out_unlock:
3970 	mutex_unlock(&event_mutex);
3971 
3972 	return ret;
3973 }
3974 
3975 /* Must be called with event_mutex held */
event_trace_del_tracer(struct trace_array * tr)3976 int event_trace_del_tracer(struct trace_array *tr)
3977 {
3978 	lockdep_assert_held(&event_mutex);
3979 
3980 	/* Disable any event triggers and associated soft-disabled events */
3981 	clear_event_triggers(tr);
3982 
3983 	/* Clear the pid list */
3984 	__ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
3985 
3986 	/* Disable any running events */
3987 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
3988 
3989 	/* Make sure no more events are being executed */
3990 	tracepoint_synchronize_unregister();
3991 
3992 	down_write(&trace_event_sem);
3993 	__trace_remove_event_dirs(tr);
3994 	eventfs_remove_events_dir(tr->event_dir);
3995 	up_write(&trace_event_sem);
3996 
3997 	tr->event_dir = NULL;
3998 
3999 	return 0;
4000 }
4001 
event_trace_memsetup(void)4002 static __init int event_trace_memsetup(void)
4003 {
4004 	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
4005 	file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
4006 	return 0;
4007 }
4008 
4009 __init void
early_enable_events(struct trace_array * tr,char * buf,bool disable_first)4010 early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
4011 {
4012 	char *token;
4013 	int ret;
4014 
4015 	while (true) {
4016 		token = strsep(&buf, ",");
4017 
4018 		if (!token)
4019 			break;
4020 
4021 		if (*token) {
4022 			/* Restarting syscalls requires that we stop them first */
4023 			if (disable_first)
4024 				ftrace_set_clr_event(tr, token, 0);
4025 
4026 			ret = ftrace_set_clr_event(tr, token, 1);
4027 			if (ret)
4028 				pr_warn("Failed to enable trace event: %s\n", token);
4029 		}
4030 
4031 		/* Put back the comma to allow this to be called again */
4032 		if (buf)
4033 			*(buf - 1) = ',';
4034 	}
4035 }
4036 
event_trace_enable(void)4037 static __init int event_trace_enable(void)
4038 {
4039 	struct trace_array *tr = top_trace_array();
4040 	struct trace_event_call **iter, *call;
4041 	int ret;
4042 
4043 	if (!tr)
4044 		return -ENODEV;
4045 
4046 	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
4047 
4048 		call = *iter;
4049 		ret = event_init(call);
4050 		if (!ret)
4051 			list_add(&call->list, &ftrace_events);
4052 	}
4053 
4054 	register_trigger_cmds();
4055 
4056 	/*
4057 	 * We need the top trace array to have a working set of trace
4058 	 * points at early init, before the debug files and directories
4059 	 * are created. Create the file entries now, and attach them
4060 	 * to the actual file dentries later.
4061 	 */
4062 	__trace_early_add_events(tr);
4063 
4064 	early_enable_events(tr, bootup_event_buf, false);
4065 
4066 	trace_printk_start_comm();
4067 
4068 	register_event_cmds();
4069 
4070 
4071 	return 0;
4072 }
4073 
4074 /*
4075  * event_trace_enable() is called from trace_event_init() first to
4076  * initialize events and perhaps start any events that are on the
4077  * command line. Unfortunately, there are some events that will not
4078  * start this early, like the system call tracepoints that need
4079  * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
4080  * event_trace_enable() is called before pid 1 starts, and this flag
4081  * is never set, making the syscall tracepoint never get reached, but
4082  * the event is enabled regardless (and not doing anything).
4083  */
event_trace_enable_again(void)4084 static __init int event_trace_enable_again(void)
4085 {
4086 	struct trace_array *tr;
4087 
4088 	tr = top_trace_array();
4089 	if (!tr)
4090 		return -ENODEV;
4091 
4092 	early_enable_events(tr, bootup_event_buf, true);
4093 
4094 	return 0;
4095 }
4096 
4097 early_initcall(event_trace_enable_again);
4098 
4099 /* Init fields which doesn't related to the tracefs */
event_trace_init_fields(void)4100 static __init int event_trace_init_fields(void)
4101 {
4102 	if (trace_define_generic_fields())
4103 		pr_warn("tracing: Failed to allocated generic fields");
4104 
4105 	if (trace_define_common_fields())
4106 		pr_warn("tracing: Failed to allocate common fields");
4107 
4108 	return 0;
4109 }
4110 
event_trace_init(void)4111 __init int event_trace_init(void)
4112 {
4113 	struct trace_array *tr;
4114 	int ret;
4115 
4116 	tr = top_trace_array();
4117 	if (!tr)
4118 		return -ENODEV;
4119 
4120 	trace_create_file("available_events", TRACE_MODE_READ,
4121 			  NULL, tr, &ftrace_avail_fops);
4122 
4123 	ret = early_event_add_tracer(NULL, tr);
4124 	if (ret)
4125 		return ret;
4126 
4127 #ifdef CONFIG_MODULES
4128 	ret = register_module_notifier(&trace_module_nb);
4129 	if (ret)
4130 		pr_warn("Failed to register trace events module notifier\n");
4131 #endif
4132 
4133 	eventdir_initialized = true;
4134 
4135 	return 0;
4136 }
4137 
trace_event_init(void)4138 void __init trace_event_init(void)
4139 {
4140 	event_trace_memsetup();
4141 	init_ftrace_syscalls();
4142 	event_trace_enable();
4143 	event_trace_init_fields();
4144 }
4145 
4146 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4147 
4148 static DEFINE_SPINLOCK(test_spinlock);
4149 static DEFINE_SPINLOCK(test_spinlock_irq);
4150 static DEFINE_MUTEX(test_mutex);
4151 
test_work(struct work_struct * dummy)4152 static __init void test_work(struct work_struct *dummy)
4153 {
4154 	spin_lock(&test_spinlock);
4155 	spin_lock_irq(&test_spinlock_irq);
4156 	udelay(1);
4157 	spin_unlock_irq(&test_spinlock_irq);
4158 	spin_unlock(&test_spinlock);
4159 
4160 	mutex_lock(&test_mutex);
4161 	msleep(1);
4162 	mutex_unlock(&test_mutex);
4163 }
4164 
event_test_thread(void * unused)4165 static __init int event_test_thread(void *unused)
4166 {
4167 	void *test_malloc;
4168 
4169 	test_malloc = kmalloc(1234, GFP_KERNEL);
4170 	if (!test_malloc)
4171 		pr_info("failed to kmalloc\n");
4172 
4173 	schedule_on_each_cpu(test_work);
4174 
4175 	kfree(test_malloc);
4176 
4177 	set_current_state(TASK_INTERRUPTIBLE);
4178 	while (!kthread_should_stop()) {
4179 		schedule();
4180 		set_current_state(TASK_INTERRUPTIBLE);
4181 	}
4182 	__set_current_state(TASK_RUNNING);
4183 
4184 	return 0;
4185 }
4186 
4187 /*
4188  * Do various things that may trigger events.
4189  */
event_test_stuff(void)4190 static __init void event_test_stuff(void)
4191 {
4192 	struct task_struct *test_thread;
4193 
4194 	test_thread = kthread_run(event_test_thread, NULL, "test-events");
4195 	msleep(1);
4196 	kthread_stop(test_thread);
4197 }
4198 
4199 /*
4200  * For every trace event defined, we will test each trace point separately,
4201  * and then by groups, and finally all trace points.
4202  */
event_trace_self_tests(void)4203 static __init void event_trace_self_tests(void)
4204 {
4205 	struct trace_subsystem_dir *dir;
4206 	struct trace_event_file *file;
4207 	struct trace_event_call *call;
4208 	struct event_subsystem *system;
4209 	struct trace_array *tr;
4210 	int ret;
4211 
4212 	tr = top_trace_array();
4213 	if (!tr)
4214 		return;
4215 
4216 	pr_info("Running tests on trace events:\n");
4217 
4218 	list_for_each_entry(file, &tr->events, list) {
4219 
4220 		call = file->event_call;
4221 
4222 		/* Only test those that have a probe */
4223 		if (!call->class || !call->class->probe)
4224 			continue;
4225 
4226 /*
4227  * Testing syscall events here is pretty useless, but
4228  * we still do it if configured. But this is time consuming.
4229  * What we really need is a user thread to perform the
4230  * syscalls as we test.
4231  */
4232 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
4233 		if (call->class->system &&
4234 		    strcmp(call->class->system, "syscalls") == 0)
4235 			continue;
4236 #endif
4237 
4238 		pr_info("Testing event %s: ", trace_event_name(call));
4239 
4240 		/*
4241 		 * If an event is already enabled, someone is using
4242 		 * it and the self test should not be on.
4243 		 */
4244 		if (file->flags & EVENT_FILE_FL_ENABLED) {
4245 			pr_warn("Enabled event during self test!\n");
4246 			WARN_ON_ONCE(1);
4247 			continue;
4248 		}
4249 
4250 		ftrace_event_enable_disable(file, 1);
4251 		event_test_stuff();
4252 		ftrace_event_enable_disable(file, 0);
4253 
4254 		pr_cont("OK\n");
4255 	}
4256 
4257 	/* Now test at the sub system level */
4258 
4259 	pr_info("Running tests on trace event systems:\n");
4260 
4261 	list_for_each_entry(dir, &tr->systems, list) {
4262 
4263 		system = dir->subsystem;
4264 
4265 		/* the ftrace system is special, skip it */
4266 		if (strcmp(system->name, "ftrace") == 0)
4267 			continue;
4268 
4269 		pr_info("Testing event system %s: ", system->name);
4270 
4271 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
4272 		if (WARN_ON_ONCE(ret)) {
4273 			pr_warn("error enabling system %s\n",
4274 				system->name);
4275 			continue;
4276 		}
4277 
4278 		event_test_stuff();
4279 
4280 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
4281 		if (WARN_ON_ONCE(ret)) {
4282 			pr_warn("error disabling system %s\n",
4283 				system->name);
4284 			continue;
4285 		}
4286 
4287 		pr_cont("OK\n");
4288 	}
4289 
4290 	/* Test with all events enabled */
4291 
4292 	pr_info("Running tests on all trace events:\n");
4293 	pr_info("Testing all events: ");
4294 
4295 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
4296 	if (WARN_ON_ONCE(ret)) {
4297 		pr_warn("error enabling all events\n");
4298 		return;
4299 	}
4300 
4301 	event_test_stuff();
4302 
4303 	/* reset sysname */
4304 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
4305 	if (WARN_ON_ONCE(ret)) {
4306 		pr_warn("error disabling all events\n");
4307 		return;
4308 	}
4309 
4310 	pr_cont("OK\n");
4311 }
4312 
4313 #ifdef CONFIG_FUNCTION_TRACER
4314 
4315 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4316 
4317 static struct trace_event_file event_trace_file __initdata;
4318 
4319 static void __init
function_test_events_call(unsigned long ip,unsigned long parent_ip,struct ftrace_ops * op,struct ftrace_regs * regs)4320 function_test_events_call(unsigned long ip, unsigned long parent_ip,
4321 			  struct ftrace_ops *op, struct ftrace_regs *regs)
4322 {
4323 	struct trace_buffer *buffer;
4324 	struct ring_buffer_event *event;
4325 	struct ftrace_entry *entry;
4326 	unsigned int trace_ctx;
4327 	long disabled;
4328 	int cpu;
4329 
4330 	trace_ctx = tracing_gen_ctx();
4331 	preempt_disable_notrace();
4332 	cpu = raw_smp_processor_id();
4333 	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4334 
4335 	if (disabled != 1)
4336 		goto out;
4337 
4338 	event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4339 						TRACE_FN, sizeof(*entry),
4340 						trace_ctx);
4341 	if (!event)
4342 		goto out;
4343 	entry	= ring_buffer_event_data(event);
4344 	entry->ip			= ip;
4345 	entry->parent_ip		= parent_ip;
4346 
4347 	event_trigger_unlock_commit(&event_trace_file, buffer, event,
4348 				    entry, trace_ctx);
4349  out:
4350 	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4351 	preempt_enable_notrace();
4352 }
4353 
4354 static struct ftrace_ops trace_ops __initdata  =
4355 {
4356 	.func = function_test_events_call,
4357 };
4358 
event_trace_self_test_with_function(void)4359 static __init void event_trace_self_test_with_function(void)
4360 {
4361 	int ret;
4362 
4363 	event_trace_file.tr = top_trace_array();
4364 	if (WARN_ON(!event_trace_file.tr))
4365 		return;
4366 
4367 	ret = register_ftrace_function(&trace_ops);
4368 	if (WARN_ON(ret < 0)) {
4369 		pr_info("Failed to enable function tracer for event tests\n");
4370 		return;
4371 	}
4372 	pr_info("Running tests again, along with the function tracer\n");
4373 	event_trace_self_tests();
4374 	unregister_ftrace_function(&trace_ops);
4375 }
4376 #else
event_trace_self_test_with_function(void)4377 static __init void event_trace_self_test_with_function(void)
4378 {
4379 }
4380 #endif
4381 
event_trace_self_tests_init(void)4382 static __init int event_trace_self_tests_init(void)
4383 {
4384 	if (!tracing_selftest_disabled) {
4385 		event_trace_self_tests();
4386 		event_trace_self_test_with_function();
4387 	}
4388 
4389 	return 0;
4390 }
4391 
4392 late_initcall(event_trace_self_tests_init);
4393 
4394 #endif
4395