72 #define do_for_each_event_file(tr, file)			\  argument
73 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
74 		list_for_each_entry(file, &tr->events, list)
76 #define do_for_each_event_file_safe(tr, file)			\  argument
77 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
79 		list_for_each_entry_safe(file, ___n, &tr->events, list)
478 	struct trace_array *tr = trace_file->tr;  in trace_event_ignore_this_pid()  local
483 	pid_list = rcu_dereference_raw(tr->filtered_pids);  in trace_event_ignore_this_pid()
484 	no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);  in trace_event_ignore_this_pid()
489 	data = this_cpu_ptr(tr->array_buffer.data);  in trace_event_ignore_this_pid()
568 	struct trace_array *tr;  in trace_event_enable_cmd_record()  local
572 	do_for_each_event_file(tr, file) {  in trace_event_enable_cmd_record()
590 	struct trace_array *tr;  in trace_event_enable_tgid_record()  local
594 	do_for_each_event_file(tr, file) {  in trace_event_enable_tgid_record()
613 	struct trace_array *tr = file->tr;  in __ftrace_event_enable_disable()  local
688 			if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {  in __ftrace_event_enable_disable()
694 			if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {  in __ftrace_event_enable_disable()
733 static void ftrace_clear_events(struct trace_array *tr)  in ftrace_clear_events()  argument
738 	list_for_each_entry(file, &tr->events, list) {  in ftrace_clear_events()
748 	struct trace_array *tr = data;  in event_filter_pid_sched_process_exit()  local
750 	pid_list = rcu_dereference_raw(tr->filtered_pids);  in event_filter_pid_sched_process_exit()
753 	pid_list = rcu_dereference_raw(tr->filtered_no_pids);  in event_filter_pid_sched_process_exit()
763 	struct trace_array *tr = data;  in event_filter_pid_sched_process_fork()  local
765 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_process_fork()
768 	pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_process_fork()
772 void trace_event_follow_fork(struct trace_array *tr, bool enable)  in trace_event_follow_fork()  argument
776 						       tr, INT_MIN);  in trace_event_follow_fork()
778 						       tr, INT_MAX);  in trace_event_follow_fork()
781 						    tr);  in trace_event_follow_fork()
783 						    tr);  in trace_event_follow_fork()
793 	struct trace_array *tr = data;  in event_filter_pid_sched_switch_probe_pre()  local
798 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_switch_probe_pre()
799 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_switch_probe_pre()
808 	this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||  in event_filter_pid_sched_switch_probe_pre()
819 	struct trace_array *tr = data;  in event_filter_pid_sched_switch_probe_post()  local
823 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_switch_probe_post()
824 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_switch_probe_post()
826 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in event_filter_pid_sched_switch_probe_post()
833 	struct trace_array *tr = data;  in event_filter_pid_sched_wakeup_probe_pre()  local
838 	if (!this_cpu_read(tr->array_buffer.data->ignore_pid))  in event_filter_pid_sched_wakeup_probe_pre()
841 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_wakeup_probe_pre()
842 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_wakeup_probe_pre()
844 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in event_filter_pid_sched_wakeup_probe_pre()
851 	struct trace_array *tr = data;  in event_filter_pid_sched_wakeup_probe_post()  local
856 	if (this_cpu_read(tr->array_buffer.data->ignore_pid))  in event_filter_pid_sched_wakeup_probe_post()
859 	pid_list = rcu_dereference_sched(tr->filtered_pids);  in event_filter_pid_sched_wakeup_probe_post()
860 	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in event_filter_pid_sched_wakeup_probe_post()
863 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in event_filter_pid_sched_wakeup_probe_post()
867 static void unregister_pid_events(struct trace_array *tr)  in unregister_pid_events()  argument
869 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);  in unregister_pid_events()
870 	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);  in unregister_pid_events()
872 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);  in unregister_pid_events()
873 	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);  in unregister_pid_events()
875 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);  in unregister_pid_events()
876 	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);  in unregister_pid_events()
878 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);  in unregister_pid_events()
879 	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);  in unregister_pid_events()
882 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)  in __ftrace_clear_event_pids()  argument
889 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in __ftrace_clear_event_pids()
891 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,  in __ftrace_clear_event_pids()
899 		unregister_pid_events(tr);  in __ftrace_clear_event_pids()
901 		list_for_each_entry(file, &tr->events, list) {  in __ftrace_clear_event_pids()
906 			per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;  in __ftrace_clear_event_pids()
910 		rcu_assign_pointer(tr->filtered_pids, NULL);  in __ftrace_clear_event_pids()
913 		rcu_assign_pointer(tr->filtered_no_pids, NULL);  in __ftrace_clear_event_pids()
925 static void ftrace_clear_event_pids(struct trace_array *tr, int type)  in ftrace_clear_event_pids()  argument
928 	__ftrace_clear_event_pids(tr, type);  in ftrace_clear_event_pids()
1028 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,  in __ftrace_set_clr_event_nolock()  argument
1037 	list_for_each_entry(file, &tr->events, list) {  in __ftrace_set_clr_event_nolock()
1075 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,  in __ftrace_set_clr_event()  argument
1081 	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);  in __ftrace_set_clr_event()
1087 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)  in ftrace_set_clr_event()  argument
1092 	if (!tr)  in ftrace_set_clr_event()
1118 	ret = __ftrace_set_clr_event(tr, match, sub, event, set);  in ftrace_set_clr_event()
1141 	struct trace_array *tr = top_trace_array();  in trace_set_clr_event()  local
1143 	if (!tr)  in trace_set_clr_event()
1146 	return __ftrace_set_clr_event(tr, NULL, system, event, set);  in trace_set_clr_event()
1163 int trace_array_set_clr_event(struct trace_array *tr, const char *system,  in trace_array_set_clr_event()  argument
1168 	if (!tr)  in trace_array_set_clr_event()
1172 	return __ftrace_set_clr_event(tr, NULL, system, event, set);  in trace_array_set_clr_event()
1185 	struct trace_array *tr = m->private;  in ftrace_event_write()  local
1191 	ret = tracing_update_buffers(tr);  in ftrace_event_write()
1206 		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);  in ftrace_event_write()
1224 	struct trace_array *tr = m->private;  in t_next()  local
1228 	list_for_each_entry_continue(file, &tr->events, list) {  in t_next()
1245 	struct trace_array *tr = m->private;  in t_start()  local
1250 	file = list_entry(&tr->events, struct trace_event_file, list);  in t_start()
1263 	struct trace_array *tr = m->private;  in s_next()  local
1267 	list_for_each_entry_continue(file, &tr->events, list) {  in s_next()
1278 	struct trace_array *tr = m->private;  in s_start()  local
1283 	file = list_entry(&tr->events, struct trace_event_file, list);  in s_start()
1312 	struct trace_array *tr = m->private;  in __next()  local
1316 		pid_list = rcu_dereference_sched(tr->filtered_pids);  in __next()
1318 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in __next()
1339 	struct trace_array *tr = m->private;  in __start()  local
1351 		pid_list = rcu_dereference_sched(tr->filtered_pids);  in __start()
1353 		pid_list = rcu_dereference_sched(tr->filtered_no_pids);  in __start()
1429 			ret = tracing_update_buffers(file->tr);  in event_enable_write()
1457 	struct trace_array *tr = dir->tr;  in system_enable_read()  local
1463 	list_for_each_entry(file, &tr->events, list) {  in system_enable_read()
1509 	ret = tracing_update_buffers(dir->tr);  in system_enable_write()
1523 	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);  in system_enable_write()
1766 	struct trace_array *tr = NULL, *iter_tr;  in subsystem_open()  local
1780 				tr = iter_tr;  in subsystem_open()
1798 	if (trace_array_get(tr) < 0) {  in subsystem_open()
1805 		trace_array_put(tr);  in subsystem_open()
1815 	struct trace_array *tr = inode->i_private;  in system_tr_open()  local
1828 	dir->tr = tr;  in system_tr_open()
1838 	trace_array_put(dir->tr);  in subsystem_release()
1908 	struct trace_array *tr = filp->private_data;  in show_header_page_file()  local
1921 	ring_buffer_print_page_header(tr->array_buffer.buffer, s);  in show_header_page_file()
1956 	struct trace_array *tr = data;  in ignore_task_cpu()  local
1964 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in ignore_task_cpu()
1966 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,  in ignore_task_cpu()
1969 	this_cpu_write(tr->array_buffer.data->ignore_pid,  in ignore_task_cpu()
1973 static void register_pid_events(struct trace_array *tr)  in register_pid_events()  argument
1982 					 tr, INT_MAX);  in register_pid_events()
1984 					 tr, 0);  in register_pid_events()
1987 					 tr, INT_MAX);  in register_pid_events()
1989 					 tr, 0);  in register_pid_events()
1992 					     tr, INT_MAX);  in register_pid_events()
1994 					     tr, 0);  in register_pid_events()
1997 					 tr, INT_MAX);  in register_pid_events()
1999 					 tr, 0);  in register_pid_events()
2007 	struct trace_array *tr = m->private;  in event_pid_write()  local
2017 	ret = tracing_update_buffers(tr);  in event_pid_write()
2024 		filtered_pids = rcu_dereference_protected(tr->filtered_pids,  in event_pid_write()
2026 		other_pids = rcu_dereference_protected(tr->filtered_no_pids,  in event_pid_write()
2029 		filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,  in event_pid_write()
2031 		other_pids = rcu_dereference_protected(tr->filtered_pids,  in event_pid_write()
2040 		rcu_assign_pointer(tr->filtered_pids, pid_list);  in event_pid_write()
2042 		rcu_assign_pointer(tr->filtered_no_pids, pid_list);  in event_pid_write()
2044 	list_for_each_entry(file, &tr->events, list) {  in event_pid_write()
2052 		register_pid_events(tr);  in event_pid_write()
2060 	on_each_cpu(ignore_task_cpu, tr, 1);  in event_pid_write()
2241 	struct trace_array *tr = inode->i_private;  in ftrace_event_release()  local
2243 	trace_array_put(tr);  in ftrace_event_release()
2261 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_open()  local
2264 	ret = tracing_check_open_get_tr(tr);  in ftrace_event_set_open()
2270 		ftrace_clear_events(tr);  in ftrace_event_set_open()
2274 		trace_array_put(tr);  in ftrace_event_set_open()
2282 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_pid_open()  local
2285 	ret = tracing_check_open_get_tr(tr);  in ftrace_event_set_pid_open()
2291 		ftrace_clear_event_pids(tr, TRACE_PIDS);  in ftrace_event_set_pid_open()
2295 		trace_array_put(tr);  in ftrace_event_set_pid_open()
2303 	struct trace_array *tr = inode->i_private;  in ftrace_event_set_npid_open()  local
2306 	ret = tracing_check_open_get_tr(tr);  in ftrace_event_set_npid_open()
2312 		ftrace_clear_event_pids(tr, TRACE_NO_PIDS);  in ftrace_event_set_npid_open()
2316 		trace_array_put(tr);  in ftrace_event_set_npid_open()
2368 event_subsystem_dir(struct trace_array *tr, const char *name,  in event_subsystem_dir()  argument
2387 	list_for_each_entry(dir, &tr->systems, list) {  in event_subsystem_dir()
2430 	dir->tr = tr;  in event_subsystem_dir()
2436 	list_add(&dir->list, &tr->systems);  in event_subsystem_dir()
2574 	struct trace_array *tr = file->tr;  in event_create_dir()  local
2632 	e_events = event_subsystem_dir(tr, call->class->system, file, parent);  in event_create_dir()
2662 	struct trace_array *tr;  in remove_event_from_tracers()  local
2664 	do_for_each_event_file_safe(tr, file) {  in remove_event_from_tracers()
2681 	struct trace_array *tr;  in event_remove()  local
2684 	do_for_each_event_file(tr, file) {  in event_remove()
2689 			tr->clear_trace = true;  in event_remove()
2977 		       struct trace_array *tr)  in trace_create_new_event()  argument
2984 	if (!event_in_systems(call, tr->system_names))  in trace_create_new_event()
2991 	pid_list = rcu_dereference_protected(tr->filtered_pids,  in trace_create_new_event()
2993 	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,  in trace_create_new_event()
3001 	file->tr = tr;  in trace_create_new_event()
3005 	list_add(&file->list, &tr->events);  in trace_create_new_event()
3049 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)  in __trace_add_new_event()  argument
3053 	file = trace_create_new_event(call, tr);  in __trace_add_new_event()
3067 		return event_create_dir(tr->event_dir, file);  in __trace_add_new_event()
3097 			    struct trace_array *tr)  in __trace_early_add_new_event()  argument
3102 	file = trace_create_new_event(call, tr);  in __trace_early_add_new_event()
3158 	struct trace_array *tr;  in probe_remove_event_call()  local
3165 	do_for_each_event_file(tr, file) {  in probe_remove_event_call()
3177 			tr->clear_trace = true;  in probe_remove_event_call()
3192 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {  in probe_remove_event_call()
3193 		tr->clear_trace = false;  in probe_remove_event_call()
3307 __trace_add_event_dirs(struct trace_array *tr)  in __trace_add_event_dirs()  argument
3313 		ret = __trace_add_new_event(call, tr);  in __trace_add_event_dirs()
3322 __find_event_file(struct trace_array *tr, const char *system, const char *event)  in __find_event_file()  argument
3328 	list_for_each_entry(file, &tr->events, list) {  in __find_event_file()
3345 find_event_file(struct trace_array *tr, const char *system, const char *event)  in find_event_file()  argument
3349 	file = __find_event_file(tr, system, event);  in find_event_file()
3381 	struct trace_array *tr = top_trace_array();  in trace_get_event_file()  local
3386 		tr = trace_array_find_get(instance);  in trace_get_event_file()
3387 		if (!tr)  in trace_get_event_file()
3390 		ret = trace_array_get(tr);  in trace_get_event_file()
3397 	file = find_event_file(tr, system, event);  in trace_get_event_file()
3399 		trace_array_put(tr);  in trace_get_event_file()
3407 		trace_array_put(tr);  in trace_get_event_file()
3438 	trace_array_put(file->tr);  in trace_put_event_file()
3465 		   struct trace_array *tr, struct ftrace_probe_ops *ops,  in event_enable_probe()  argument
3482 			 struct trace_array *tr, struct ftrace_probe_ops *ops,  in event_enable_count_probe()  argument
3539 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,  in event_enable_init()  argument
3577 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,  in event_enable_free()  argument
3630 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,  in event_enable_func()  argument
3642 	if (!tr)  in event_enable_func()
3658 	file = find_event_file(tr, system, event);  in event_enable_func()
3670 		ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);  in event_enable_func()
3713 	ret = register_ftrace_function_probe(glob, tr, ops, data);  in event_enable_func()
3772 static void __trace_early_add_event_dirs(struct trace_array *tr)  in __trace_early_add_event_dirs()  argument
3778 	list_for_each_entry(file, &tr->events, list) {  in __trace_early_add_event_dirs()
3779 		ret = event_create_dir(tr->event_dir, file);  in __trace_early_add_event_dirs()
3792 void __trace_early_add_events(struct trace_array *tr)  in __trace_early_add_events()  argument
3803 		ret = __trace_early_add_new_event(call, tr);  in __trace_early_add_events()
3812 __trace_remove_event_dirs(struct trace_array *tr)  in __trace_remove_event_dirs()  argument
3816 	list_for_each_entry_safe(file, next, &tr->events, list)  in __trace_remove_event_dirs()
3822 	struct trace_array *tr;  in __add_event_to_tracers()  local
3824 	list_for_each_entry(tr, &ftrace_trace_arrays, list)  in __add_event_to_tracers()
3825 		__trace_add_new_event(call, tr);  in __add_event_to_tracers()
3867 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)  in create_event_toplevel_files()  argument
3888 				  tr, &ftrace_set_event_fops);  in create_event_toplevel_files()
3895 					     nr_entries, tr);  in create_event_toplevel_files()
3904 			  tr, &ftrace_set_event_pid_fops);  in create_event_toplevel_files()
3907 			  TRACE_MODE_WRITE, parent, tr,  in create_event_toplevel_files()
3910 	tr->event_dir = e_events;  in create_event_toplevel_files()
3928 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)  in event_trace_add_tracer()  argument
3934 	ret = create_event_toplevel_files(parent, tr);  in event_trace_add_tracer()
3940 	if (unlikely(!list_empty(&tr->events)))  in event_trace_add_tracer()
3941 		__trace_early_add_event_dirs(tr);  in event_trace_add_tracer()
3943 		__trace_add_event_dirs(tr);  in event_trace_add_tracer()
3955 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)  in early_event_add_tracer()  argument
3961 	ret = create_event_toplevel_files(parent, tr);  in early_event_add_tracer()
3966 	__trace_early_add_event_dirs(tr);  in early_event_add_tracer()
3976 int event_trace_del_tracer(struct trace_array *tr)  in event_trace_del_tracer()  argument
3981 	clear_event_triggers(tr);  in event_trace_del_tracer()
3984 	__ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);  in event_trace_del_tracer()
3987 	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);  in event_trace_del_tracer()
3993 	__trace_remove_event_dirs(tr);  in event_trace_del_tracer()
3994 	eventfs_remove_events_dir(tr->event_dir);  in event_trace_del_tracer()
3997 	tr->event_dir = NULL;  in event_trace_del_tracer()
4010 early_enable_events(struct trace_array *tr, char *buf, bool disable_first)  in early_enable_events()  argument
4024 				ftrace_set_clr_event(tr, token, 0);  in early_enable_events()
4026 			ret = ftrace_set_clr_event(tr, token, 1);  in early_enable_events()
4039 	struct trace_array *tr = top_trace_array();  in event_trace_enable()  local
4043 	if (!tr)  in event_trace_enable()
4062 	__trace_early_add_events(tr);  in event_trace_enable()
4064 	early_enable_events(tr, bootup_event_buf, false);  in event_trace_enable()
4086 	struct trace_array *tr;  in event_trace_enable_again()  local
4088 	tr = top_trace_array();  in event_trace_enable_again()
4089 	if (!tr)  in event_trace_enable_again()
4092 	early_enable_events(tr, bootup_event_buf, true);  in event_trace_enable_again()
4113 	struct trace_array *tr;  in event_trace_init()  local
4116 	tr = top_trace_array();  in event_trace_init()
4117 	if (!tr)  in event_trace_init()
4121 			  NULL, tr, &ftrace_avail_fops);  in event_trace_init()
4123 	ret = early_event_add_tracer(NULL, tr);  in event_trace_init()
4209 	struct trace_array *tr;  in event_trace_self_tests()  local
4212 	tr = top_trace_array();  in event_trace_self_tests()
4213 	if (!tr)  in event_trace_self_tests()
4218 	list_for_each_entry(file, &tr->events, list) {  in event_trace_self_tests()
4261 	list_for_each_entry(dir, &tr->systems, list) {  in event_trace_self_tests()
4271 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);  in event_trace_self_tests()
4280 		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);  in event_trace_self_tests()
4295 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);  in event_trace_self_tests()
4304 	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);  in event_trace_self_tests()
4363 	event_trace_file.tr = top_trace_array();  in event_trace_self_test_with_function()
4364 	if (WARN_ON(!event_trace_file.tr))  in event_trace_self_test_with_function()