1 // SPDX-License-Identifier: GPL-2.0
3  * Fprobe-based tracing events
24 #define TRACEPOINT_STUB ERR_PTR(-ENOENT)
50 	struct trace_probe	tp;  member
55 	return ev->ops == &trace_fprobe_ops;  in is_trace_fprobe()
64  * for_each_trace_fprobe - iterate over the trace_fprobe list
74 	return tf->fp.exit_handler != NULL;  in trace_fprobe_is_return()
79 	return tf->tpoint != NULL;  in trace_fprobe_is_tracepoint()
84 	return tf->symbol ? tf->symbol : "unknown";  in trace_fprobe_symbol()
91 	return trace_probe_is_enabled(&tf->tp);  in trace_fprobe_is_busy()
105 	argc--; argv++;  in trace_fprobe_match_command_head()
107 	return trace_probe_match_command_args(&tf->tp, argc, argv);  in trace_fprobe_match_command_head()
115 	if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))  in trace_fprobe_match()
118 	if (system && strcmp(trace_probe_group_name(&tf->tp), system))  in trace_fprobe_match()
126 	return fprobe_is_registered(&tf->fp);  in trace_fprobe_is_registered()
143 	switch (code->op) {  in process_fetch_insn()
145 		val = regs_get_kernel_stack_nth(regs, code->param);  in process_fetch_insn()
155 		val = regs_get_kernel_argument(regs, code->param);  in process_fetch_insn()
158 		val = *(unsigned long *)((unsigned long)edata + code->offset);  in process_fetch_insn()
182 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in NOKPROBE_SYMBOL()
186 	if (WARN_ON_ONCE(call != trace_file->event_call))  in NOKPROBE_SYMBOL()
192 	dsize = __get_data_size(&tf->tp, regs, NULL);  in NOKPROBE_SYMBOL()
195 					   sizeof(*entry) + tf->tp.size + dsize);  in NOKPROBE_SYMBOL()
201 	entry->ip = entry_ip;  in NOKPROBE_SYMBOL()
202 	store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize);  in NOKPROBE_SYMBOL()
211 	struct event_file_link *link;  in fentry_trace_func()  local
213 	trace_probe_for_each_link_rcu(link, &tf->tp)  in fentry_trace_func()
214 		__fentry_trace_func(tf, entry_ip, regs, link->file);  in fentry_trace_func()
225 	if (tf->tp.entry_arg)  in trace_fprobe_entry_handler()
226 		store_trace_entry_data(entry_data, &tf->tp, regs);  in trace_fprobe_entry_handler()
239 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in NOKPROBE_SYMBOL()
242 	if (WARN_ON_ONCE(call != trace_file->event_call))  in NOKPROBE_SYMBOL()
248 	dsize = __get_data_size(&tf->tp, regs, entry_data);  in NOKPROBE_SYMBOL()
251 					   sizeof(*entry) + tf->tp.size + dsize);  in NOKPROBE_SYMBOL()
257 	entry->func = entry_ip;  in NOKPROBE_SYMBOL()
258 	entry->ret_ip = ret_ip;  in NOKPROBE_SYMBOL()
259 	store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize);  in NOKPROBE_SYMBOL()
268 	struct event_file_link *link;  in fexit_trace_func()  local
270 	trace_probe_for_each_link_rcu(link, &tf->tp)  in fexit_trace_func()
271 		__fexit_trace_func(tf, entry_ip, ret_ip, regs, entry_data, link->file);  in fexit_trace_func()
280 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in fentry_perf_func()
286 	head = this_cpu_ptr(call->perf_events);  in fentry_perf_func()
290 	dsize = __get_data_size(&tf->tp, regs, NULL);  in fentry_perf_func()
291 	__size = sizeof(*entry) + tf->tp.size + dsize;  in fentry_perf_func()
293 	size -= sizeof(u32);  in fentry_perf_func()
299 	entry->ip = entry_ip;  in fentry_perf_func()
301 	store_trace_args(&entry[1], &tf->tp, regs, NULL, sizeof(*entry), dsize);  in fentry_perf_func()
302 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,  in fentry_perf_func()
313 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in fexit_perf_func()
319 	head = this_cpu_ptr(call->perf_events);  in fexit_perf_func()
323 	dsize = __get_data_size(&tf->tp, regs, entry_data);  in fexit_perf_func()
324 	__size = sizeof(*entry) + tf->tp.size + dsize;  in fexit_perf_func()
326 	size -= sizeof(u32);  in fexit_perf_func()
332 	entry->func = entry_ip;  in fexit_perf_func()
333 	entry->ret_ip = ret_ip;  in fexit_perf_func()
334 	store_trace_args(&entry[1], &tf->tp, regs, entry_data, sizeof(*entry), dsize);  in fexit_perf_func()
335 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,  in fexit_perf_func()
348 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))  in fentry_dispatcher()
351 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))  in fentry_dispatcher()
364 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_TRACE))  in fexit_dispatcher()
367 	if (trace_probe_test_flag(&tf->tp, TP_FLAG_PROFILE))  in fexit_dispatcher()
376 		trace_probe_cleanup(&tf->tp);  in free_trace_fprobe()
377 		kfree(tf->symbol);  in free_trace_fprobe()
394 	int ret = -ENOMEM;  in alloc_trace_fprobe()
396 	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);  in alloc_trace_fprobe()
400 	tf->symbol = kstrdup(symbol, GFP_KERNEL);  in alloc_trace_fprobe()
401 	if (!tf->symbol)  in alloc_trace_fprobe()
405 		tf->fp.exit_handler = fexit_dispatcher;  in alloc_trace_fprobe()
407 		tf->fp.entry_handler = fentry_dispatcher;  in alloc_trace_fprobe()
409 	tf->tpoint = tpoint;  in alloc_trace_fprobe()
410 	tf->mod = mod;  in alloc_trace_fprobe()
411 	tf->fp.nr_maxactive = maxactive;  in alloc_trace_fprobe()
413 	ret = trace_probe_init(&tf->tp, event, group, false, nargs);  in alloc_trace_fprobe()
417 	dyn_event_init(&tf->devent, &trace_fprobe_ops);  in alloc_trace_fprobe()
431 		if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&  in find_trace_fprobe()
432 		    strcmp(trace_probe_group_name(&tf->tp), group) == 0)  in find_trace_fprobe()
440 		enable_fprobe(&tf->fp);  in __enable_trace_fprobe()
445 static void __disable_trace_fprobe(struct trace_probe *tp)  in __disable_trace_fprobe()  argument
449 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in __disable_trace_fprobe()
452 		disable_fprobe(&tf->fp);  in __disable_trace_fprobe()
463 	struct trace_probe *tp;  in enable_trace_fprobe()  local
468 	tp = trace_probe_primary_from_call(call);  in enable_trace_fprobe()
469 	if (WARN_ON_ONCE(!tp))  in enable_trace_fprobe()
470 		return -ENODEV;  in enable_trace_fprobe()
471 	enabled = trace_probe_is_enabled(tp);  in enable_trace_fprobe()
475 		ret = trace_probe_add_file(tp, file);  in enable_trace_fprobe()
479 		trace_probe_set_flag(tp, TP_FLAG_PROFILE);  in enable_trace_fprobe()
482 		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in enable_trace_fprobe()
498 	struct trace_probe *tp;  in disable_trace_fprobe()  local
500 	tp = trace_probe_primary_from_call(call);  in disable_trace_fprobe()
501 	if (WARN_ON_ONCE(!tp))  in disable_trace_fprobe()
502 		return -ENODEV;  in disable_trace_fprobe()
505 		if (!trace_probe_get_file_link(tp, file))  in disable_trace_fprobe()
506 			return -ENOENT;  in disable_trace_fprobe()
507 		if (!trace_probe_has_single_file(tp))  in disable_trace_fprobe()
509 		trace_probe_clear_flag(tp, TP_FLAG_TRACE);  in disable_trace_fprobe()
511 		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);  in disable_trace_fprobe()
513 	if (!trace_probe_is_enabled(tp))  in disable_trace_fprobe()
514 		__disable_trace_fprobe(tp);  in disable_trace_fprobe()
524 		trace_probe_remove_file(tp, file);  in disable_trace_fprobe()
535 	struct trace_seq *s = &iter->seq;  in print_fentry_event()
536 	struct trace_probe *tp;  in print_fentry_event()  local
538 	field = (struct fentry_trace_entry_head *)iter->ent;  in print_fentry_event()
539 	tp = trace_probe_primary_from_call(  in print_fentry_event()
541 	if (WARN_ON_ONCE(!tp))  in print_fentry_event()
544 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));  in print_fentry_event()
546 	if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))  in print_fentry_event()
551 	if (trace_probe_print_args(s, tp->args, tp->nr_args,  in print_fentry_event()
565 	struct trace_seq *s = &iter->seq;  in print_fexit_event()
566 	struct trace_probe *tp;  in print_fexit_event()  local
568 	field = (struct fexit_trace_entry_head *)iter->ent;  in print_fexit_event()
569 	tp = trace_probe_primary_from_call(  in print_fexit_event()
571 	if (WARN_ON_ONCE(!tp))  in print_fexit_event()
574 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));  in print_fexit_event()
576 	if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))  in print_fexit_event()
579 	trace_seq_puts(s, " <- ");  in print_fexit_event()
581 	if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))  in print_fexit_event()
586 	if (trace_probe_print_args(s, tp->args, tp->nr_args,  in print_fexit_event()
600 	struct trace_probe *tp;  in fentry_event_define_fields()  local
602 	tp = trace_probe_primary_from_call(event_call);  in fentry_event_define_fields()
603 	if (WARN_ON_ONCE(!tp))  in fentry_event_define_fields()
604 		return -ENOENT;  in fentry_event_define_fields()
608 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);  in fentry_event_define_fields()
615 	struct trace_probe *tp;  in fexit_event_define_fields()  local
617 	tp = trace_probe_primary_from_call(event_call);  in fexit_event_define_fields()
618 	if (WARN_ON_ONCE(!tp))  in fexit_event_define_fields()
619 		return -ENOENT;  in fexit_event_define_fields()
624 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);  in fexit_event_define_fields()
652 	struct trace_event_call *call = trace_probe_event_call(&tf->tp);  in init_trace_event_call()
655 		call->event.funcs = &fexit_funcs;  in init_trace_event_call()
656 		call->class->fields_array = fexit_fields_array;  in init_trace_event_call()
658 		call->event.funcs = &fentry_funcs;  in init_trace_event_call()
659 		call->class->fields_array = fentry_fields_array;  in init_trace_event_call()
662 	call->flags = TRACE_EVENT_FL_FPROBE;  in init_trace_event_call()
663 	call->class->reg = fprobe_register;  in init_trace_event_call()
670 	return trace_probe_register_event_call(&tf->tp);  in register_fprobe_event()
675 	return trace_probe_unregister_event_call(&tf->tp);  in unregister_fprobe_event()
680 	struct tracepoint *tpoint = tf->tpoint;  in __regsiter_tracepoint_fprobe()
681 	unsigned long ip = (unsigned long)tpoint->probestub;  in __regsiter_tracepoint_fprobe()
686 	 * At first, put __probestub_##TP function on the tracepoint  in __regsiter_tracepoint_fprobe()
690 				tpoint->probestub, NULL, 0);  in __regsiter_tracepoint_fprobe()
693 	return register_fprobe_ips(&tf->fp, &ip, 1);  in __regsiter_tracepoint_fprobe()
696 /* Internal register function - just handle fprobe and flags */
707 		return -EINVAL;  in __register_trace_fprobe()
709 	for (i = 0; i < tf->tp.nr_args; i++) {  in __register_trace_fprobe()
710 		ret = traceprobe_update_arg(&tf->tp.args[i]);  in __register_trace_fprobe()
715 	/* Set/clear disabled flag according to tp->flag */  in __register_trace_fprobe()
716 	if (trace_probe_is_enabled(&tf->tp))  in __register_trace_fprobe()
717 		tf->fp.flags &= ~FPROBE_FL_DISABLED;  in __register_trace_fprobe()
719 		tf->fp.flags |= FPROBE_FL_DISABLED;  in __register_trace_fprobe()
724 		if (tf->tpoint == TRACEPOINT_STUB)  in __register_trace_fprobe()
731 	return register_fprobe(&tf->fp, tf->symbol, NULL);  in __register_trace_fprobe()
734 /* Internal unregister function - just handle fprobe and flags */
738 		unregister_fprobe(&tf->fp);  in __unregister_trace_fprobe()
739 		memset(&tf->fp, 0, sizeof(tf->fp));  in __unregister_trace_fprobe()
741 			tracepoint_probe_unregister(tf->tpoint,  in __unregister_trace_fprobe()
742 					tf->tpoint->probestub, NULL);  in __unregister_trace_fprobe()
743 			tf->tpoint = NULL;  in __unregister_trace_fprobe()
744 			tf->mod = NULL;  in __unregister_trace_fprobe()
754 	if (trace_probe_has_sibling(&tf->tp))  in unregister_trace_fprobe()
758 	if (trace_probe_is_enabled(&tf->tp))  in unregister_trace_fprobe()
759 		return -EBUSY;  in unregister_trace_fprobe()
762 	if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))  in unregister_trace_fprobe()
763 		return -EBUSY;  in unregister_trace_fprobe()
767 		return -EBUSY;  in unregister_trace_fprobe()
771 	dyn_event_remove(&tf->devent);  in unregister_trace_fprobe()
772 	trace_probe_unlink(&tf->tp);  in unregister_trace_fprobe()
780 	struct trace_probe_event *tpe = orig->tp.event;  in trace_fprobe_has_same_fprobe()
783 	list_for_each_entry(orig, &tpe->probes, tp.list) {  in trace_fprobe_has_same_fprobe()
792 		for (i = 0; i < orig->tp.nr_args; i++) {  in trace_fprobe_has_same_fprobe()
793 			if (strcmp(orig->tp.args[i].comm,  in trace_fprobe_has_same_fprobe()
794 				   comp->tp.args[i].comm))  in trace_fprobe_has_same_fprobe()
798 		if (i == orig->tp.nr_args)  in trace_fprobe_has_same_fprobe()
813 		return -EEXIST;  in append_trace_fprobe()
815 	ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);  in append_trace_fprobe()
820 		return -EEXIST;  in append_trace_fprobe()
825 		return -EEXIST;  in append_trace_fprobe()
829 	ret = trace_probe_append(&tf->tp, &to->tp);  in append_trace_fprobe()
835 		trace_probe_unlink(&tf->tp);  in append_trace_fprobe()
837 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));  in append_trace_fprobe()
850 	old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),  in register_trace_fprobe()
851 				   trace_probe_group_name(&tf->tp));  in register_trace_fprobe()
860 		if (ret == -EEXIST) {  in register_trace_fprobe()
873 		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));  in register_trace_fprobe()
886 static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv)  in __find_tracepoint_module_cb()  argument
890 	if (!data->tpoint && !strcmp(data->tp_name, tp->name)) {  in __find_tracepoint_module_cb()
891 		data->tpoint = tp;  in __find_tracepoint_module_cb()
892 		if (!data->mod) {  in __find_tracepoint_module_cb()
893 			data->mod = mod;  in __find_tracepoint_module_cb()
894 			if (!try_module_get(data->mod)) {  in __find_tracepoint_module_cb()
895 				data->tpoint = NULL;  in __find_tracepoint_module_cb()
896 				data->mod = NULL;  in __find_tracepoint_module_cb()
902 static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)  in __find_tracepoint_cb()  argument
906 	if (!data->tpoint && !strcmp(data->tp_name, tp->name))  in __find_tracepoint_cb()
907 		data->tpoint = tp;  in __find_tracepoint_cb()
938 	struct trace_probe *tp = &tf->tp;  in reenable_trace_fprobe()  local
940 	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {  in reenable_trace_fprobe()
970 		if (val == MODULE_STATE_COMING && tf->tpoint == TRACEPOINT_STUB) {  in __tracepoint_probe_module_cb()
971 			tpoint = find_tracepoint_in_module(tp_mod->mod, tf->symbol);  in __tracepoint_probe_module_cb()
973 				tf->tpoint = tpoint;  in __tracepoint_probe_module_cb()
974 				tf->mod = tp_mod->mod;  in __tracepoint_probe_module_cb()
976 				    trace_probe_is_enabled(&tf->tp))  in __tracepoint_probe_module_cb()
979 		} else if (val == MODULE_STATE_GOING && tp_mod->mod == tf->mod) {  in __tracepoint_probe_module_cb()
980 			tracepoint_probe_unregister(tf->tpoint,  in __tracepoint_probe_module_cb()
981 					tf->tpoint->probestub, NULL);  in __tracepoint_probe_module_cb()
982 			tf->tpoint = NULL;  in __tracepoint_probe_module_cb()
983 			tf->mod = NULL;  in __tracepoint_probe_module_cb()
1004 		int len = tmp - argv[1];  in parse_symbol_and_return()
1010 			return -EINVAL;  in parse_symbol_and_return()
1016 		return -ENOMEM;  in parse_symbol_and_return()
1027 				trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);  in parse_symbol_and_return()
1028 				return -EINVAL;  in parse_symbol_and_return()
1041 	 *  - Add fentry probe:  in __trace_fprobe_create()
1043 	 *  - Add fexit probe:  in __trace_fprobe_create()
1045 	 *  - Add tracepoint probe:  in __trace_fprobe_create()
1051 	 *  $stackN	: fetch Nth entry of stack (N:0-)  in __trace_fprobe_create()
1052 	 *  $argN	: fetch Nth argument (N:1-)  in __trace_fprobe_create()
1055 	 *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)  in __trace_fprobe_create()
1057 	 *  +|-offs(ARG) : fetch memory at ARG +|- offs address.  in __trace_fprobe_create()
1083 		return -ECANCELED;  in __trace_fprobe_create()
1098 			len = event - &argv[0][1] - 1;  in __trace_fprobe_create()
1101 		if (len > MAX_EVENT_NAME_LEN - 1) {  in __trace_fprobe_create()
1137 						  event - argv[0]);  in __trace_fprobe_create()
1164 				(unsigned long)tpoint->probestub,  in __trace_fprobe_create()
1178 	argc -= 2; argv += 2;  in __trace_fprobe_create()
1191 		ret = -E2BIG;  in __trace_fprobe_create()
1204 		/* This must return -ENOMEM, else there is a bug */  in __trace_fprobe_create()
1205 		WARN_ON_ONCE(ret != -ENOMEM);  in __trace_fprobe_create()
1213 		ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], &ctx);  in __trace_fprobe_create()
1215 			goto error;	/* This can be -ENOMEM */  in __trace_fprobe_create()
1218 	if (is_return && tf->tp.entry_arg) {  in __trace_fprobe_create()
1219 		tf->fp.entry_handler = trace_fprobe_entry_handler;  in __trace_fprobe_create()
1220 		tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp);  in __trace_fprobe_create()
1223 	ret = traceprobe_set_print_fmt(&tf->tp,  in __trace_fprobe_create()
1231 		if (ret == -EILSEQ)  in __trace_fprobe_create()
1233 		else if (ret == -ENOENT)  in __trace_fprobe_create()
1235 		else if (ret != -ENOMEM && ret != -EEXIST)  in __trace_fprobe_create()
1251 	ret = -EINVAL;  in __trace_fprobe_create()
1281 	if (trace_fprobe_is_return(tf) && tf->fp.nr_maxactive)  in trace_fprobe_show()
1282 		seq_printf(m, "%d", tf->fp.nr_maxactive);  in trace_fprobe_show()
1283 	seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),  in trace_fprobe_show()
1284 				trace_probe_name(&tf->tp));  in trace_fprobe_show()
1289 	for (i = 0; i < tf->tp.nr_args; i++)  in trace_fprobe_show()
1290 		seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);  in trace_fprobe_show()