1 // SPDX-License-Identifier: GPL-2.0
3  * Kprobes-based tracing events
10 #include <linux/bpf-cgroup.h>
15 #include <linux/error-injection.h>
63 	struct trace_probe	tp;  member
68 	return ev->ops == &trace_kprobe_ops;  in is_trace_kprobe()
77  * for_each_trace_kprobe - iterate over the trace_kprobe list
87 	return tk->rp.handler != NULL;  in trace_kprobe_is_return()
92 	return tk->symbol ? tk->symbol : "unknown";  in trace_kprobe_symbol()
97 	return tk->rp.kp.offset;  in trace_kprobe_offset()
102 	return kprobe_gone(&tk->rp.kp);  in trace_kprobe_has_gone()
120 	if (!tk->symbol)  in trace_kprobe_module_exist()
122 	p = strchr(tk->symbol, ':');  in trace_kprobe_module_exist()
127 	ret = !!find_module(tk->symbol);  in trace_kprobe_module_exist()
144 	return trace_probe_is_enabled(&tk->tp);  in trace_kprobe_is_busy()
155 	if (!tk->symbol)  in trace_kprobe_match_command_head()
156 		snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);  in trace_kprobe_match_command_head()
157 	else if (tk->rp.kp.offset)  in trace_kprobe_match_command_head()
159 			 trace_kprobe_symbol(tk), tk->rp.kp.offset);  in trace_kprobe_match_command_head()
164 	argc--; argv++;  in trace_kprobe_match_command_head()
166 	return trace_probe_match_command_args(&tk->tp, argc, argv);  in trace_kprobe_match_command_head()
175 		strcmp(trace_probe_name(&tk->tp), event) == 0) &&  in trace_kprobe_match()
176 	    (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&  in trace_kprobe_match()
186 		nhit += *per_cpu_ptr(tk->nhit, cpu);  in trace_kprobe_nhit()
193 	return !(list_empty(&tk->rp.kp.list) &&  in trace_kprobe_is_registered()
194 		 hlist_unhashed(&tk->rp.kp.hlist));  in trace_kprobe_is_registered()
203 	if (tk->symbol) {  in trace_kprobe_address()
207 			addr += tk->rp.kp.offset;  in trace_kprobe_address()
209 		addr = (unsigned long)tk->rp.kp.addr;  in trace_kprobe_address()
217 	struct trace_probe *tp;  in trace_kprobe_primary_from_call()  local
219 	tp = trace_probe_primary_from_call(call);  in trace_kprobe_primary_from_call()
220 	if (WARN_ON_ONCE(!tp))  in trace_kprobe_primary_from_call()
223 	return container_of(tp, struct trace_kprobe, tp);  in trace_kprobe_primary_from_call()
230 	return tk ? (kprobe_on_func_entry(tk->rp.kp.addr,  in trace_kprobe_on_func_entry()
231 			tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,  in trace_kprobe_on_func_entry()
232 			tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false;  in trace_kprobe_on_func_entry()
253 		trace_probe_cleanup(&tk->tp);  in free_trace_kprobe()
254 		kfree(tk->symbol);  in free_trace_kprobe()
255 		free_percpu(tk->nhit);  in free_trace_kprobe()
272 	int ret = -ENOMEM;  in alloc_trace_kprobe()
274 	tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL);  in alloc_trace_kprobe()
278 	tk->nhit = alloc_percpu(unsigned long);  in alloc_trace_kprobe()
279 	if (!tk->nhit)  in alloc_trace_kprobe()
283 		tk->symbol = kstrdup(symbol, GFP_KERNEL);  in alloc_trace_kprobe()
284 		if (!tk->symbol)  in alloc_trace_kprobe()
286 		tk->rp.kp.symbol_name = tk->symbol;  in alloc_trace_kprobe()
287 		tk->rp.kp.offset = offs;  in alloc_trace_kprobe()
289 		tk->rp.kp.addr = addr;  in alloc_trace_kprobe()
292 		tk->rp.handler = kretprobe_dispatcher;  in alloc_trace_kprobe()
294 		tk->rp.kp.pre_handler = kprobe_dispatcher;  in alloc_trace_kprobe()
296 	tk->rp.maxactive = maxactive;  in alloc_trace_kprobe()
297 	INIT_HLIST_NODE(&tk->rp.kp.hlist);  in alloc_trace_kprobe()
298 	INIT_LIST_HEAD(&tk->rp.kp.list);  in alloc_trace_kprobe()
300 	ret = trace_probe_init(&tk->tp, event, group, false, nargs);  in alloc_trace_kprobe()
304 	dyn_event_init(&tk->devent, &trace_kprobe_ops);  in alloc_trace_kprobe()
318 		if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&  in find_trace_kprobe()
319 		    strcmp(trace_probe_group_name(&tk->tp), group) == 0)  in find_trace_kprobe()
330 			ret = enable_kretprobe(&tk->rp);  in __enable_trace_kprobe()
332 			ret = enable_kprobe(&tk->rp.kp);  in __enable_trace_kprobe()
338 static void __disable_trace_kprobe(struct trace_probe *tp)  in __disable_trace_kprobe()  argument
342 	list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {  in __disable_trace_kprobe()
346 			disable_kretprobe(&tk->rp);  in __disable_trace_kprobe()
348 			disable_kprobe(&tk->rp.kp);  in __disable_trace_kprobe()
359 	struct trace_probe *tp;  in enable_trace_kprobe()  local
364 	tp = trace_probe_primary_from_call(call);  in enable_trace_kprobe()
365 	if (WARN_ON_ONCE(!tp))  in enable_trace_kprobe()
366 		return -ENODEV;  in enable_trace_kprobe()
367 	enabled = trace_probe_is_enabled(tp);  in enable_trace_kprobe()
371 		ret = trace_probe_add_file(tp, file);  in enable_trace_kprobe()
375 		trace_probe_set_flag(tp, TP_FLAG_PROFILE);  in enable_trace_kprobe()
380 	list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {  in enable_trace_kprobe()
392 			__disable_trace_kprobe(tp);  in enable_trace_kprobe()
394 			trace_probe_remove_file(tp, file);  in enable_trace_kprobe()
396 			trace_probe_clear_flag(tp, TP_FLAG_PROFILE);  in enable_trace_kprobe()
409 	struct trace_probe *tp;  in disable_trace_kprobe()  local
411 	tp = trace_probe_primary_from_call(call);  in disable_trace_kprobe()
412 	if (WARN_ON_ONCE(!tp))  in disable_trace_kprobe()
413 		return -ENODEV;  in disable_trace_kprobe()
416 		if (!trace_probe_get_file_link(tp, file))  in disable_trace_kprobe()
417 			return -ENOENT;  in disable_trace_kprobe()
418 		if (!trace_probe_has_single_file(tp))  in disable_trace_kprobe()
420 		trace_probe_clear_flag(tp, TP_FLAG_TRACE);  in disable_trace_kprobe()
422 		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);  in disable_trace_kprobe()
424 	if (!trace_probe_is_enabled(tp))  in disable_trace_kprobe()
425 		__disable_trace_kprobe(tp);  in disable_trace_kprobe()
435 		trace_probe_remove_file(tp, file);  in disable_trace_kprobe()
450 	addr -= offset;  in __within_notrace_func()
456 	return !ftrace_location_range(addr, addr + size - 1);  in __within_notrace_func()
467 	/* Check if the address is on a suffixed-symbol */  in within_notrace_func()
484 /* Internal register function - just handle k*probes and flags */
494 		return -EINVAL;  in __register_trace_kprobe()
499 		return -EINVAL;  in __register_trace_kprobe()
502 	for (i = 0; i < tk->tp.nr_args; i++) {  in __register_trace_kprobe()
503 		ret = traceprobe_update_arg(&tk->tp.args[i]);  in __register_trace_kprobe()
508 	/* Set/clear disabled flag according to tp->flag */  in __register_trace_kprobe()
509 	if (trace_probe_is_enabled(&tk->tp))  in __register_trace_kprobe()
510 		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;  in __register_trace_kprobe()
512 		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;  in __register_trace_kprobe()
515 		ret = register_kretprobe(&tk->rp);  in __register_trace_kprobe()
517 		ret = register_kprobe(&tk->rp.kp);  in __register_trace_kprobe()
522 /* Internal unregister function - just handle k*probes and flags */
527 			unregister_kretprobe(&tk->rp);  in __unregister_trace_kprobe()
529 			unregister_kprobe(&tk->rp.kp);  in __unregister_trace_kprobe()
531 		INIT_HLIST_NODE(&tk->rp.kp.hlist);  in __unregister_trace_kprobe()
532 		INIT_LIST_HEAD(&tk->rp.kp.list);  in __unregister_trace_kprobe()
533 		if (tk->rp.kp.symbol_name)  in __unregister_trace_kprobe()
534 			tk->rp.kp.addr = NULL;  in __unregister_trace_kprobe()
542 	if (trace_probe_has_sibling(&tk->tp))  in unregister_trace_kprobe()
546 	if (trace_probe_is_enabled(&tk->tp))  in unregister_trace_kprobe()
547 		return -EBUSY;  in unregister_trace_kprobe()
550 	if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp)))  in unregister_trace_kprobe()
551 		return -EBUSY;  in unregister_trace_kprobe()
555 		return -EBUSY;  in unregister_trace_kprobe()
559 	dyn_event_remove(&tk->devent);  in unregister_trace_kprobe()
560 	trace_probe_unlink(&tk->tp);  in unregister_trace_kprobe()
568 	struct trace_probe_event *tpe = orig->tp.event;  in trace_kprobe_has_same_kprobe()
571 	list_for_each_entry(orig, &tpe->probes, tp.list) {  in trace_kprobe_has_same_kprobe()
581 		for (i = 0; i < orig->tp.nr_args; i++) {  in trace_kprobe_has_same_kprobe()
582 			if (strcmp(orig->tp.args[i].comm,  in trace_kprobe_has_same_kprobe()
583 				   comp->tp.args[i].comm))  in trace_kprobe_has_same_kprobe()
587 		if (i == orig->tp.nr_args)  in trace_kprobe_has_same_kprobe()
598 	ret = trace_probe_compare_arg_type(&tk->tp, &to->tp);  in append_trace_kprobe()
603 		return -EEXIST;  in append_trace_kprobe()
608 		return -EEXIST;  in append_trace_kprobe()
612 	ret = trace_probe_append(&tk->tp, &to->tp);  in append_trace_kprobe()
618 	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {  in append_trace_kprobe()
624 		trace_probe_unlink(&tk->tp);  in append_trace_kprobe()
626 		dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));  in append_trace_kprobe()
639 	old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),  in register_trace_kprobe()
640 				   trace_probe_group_name(&tk->tp));  in register_trace_kprobe()
645 			ret = -EEXIST;  in register_trace_kprobe()
655 		if (ret == -EEXIST) {  in register_trace_kprobe()
665 	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {  in register_trace_kprobe()
673 		dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));  in register_trace_kprobe()
712 			/* Don't need to check busy - this should have gone. */  in trace_kprobe_module_callback()
716 				pr_warn("Failed to re-register probe %s on %s: %d\n",  in trace_kprobe_module_callback()
717 					trace_probe_name(&tk->tp),  in trace_kprobe_module_callback()
759 	if (strcmp(name, ctx->name) == 0)  in count_mod_symbols()
760 		ctx->count++;  in count_mod_symbols()
786 		return -EADDRNOTAVAIL;  in validate_module_probe_symbol()
792 		return -ENOENT;  in validate_module_probe_symbol()
847 	 *  - Add kprobe:  in __trace_kprobe_create()
849 	 *  - Add kretprobe:  in __trace_kprobe_create()
857 	 *  $stackN	: fetch Nth of stack (N:0-)  in __trace_kprobe_create()
860 	 *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)  in __trace_kprobe_create()
863 	 *  +|-offs(ARG) : fetch memory at ARG +|- offs address.  in __trace_kprobe_create()
892 		return -ECANCELED;  in __trace_kprobe_create()
895 		return -ECANCELED;  in __trace_kprobe_create()
909 			len = event - &argv[0][1] - 1;  in __trace_kprobe_create()
912 		if (len > MAX_EVENT_NAME_LEN - 1) {  in __trace_kprobe_create()
938 			ret = -ECANCELED;  in __trace_kprobe_create()
944 			return -ENOMEM;  in __trace_kprobe_create()
952 				trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX);  in __trace_kprobe_create()
965 			if (ret == -EADDRNOTAVAIL)  in __trace_kprobe_create()
977 		if (ret == -EINVAL && is_return) {  in __trace_kprobe_create()
986 						  event - argv[0]);  in __trace_kprobe_create()
1003 	argc -= 2; argv += 2;  in __trace_kprobe_create()
1017 		ret = -E2BIG;  in __trace_kprobe_create()
1030 		/* This must return -ENOMEM, else there is a bug */  in __trace_kprobe_create()
1031 		WARN_ON_ONCE(ret != -ENOMEM);  in __trace_kprobe_create()
1039 		ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], &ctx);  in __trace_kprobe_create()
1041 			goto error;	/* This can be -ENOMEM */  in __trace_kprobe_create()
1044 	if (is_return && tk->tp.entry_arg) {  in __trace_kprobe_create()
1045 		tk->rp.entry_handler = trace_kprobe_entry_handler;  in __trace_kprobe_create()
1046 		tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp);  in __trace_kprobe_create()
1050 	ret = traceprobe_set_print_fmt(&tk->tp, ptype);  in __trace_kprobe_create()
1057 		if (ret == -EILSEQ)  in __trace_kprobe_create()
1059 		else if (ret == -ENOENT)  in __trace_kprobe_create()
1061 		else if (ret != -ENOMEM && ret != -EEXIST)  in __trace_kprobe_create()
1075 	ret = -EINVAL;  in __trace_kprobe_create()
1090 	if (raw_command[0] == '-')  in create_or_delete_trace_kprobe()
1094 	return ret == -ECANCELED ? -EINVAL : ret;  in create_or_delete_trace_kprobe()
1099 	return create_or_delete_trace_kprobe(cmd->seq.buffer);  in trace_kprobe_run_command()
1103  * kprobe_event_cmd_init - Initialize a kprobe event command object
1119  * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
1137  * returns -EINVAL if @loc == NULL.
1149 	if (cmd->type != DYNEVENT_TYPE_KPROBE)  in __kprobe_event_gen_cmd_start()
1150 		return -EINVAL;  in __kprobe_event_gen_cmd_start()
1153 		return -EINVAL;  in __kprobe_event_gen_cmd_start()
1178 		if (++cmd->n_fields > MAX_TRACE_ARGS) {  in __kprobe_event_gen_cmd_start()
1179 			ret = -EINVAL;  in __kprobe_event_gen_cmd_start()
1195  * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
1216 	if (cmd->type != DYNEVENT_TYPE_KPROBE)  in __kprobe_event_add_fields()
1217 		return -EINVAL;  in __kprobe_event_add_fields()
1229 		if (++cmd->n_fields > MAX_TRACE_ARGS) {  in __kprobe_event_add_fields()
1230 			ret = -EINVAL;  in __kprobe_event_add_fields()
1246  * kprobe_event_delete - Delete a kprobe event
1258 	snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name);  in kprobe_event_delete()
1280 	if (trace_kprobe_is_return(tk) && tk->rp.maxactive)  in trace_kprobe_show()
1281 		seq_printf(m, "%d", tk->rp.maxactive);  in trace_kprobe_show()
1282 	seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),  in trace_kprobe_show()
1283 				trace_probe_name(&tk->tp));  in trace_kprobe_show()
1285 	if (!tk->symbol)  in trace_kprobe_show()
1286 		seq_printf(m, " 0x%p", tk->rp.kp.addr);  in trace_kprobe_show()
1287 	else if (tk->rp.kp.offset)  in trace_kprobe_show()
1289 			   tk->rp.kp.offset);  in trace_kprobe_show()
1293 	for (i = 0; i < tk->tp.nr_args; i++)  in trace_kprobe_show()
1294 		seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);  in trace_kprobe_show()
1325 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {  in probes_open()
1353 		tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;  in trace_kprobe_missed()
1368 	seq_printf(m, "  %-44s %15lu %15lu\n",  in probes_profile_seq_show()
1369 		   trace_probe_name(&tk->tp),  in probes_profile_seq_show()
1413 	switch (code->op) {  in process_fetch_insn()
1415 		val = regs_get_register(regs, code->param);  in process_fetch_insn()
1418 		val = regs_get_kernel_stack_nth(regs, code->param);  in process_fetch_insn()
1428 		val = regs_get_kernel_argument(regs, code->param);  in process_fetch_insn()
1431 		val = *(unsigned long *)((unsigned long)edata + code->offset);  in process_fetch_insn()
1454 	struct trace_event_call *call = trace_probe_event_call(&tk->tp);  in NOKPROBE_SYMBOL()
1458 	WARN_ON(call != trace_file->event_call);  in NOKPROBE_SYMBOL()
1463 	dsize = __get_data_size(&tk->tp, regs, NULL);  in NOKPROBE_SYMBOL()
1466 					   sizeof(*entry) + tk->tp.size + dsize);  in NOKPROBE_SYMBOL()
1471 	entry->ip = (unsigned long)tk->rp.kp.addr;  in NOKPROBE_SYMBOL()
1472 	store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);  in NOKPROBE_SYMBOL()
1480 	struct event_file_link *link;  in kprobe_trace_func()  local
1482 	trace_probe_for_each_link_rcu(link, &tk->tp)  in kprobe_trace_func()
1483 		__kprobe_trace_func(tk, regs, link->file);  in kprobe_trace_func()
1501 		return -ENOENT;  in trace_kprobe_entry_handler()
1505 	/* store argument values into ri->data as entry data */  in trace_kprobe_entry_handler()
1506 	if (tk->tp.entry_arg)  in trace_kprobe_entry_handler()
1507 		store_trace_entry_data(ri->data, &tk->tp, regs);  in trace_kprobe_entry_handler()
1520 	struct trace_event_call *call = trace_probe_event_call(&tk->tp);  in __kretprobe_trace_func()
1523 	WARN_ON(call != trace_file->event_call);  in __kretprobe_trace_func()
1528 	dsize = __get_data_size(&tk->tp, regs, ri->data);  in __kretprobe_trace_func()
1531 					   sizeof(*entry) + tk->tp.size + dsize);  in __kretprobe_trace_func()
1536 	entry->func = (unsigned long)tk->rp.kp.addr;  in __kretprobe_trace_func()
1537 	entry->ret_ip = get_kretprobe_retaddr(ri);  in __kretprobe_trace_func()
1538 	store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);  in __kretprobe_trace_func()
1547 	struct event_file_link *link;  in kretprobe_trace_func()  local
1549 	trace_probe_for_each_link_rcu(link, &tk->tp)  in kretprobe_trace_func()
1550 		__kretprobe_trace_func(tk, ri, regs, link->file);  in kretprobe_trace_func()
1560 	struct trace_seq *s = &iter->seq;  in print_kprobe_event()
1561 	struct trace_probe *tp;  in print_kprobe_event()  local
1563 	field = (struct kprobe_trace_entry_head *)iter->ent;  in print_kprobe_event()
1564 	tp = trace_probe_primary_from_call(  in print_kprobe_event()
1566 	if (WARN_ON_ONCE(!tp))  in print_kprobe_event()
1569 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));  in print_kprobe_event()
1571 	if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))  in print_kprobe_event()
1576 	if (trace_probe_print_args(s, tp->args, tp->nr_args,  in print_kprobe_event()
1590 	struct trace_seq *s = &iter->seq;  in print_kretprobe_event()
1591 	struct trace_probe *tp;  in print_kretprobe_event()  local
1593 	field = (struct kretprobe_trace_entry_head *)iter->ent;  in print_kretprobe_event()
1594 	tp = trace_probe_primary_from_call(  in print_kretprobe_event()
1596 	if (WARN_ON_ONCE(!tp))  in print_kretprobe_event()
1599 	trace_seq_printf(s, "%s: (", trace_probe_name(tp));  in print_kretprobe_event()
1601 	if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))  in print_kretprobe_event()
1604 	trace_seq_puts(s, " <- ");  in print_kretprobe_event()
1606 	if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))  in print_kretprobe_event()
1611 	if (trace_probe_print_args(s, tp->args, tp->nr_args,  in print_kretprobe_event()
1626 	struct trace_probe *tp;  in kprobe_event_define_fields()  local
1628 	tp = trace_probe_primary_from_call(event_call);  in kprobe_event_define_fields()
1629 	if (WARN_ON_ONCE(!tp))  in kprobe_event_define_fields()
1630 		return -ENOENT;  in kprobe_event_define_fields()
1634 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);  in kprobe_event_define_fields()
1641 	struct trace_probe *tp;  in kretprobe_event_define_fields()  local
1643 	tp = trace_probe_primary_from_call(event_call);  in kretprobe_event_define_fields()
1644 	if (WARN_ON_ONCE(!tp))  in kretprobe_event_define_fields()
1645 		return -ENOENT;  in kretprobe_event_define_fields()
1650 	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);  in kretprobe_event_define_fields()
1659 	struct trace_event_call *call = trace_probe_event_call(&tk->tp);  in kprobe_perf_func()
1682 	head = this_cpu_ptr(call->perf_events);  in kprobe_perf_func()
1686 	dsize = __get_data_size(&tk->tp, regs, NULL);  in kprobe_perf_func()
1687 	__size = sizeof(*entry) + tk->tp.size + dsize;  in kprobe_perf_func()
1689 	size -= sizeof(u32);  in kprobe_perf_func()
1695 	entry->ip = (unsigned long)tk->rp.kp.addr;  in kprobe_perf_func()
1697 	store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);  in kprobe_perf_func()
1698 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,  in kprobe_perf_func()
1709 	struct trace_event_call *call = trace_probe_event_call(&tk->tp);  in kretprobe_perf_func()
1718 	head = this_cpu_ptr(call->perf_events);  in kretprobe_perf_func()
1722 	dsize = __get_data_size(&tk->tp, regs, ri->data);  in kretprobe_perf_func()
1723 	__size = sizeof(*entry) + tk->tp.size + dsize;  in kretprobe_perf_func()
1725 	size -= sizeof(u32);  in kretprobe_perf_func()
1731 	entry->func = (unsigned long)tk->rp.kp.addr;  in kretprobe_perf_func()
1732 	entry->ret_ip = get_kretprobe_retaddr(ri);  in kretprobe_perf_func()
1733 	store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);  in kretprobe_perf_func()
1734 	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,  in kretprobe_perf_func()
1744 	const char *pevent = trace_event_name(event->tp_event);  in bpf_get_kprobe_info()
1745 	const char *group = event->tp_event->class->system;  in bpf_get_kprobe_info()
1751 		tk = trace_kprobe_primary_from_call(event->tp_event);  in bpf_get_kprobe_info()
1753 		return -EINVAL;  in bpf_get_kprobe_info()
1757 	*probe_offset = tk->rp.kp.offset;  in bpf_get_kprobe_info()
1759 		      (unsigned long)tk->rp.kp.addr : 0;  in bpf_get_kprobe_info()
1760 	*symbol = tk->symbol;  in bpf_get_kprobe_info()
1804 	raw_cpu_inc(*tk->nhit);  in kprobe_dispatcher()
1806 	if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))  in kprobe_dispatcher()
1809 	if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))  in kprobe_dispatcher()
1831 	raw_cpu_inc(*tk->nhit);  in kretprobe_dispatcher()
1833 	if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))  in kretprobe_dispatcher()
1836 	if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))  in kretprobe_dispatcher()
1865 	struct trace_event_call *call = trace_probe_event_call(&tk->tp);  in init_trace_event_call()
1868 		call->event.funcs = &kretprobe_funcs;  in init_trace_event_call()
1869 		call->class->fields_array = kretprobe_fields_array;  in init_trace_event_call()
1871 		call->event.funcs = &kprobe_funcs;  in init_trace_event_call()
1872 		call->class->fields_array = kprobe_fields_array;  in init_trace_event_call()
1875 	call->flags = TRACE_EVENT_FL_KPROBE;  in init_trace_event_call()
1876 	call->class->reg = kprobe_register;  in init_trace_event_call()
1883 	return trace_probe_register_event_call(&tk->tp);  in register_kprobe_event()
1888 	return trace_probe_unregister_event_call(&tk->tp);  in unregister_kprobe_event()
1930 	if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0) {  in create_local_trace_kprobe()
1931 		ret = -ENOMEM;  in create_local_trace_kprobe()
1939 	return trace_probe_event_call(&tk->tp);  in create_local_trace_kprobe()
1953 	if (trace_probe_is_enabled(&tk->tp)) {  in destroy_local_trace_kprobe()
1973 		list_for_each_entry(file, &tr->events, list)  in enable_boot_kprobe_events()
1974 			if (file->event_call == trace_probe_event_call(&tk->tp))  in enable_boot_kprobe_events()
2015 		return -EINVAL;  in init_kprobe_trace_early()
2051 	list_for_each_entry(file, &tr->events, list)  in find_trace_probe_file()
2052 		if (file->event_call == trace_probe_event_call(&tk->tp))  in find_trace_probe_file()
2070 		return -ENODEV;  in kprobe_trace_self_tests_init()
2093 					trace_probe_event_call(&tk->tp), file);  in kprobe_trace_self_tests_init()
2111 					trace_probe_event_call(&tk->tp), file);  in kprobe_trace_self_tests_init()
2123 	 * are no side-effects and the call is never performed.  in kprobe_trace_self_tests_init()
2142 				trace_probe_event_call(&tk->tp), file);  in kprobe_trace_self_tests_init()
2158 				trace_probe_event_call(&tk->tp), file);  in kprobe_trace_self_tests_init()
2161 	ret = create_or_delete_trace_kprobe("-:testprobe");  in kprobe_trace_self_tests_init()
2165 	ret = create_or_delete_trace_kprobe("-:testprobe2");  in kprobe_trace_self_tests_init()