1 // SPDX-License-Identifier: GPL-2.0
2 #include <trace/syscall.h>
3 #include <trace/events/syscalls.h>
15 #include "trace.h"
17 static DEFINE_MUTEX(syscall_trace_lock);
19 static int syscall_enter_register(struct trace_event_call *event,
21 static int syscall_exit_register(struct trace_event_call *event,
24 static struct list_head *
27 	struct syscall_metadata *entry = call->data;  in syscall_get_enter_fields()
29 	return &entry->enter_fields;  in syscall_get_enter_fields()
35 static DEFINE_XARRAY(syscalls_metadata_sparse);
36 static struct syscall_metadata **syscalls_metadata;
39 static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)  in arch_syscall_match_sym_name()
67 static int
71 		return -1;  in trace_get_syscall_nr()
76 static inline int
83 static __init struct syscall_metadata *
99 		if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))  in find_syscall_meta()
105 static struct syscall_metadata *syscall_nr_to_meta(int nr)  in syscall_nr_to_meta()
124 	return entry->name;  in get_syscall_name()
130 static void get_dynamic_len_ptr(struct syscall_trace_enter *trace,  in get_dynamic_len_ptr()  argument
139 	ptr = (void *)trace->args + sizeof(long) * entry->nb_args + offset;  in get_dynamic_len_ptr()
143 	ptr = (void *)trace + (val & 0xffff);  in get_dynamic_len_ptr()
151 static enum print_line_t
152 sys_enter_openat_print(struct syscall_trace_enter *trace, struct syscall_metadata *entry,  in sys_enter_openat_print()  argument
159 	static const struct trace_print_flags __flags[] =  in sys_enter_openat_print()
177 			{ -1, NULL }  in sys_enter_openat_print()
180 	trace_seq_printf(s, "%s(", entry->name);  in sys_enter_openat_print()
182 	for (int i = 0; !done && i < entry->nb_args; i++) {  in sys_enter_openat_print()
192 			bits = trace->args[2];  in sys_enter_openat_print()
218 			trace_seq_printf(s, "%s: 0%03o", entry->args[i],  in sys_enter_openat_print()
219 					 (unsigned int)trace->args[i]);  in sys_enter_openat_print()
223 		trace_seq_printf(s, "%s: %lu", entry->args[i],  in sys_enter_openat_print()
224 				 trace->args[i]);  in sys_enter_openat_print()
226 		if (!(BIT(i) & entry->user_mask))  in sys_enter_openat_print()
229 		get_dynamic_len_ptr(trace, entry, &offset, &len, &ptr);  in sys_enter_openat_print()
240 static enum print_line_t
244 	struct trace_array *tr = iter->tr;  in print_syscall_enter()
245 	struct trace_seq *s = &iter->seq;  in print_syscall_enter()
246 	struct trace_entry *ent = iter->ent;  in print_syscall_enter()
247 	struct syscall_trace_enter *trace;  in print_syscall_enter()  local
253 	trace = (typeof(trace))ent;  in print_syscall_enter()
254 	syscall = trace->nr;  in print_syscall_enter()
260 	if (entry->enter_event->event.type != ent->type) {  in print_syscall_enter()
265 	switch (entry->syscall_nr) {  in print_syscall_enter()
267 		if (!tr || !(tr->trace_flags & TRACE_ITER(VERBOSE)))  in print_syscall_enter()
268 			return sys_enter_openat_print(trace, entry, s, event);  in print_syscall_enter()
274 	trace_seq_printf(s, "%s(", entry->name);  in print_syscall_enter()
276 	for (i = 0; i < entry->nb_args; i++) {  in print_syscall_enter()
287 		if (tr && tr->trace_flags & TRACE_ITER(VERBOSE))  in print_syscall_enter()
288 			trace_seq_printf(s, "%s ", entry->types[i]);  in print_syscall_enter()
291 		if (trace->args[i] < 10)  in print_syscall_enter()
292 			trace_seq_printf(s, "%s: %lu", entry->args[i],  in print_syscall_enter()
293 					 trace->args[i]);  in print_syscall_enter()
295 			trace_seq_printf(s, "%s: 0x%lx", entry->args[i],  in print_syscall_enter()
296 					 trace->args[i]);  in print_syscall_enter()
298 		if (!(BIT(i) & entry->user_mask))  in print_syscall_enter()
301 		get_dynamic_len_ptr(trace, entry, &offset, &len, &ptr);  in print_syscall_enter()
303 		if (entry->user_arg_size < 0 || entry->user_arg_is_str) {  in print_syscall_enter()
308 		val = trace->args[entry->user_arg_size];  in print_syscall_enter()
348 static enum print_line_t
352 	struct trace_seq *s = &iter->seq;  in print_syscall_exit()
353 	struct trace_entry *ent = iter->ent;  in print_syscall_exit()
354 	struct syscall_trace_exit *trace;  in print_syscall_exit()  local
358 	trace = (typeof(trace))ent;  in print_syscall_exit()
359 	syscall = trace->nr;  in print_syscall_exit()
367 	if (entry->exit_event->event.type != ent->type) {  in print_syscall_exit()
372 	trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,  in print_syscall_exit()
373 				trace->ret);  in print_syscall_exit()
385 #define LEN_OR_ZERO (len ? len - pos : 0)
387 static int __init
395 			" ((unsigned long)(REC->dfd)),");  in sys_enter_openat_print_fmt()
397 			" ((unsigned long)(REC->filename)),");  in sys_enter_openat_print_fmt()
401 			" (REC->flags & ~3) && !(REC->flags & 3) ? \"O_RDONLY|\" : \"\", ");  in sys_enter_openat_print_fmt()
403 			" REC->flags ? __print_flags(REC->flags, \"|\", ");  in sys_enter_openat_print_fmt()
436 			" ((unsigned long)(REC->mode))");  in sys_enter_openat_print_fmt()
440 static int __init
443 	bool is_string = entry->user_arg_is_str;  in __set_enter_print_fmt()
447 	switch (entry->syscall_nr) {  in __set_enter_print_fmt()
455 	for (i = 0; i < entry->nb_args; i++) {  in __set_enter_print_fmt()
459 				entry->args[i], sizeof(unsigned long));  in __set_enter_print_fmt()
461 		if (!(BIT(i) & entry->user_mask))  in __set_enter_print_fmt()
465 		if (entry->user_arg_size < 0 || is_string)  in __set_enter_print_fmt()
472 	for (i = 0; i < entry->nb_args; i++) {  in __set_enter_print_fmt()
474 				", ((unsigned long)(REC->%s))", entry->args[i]);  in __set_enter_print_fmt()
475 		if (!(BIT(i) & entry->user_mask))  in __set_enter_print_fmt()
478 		if (entry->user_arg_size < 0 || is_string) {  in __set_enter_print_fmt()
480 					entry->args[i]);  in __set_enter_print_fmt()
483 					entry->args[i]);  in __set_enter_print_fmt()
493 static int __init set_syscall_print_fmt(struct trace_event_call *call)  in set_syscall_print_fmt()
497 	struct syscall_metadata *entry = call->data;  in set_syscall_print_fmt()
499 	if (entry->enter_event != call) {  in set_syscall_print_fmt()
500 		call->print_fmt = "\"0x%lx\", REC->ret";  in set_syscall_print_fmt()
509 		return -ENOMEM;  in set_syscall_print_fmt()
513 	call->print_fmt = print_fmt;  in set_syscall_print_fmt()
518 static void __init free_syscall_print_fmt(struct trace_event_call *call)  in free_syscall_print_fmt()
520 	struct syscall_metadata *entry = call->data;  in free_syscall_print_fmt()
522 	if (entry->enter_event == call)  in free_syscall_print_fmt()
523 		kfree(call->print_fmt);  in free_syscall_print_fmt()
526 static int __init syscall_enter_define_fields(struct trace_event_call *call)  in syscall_enter_define_fields()
528 	struct syscall_trace_enter trace;  in syscall_enter_define_fields()  local
529 	struct syscall_metadata *meta = call->data;  in syscall_enter_define_fields()
532 	int offset = offsetof(typeof(trace), args);  in syscall_enter_define_fields()
537 	for (i = 0; i < meta->nb_args; i++) {  in syscall_enter_define_fields()
538 		ret = trace_define_field(call, meta->types[i],  in syscall_enter_define_fields()
539 					 meta->args[i], offset,  in syscall_enter_define_fields()
547 	if (ret || !meta->user_mask)  in syscall_enter_define_fields()
550 	mask = meta->user_mask;  in syscall_enter_define_fields()
553 		int idx = ffs(mask) - 1;  in syscall_enter_define_fields()
562 		len = strlen(meta->args[idx]) + sizeof("___val");  in syscall_enter_define_fields()
565 			meta->user_mask = 0;  in syscall_enter_define_fields()
566 			return -ENOMEM;  in syscall_enter_define_fields()
569 		snprintf(arg, len, "__%s_val", meta->args[idx]);  in syscall_enter_define_fields()
587  *  (defined in kernel/trace/trace.h)
606 static struct syscall_user_buffer *syscall_buffer;
608 static int syscall_fault_buffer_enable(void)  in syscall_fault_buffer_enable()
616 		trace_user_fault_get(&syscall_buffer->buf);  in syscall_fault_buffer_enable()
622 		return -ENOMEM;  in syscall_fault_buffer_enable()
624 	ret = trace_user_fault_init(&sbuf->buf, SYSCALL_FAULT_BUF_SZ);  in syscall_fault_buffer_enable()
635 static void rcu_free_syscall_buffer(struct rcu_head *rcu)  in rcu_free_syscall_buffer()
640 	trace_user_fault_destroy(&sbuf->buf);  in rcu_free_syscall_buffer()
645 static void syscall_fault_buffer_disable(void)  in syscall_fault_buffer_disable()
651 	if (trace_user_fault_put(&sbuf->buf))  in syscall_fault_buffer_disable()
655 	call_rcu_tasks_trace(&sbuf->rcu, rcu_free_syscall_buffer);  in syscall_fault_buffer_disable()
664 static int syscall_copy_user(char *buf, const char __user *ptr,  in syscall_copy_user()
670 	for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) {  in syscall_copy_user()
671 		ptr = (char __user *)args->ptr_array[i];  in syscall_copy_user()
673 		args->read[i] = ret;  in syscall_copy_user()
678 static int syscall_copy_user_array(char *buf, const char __user *ptr,  in syscall_copy_user_array()
684 	for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) {  in syscall_copy_user_array()
685 		ptr = (char __user *)args->ptr_array[i];  in syscall_copy_user_array()
687 		args->read[i] = ret ? -1 : size;  in syscall_copy_user_array()
692 static char *sys_fault_user(unsigned int buf_size,  in sys_fault_user()
699 	unsigned long mask = sys_data->user_mask;  in sys_fault_user()
700 	unsigned long size = SYSCALL_FAULT_ARG_SZ - 1;  in sys_fault_user()
717 	if (sys_data->user_arg_size >= 0) {  in sys_fault_user()
719 		size = args[sys_data->user_arg_size];  in sys_fault_user()
720 		if (size > SYSCALL_FAULT_ARG_SZ - 1)  in sys_fault_user()
721 			size = SYSCALL_FAULT_ARG_SZ - 1;  in sys_fault_user()
726 		int idx = ffs(mask) - 1;  in sys_fault_user()
740 		data_size[i] = -1; /* Denotes no pointer */  in sys_fault_user()
747 	buffer = trace_user_fault_read(&sbuf->buf, NULL, size,  in sys_fault_user()
760 		/* For strings, replace any non-printable characters with '.' */  in sys_fault_user()
788 static int
799 		return -1;  in syscall_get_data()
818 static void syscall_put_data(struct syscall_metadata *sys_data,  in syscall_put_data()
830 	ptr = (void *)entry->args + sizeof(unsigned long) * sys_data->nb_args;  in syscall_put_data()
834 	 * the beginning of the event. That is after the static arguments  in syscall_put_data()
837 	val = (ptr - (void *)entry) + 4 * uargs;  in syscall_put_data()
842 			val += user_sizes[i - 1];  in syscall_put_data()
861 static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)  in ftrace_syscall_enter()  argument
878 	 * buffer and per-cpu data require preemption to be disabled.  in ftrace_syscall_enter()
886 	trace_file = READ_ONCE(tr->enter_syscall_files[syscall_nr]);  in ftrace_syscall_enter()
898 	mayfault = sys_data->user_mask != 0;  in ftrace_syscall_enter()
906 				     &size, user_sizes, &uargs, tr->syscall_buf_sz) < 0)  in ftrace_syscall_enter()
910 	size += sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;  in ftrace_syscall_enter()
917 	entry->nr = syscall_nr;  in ftrace_syscall_enter()
919 	memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args);  in ftrace_syscall_enter()
927 static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)  in ftrace_syscall_exit()
938 	 * buffer and per-cpu data require preemption to be disabled.  in ftrace_syscall_exit()
947 	trace_file = READ_ONCE(tr->exit_syscall_files[syscall_nr]);  in ftrace_syscall_exit()
963 	entry->nr = syscall_nr;  in ftrace_syscall_exit()
964 	entry->ret = syscall_get_return_value(current, regs);  in ftrace_syscall_exit()
969 static int reg_event_syscall_enter(struct trace_event_file *file,  in reg_event_syscall_enter()
972 	struct syscall_metadata *sys_data = call->data;  in reg_event_syscall_enter()
973 	struct trace_array *tr = file->tr;  in reg_event_syscall_enter()
977 	num = sys_data->syscall_nr;  in reg_event_syscall_enter()
979 		return -ENOSYS;  in reg_event_syscall_enter()
981 	if (sys_data->user_mask) {  in reg_event_syscall_enter()
986 	if (!tr->sys_refcount_enter) {  in reg_event_syscall_enter()
989 			if (sys_data->user_mask)  in reg_event_syscall_enter()
994 	WRITE_ONCE(tr->enter_syscall_files[num], file);  in reg_event_syscall_enter()
995 	tr->sys_refcount_enter++;  in reg_event_syscall_enter()
999 static void unreg_event_syscall_enter(struct trace_event_file *file,  in unreg_event_syscall_enter()
1002 	struct syscall_metadata *sys_data = call->data;  in unreg_event_syscall_enter()
1003 	struct trace_array *tr = file->tr;  in unreg_event_syscall_enter()
1006 	num = sys_data->syscall_nr;  in unreg_event_syscall_enter()
1010 	tr->sys_refcount_enter--;  in unreg_event_syscall_enter()
1011 	WRITE_ONCE(tr->enter_syscall_files[num], NULL);  in unreg_event_syscall_enter()
1012 	if (!tr->sys_refcount_enter)  in unreg_event_syscall_enter()
1014 	if (sys_data->user_mask)  in unreg_event_syscall_enter()
1018 static int reg_event_syscall_exit(struct trace_event_file *file,  in reg_event_syscall_exit()
1021 	struct trace_array *tr = file->tr;  in reg_event_syscall_exit()
1025 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in reg_event_syscall_exit()
1027 		return -ENOSYS;  in reg_event_syscall_exit()
1029 	if (!tr->sys_refcount_exit)  in reg_event_syscall_exit()
1032 		WRITE_ONCE(tr->exit_syscall_files[num], file);  in reg_event_syscall_exit()
1033 		tr->sys_refcount_exit++;  in reg_event_syscall_exit()
1039 static void unreg_event_syscall_exit(struct trace_event_file *file,  in unreg_event_syscall_exit()
1042 	struct trace_array *tr = file->tr;  in unreg_event_syscall_exit()
1045 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in unreg_event_syscall_exit()
1049 	tr->sys_refcount_exit--;  in unreg_event_syscall_exit()
1050 	WRITE_ONCE(tr->exit_syscall_files[num], NULL);  in unreg_event_syscall_exit()
1051 	if (!tr->sys_refcount_exit)  in unreg_event_syscall_exit()
1061 static void check_faultable_syscall(struct trace_event_call *call, int nr)  in check_faultable_syscall()
1063 	struct syscall_metadata *sys_data = call->data;  in check_faultable_syscall()
1067 	if (sys_data->enter_event != call)  in check_faultable_syscall()
1070 	sys_data->user_arg_size = -1;  in check_faultable_syscall()
1079 		sys_data->user_mask = BIT(1);  in check_faultable_syscall()
1080 		sys_data->user_arg_size = 2;  in check_faultable_syscall()
1085 		sys_data->user_mask = BIT(0);  in check_faultable_syscall()
1086 		sys_data->user_arg_size = 1;  in check_faultable_syscall()
1087 		sys_data->user_arg_is_str = 1;  in check_faultable_syscall()
1092 		sys_data->user_mask = BIT(4);  in check_faultable_syscall()
1093 		sys_data->user_arg_size = 3;  in check_faultable_syscall()
1094 		sys_data->user_arg_is_str = 1;  in check_faultable_syscall()
1156 		sys_data->user_mask = BIT(0);  in check_faultable_syscall()
1194 		sys_data->user_mask = BIT(1);  in check_faultable_syscall()
1199 		sys_data->user_mask = BIT(2);  in check_faultable_syscall()
1203 		sys_data->user_mask = BIT(4);  in check_faultable_syscall()
1226 		sys_data->user_mask = BIT(0) | BIT(1);  in check_faultable_syscall()
1230 		sys_data->user_mask = BIT(0) | BIT(2);  in check_faultable_syscall()
1243 		sys_data->user_mask = BIT(1) | BIT(3);  in check_faultable_syscall()
1246 		sys_data->user_mask = BIT(0) | BIT(1) | BIT(2);  in check_faultable_syscall()
1249 		sys_data->user_mask = 0;  in check_faultable_syscall()
1253 	if (sys_data->user_arg_size < 0)  in check_faultable_syscall()
1260 	mask = sys_data->user_mask;  in check_faultable_syscall()
1261 	if (WARN_ON(mask & (mask - 1)))  in check_faultable_syscall()
1262 		sys_data->user_arg_size = -1;  in check_faultable_syscall()
1265 static int __init init_syscall_trace(struct trace_event_call *call)  in init_syscall_trace()
1267 	int id;  in init_syscall_trace()  local
1270 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in init_syscall_trace()
1273 				((struct syscall_metadata *)call->data)->name);  in init_syscall_trace()
1274 		return -ENOSYS;  in init_syscall_trace()
1280 		return -ENOMEM;  in init_syscall_trace()
1282 	id = trace_event_raw_init(call);  in init_syscall_trace()
1284 	if (id < 0) {  in init_syscall_trace()
1286 		return id;  in init_syscall_trace()
1289 	return id;  in init_syscall_trace()
1292 static struct trace_event_fields __refdata syscall_enter_fields_array[] = {
1300 	.trace		= print_syscall_enter,
1304 	.trace		= print_syscall_exit,
1355 		meta->syscall_nr = i;  in init_ftrace_syscalls()
1371 static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
1372 static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
1373 static int sys_perf_refcount_enter;
1374 static int sys_perf_refcount_exit;
1376 static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,  in perf_call_bpf_enter()
1392 	param.syscall_nr = rec->nr;  in perf_call_bpf_enter()
1393 	for (i = 0; i < sys_data->nb_args; i++)  in perf_call_bpf_enter()
1394 		param.args[i] = rec->args[i];  in perf_call_bpf_enter()
1398 static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)  in perf_syscall_enter()  argument
1417 	 * buffer and per-cpu data require preemption to be disabled.  in perf_syscall_enter()
1435 	mayfault = sys_data->user_mask != 0;  in perf_syscall_enter()
1443 	head = this_cpu_ptr(sys_data->enter_event->perf_events);  in perf_syscall_enter()
1444 	valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);  in perf_syscall_enter()
1449 	size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);  in perf_syscall_enter()
1451 	size -= sizeof(u32);  in perf_syscall_enter()
1457 	rec->nr = syscall_nr;  in perf_syscall_enter()
1458 	memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args);  in perf_syscall_enter()
1464 	     !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) ||  in perf_syscall_enter()
1471 			      sys_data->enter_event->event.type, 1, regs,  in perf_syscall_enter()
1475 static int perf_sysenter_enable(struct trace_event_call *call)  in perf_sysenter_enable()
1477 	struct syscall_metadata *sys_data = call->data;  in perf_sysenter_enable()
1481 	num = sys_data->syscall_nr;  in perf_sysenter_enable()
1484 	if (sys_data->user_mask) {  in perf_sysenter_enable()
1492 			pr_info("event trace: Could not activate syscall entry trace point");  in perf_sysenter_enable()
1493 			if (sys_data->user_mask)  in perf_sysenter_enable()
1503 static void perf_sysenter_disable(struct trace_event_call *call)  in perf_sysenter_disable()
1505 	struct syscall_metadata *sys_data = call->data;  in perf_sysenter_disable()
1508 	num = sys_data->syscall_nr;  in perf_sysenter_disable()
1511 	sys_perf_refcount_enter--;  in perf_sysenter_disable()
1515 	if (sys_data->user_mask)  in perf_sysenter_disable()
1519 static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,  in perf_call_bpf_exit()
1531 	param.syscall_nr = rec->nr;  in perf_call_bpf_exit()
1532 	param.ret = rec->ret;  in perf_call_bpf_exit()
1536 static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)  in perf_syscall_exit()
1549 	 * buffer and per-cpu data require preemption to be disabled.  in perf_syscall_exit()
1564 	head = this_cpu_ptr(sys_data->exit_event->perf_events);  in perf_syscall_exit()
1565 	valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);  in perf_syscall_exit()
1571 	size -= sizeof(u32);  in perf_syscall_exit()
1577 	rec->nr = syscall_nr;  in perf_syscall_exit()
1578 	rec->ret = syscall_get_return_value(current, regs);  in perf_syscall_exit()
1581 	     !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) ||  in perf_syscall_exit()
1587 	perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,  in perf_syscall_exit()
1591 static int perf_sysexit_enable(struct trace_event_call *call)  in perf_sysexit_enable()
1595 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in perf_sysexit_enable()
1601 			pr_info("event trace: Could not activate syscall exit trace point");  in perf_sysexit_enable()
1610 static void perf_sysexit_disable(struct trace_event_call *call)  in perf_sysexit_disable()
1614 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in perf_sysexit_disable()
1617 	sys_perf_refcount_exit--;  in perf_sysexit_disable()
1625 static int syscall_enter_register(struct trace_event_call *event,  in syscall_enter_register()
1653 static int syscall_exit_register(struct trace_event_call *event,  in syscall_exit_register()