1 // SPDX-License-Identifier: GPL-2.0
28 	struct syscall_metadata *entry = call->data;  in syscall_get_enter_fields()
30 	return &entry->enter_fields;  in syscall_get_enter_fields()
72 		return -1;  in trace_get_syscall_nr()
100 		if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))  in find_syscall_meta()
125 	return entry->name;  in get_syscall_name()
128 /* Added to user strings or arrays when max limit is reached */
140 	ptr = (void *)trace->args + sizeof(long) * entry->nb_args + offset;  in get_dynamic_len_ptr()
180 	trace_seq_printf(s, "%s(", entry->name);  in sys_enter_openat_print()
182 	for (int i = 0; !done && i < entry->nb_args; i++) {  in sys_enter_openat_print()
192 			bits = trace->args[2];  in sys_enter_openat_print()
218 			trace_seq_printf(s, "%s: 0%03o", entry->args[i],  in sys_enter_openat_print()
219 					 (unsigned int)trace->args[i]);  in sys_enter_openat_print()
223 		trace_seq_printf(s, "%s: %lu", entry->args[i],  in sys_enter_openat_print()
224 				 trace->args[i]);  in sys_enter_openat_print()
226 		if (!(BIT(i) & entry->user_mask))  in sys_enter_openat_print()
244 	struct trace_array *tr = iter->tr;  in print_syscall_enter()
245 	struct trace_seq *s = &iter->seq;  in print_syscall_enter()
246 	struct trace_entry *ent = iter->ent;  in print_syscall_enter()
254 	syscall = trace->nr;  in print_syscall_enter()
260 	if (entry->enter_event->event.type != ent->type) {  in print_syscall_enter()
265 	switch (entry->syscall_nr) {  in print_syscall_enter()
267 		if (!tr || !(tr->trace_flags & TRACE_ITER(VERBOSE)))  in print_syscall_enter()
274 	trace_seq_printf(s, "%s(", entry->name);  in print_syscall_enter()
276 	for (i = 0; i < entry->nb_args; i++) {  in print_syscall_enter()
287 		if (tr && tr->trace_flags & TRACE_ITER(VERBOSE))  in print_syscall_enter()
288 			trace_seq_printf(s, "%s ", entry->types[i]);  in print_syscall_enter()
291 		if (trace->args[i] < 10)  in print_syscall_enter()
292 			trace_seq_printf(s, "%s: %lu", entry->args[i],  in print_syscall_enter()
293 					 trace->args[i]);  in print_syscall_enter()
295 			trace_seq_printf(s, "%s: 0x%lx", entry->args[i],  in print_syscall_enter()
296 					 trace->args[i]);  in print_syscall_enter()
298 		if (!(BIT(i) & entry->user_mask))  in print_syscall_enter()
303 		if (entry->user_arg_size < 0 || entry->user_arg_is_str) {  in print_syscall_enter()
308 		val = trace->args[entry->user_arg_size];  in print_syscall_enter()
352 	struct trace_seq *s = &iter->seq;  in print_syscall_exit()
353 	struct trace_entry *ent = iter->ent;  in print_syscall_exit()
359 	syscall = trace->nr;  in print_syscall_exit()
367 	if (entry->exit_event->event.type != ent->type) {  in print_syscall_exit()
372 	trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,  in print_syscall_exit()
373 				trace->ret);  in print_syscall_exit()
385 #define LEN_OR_ZERO (len ? len - pos : 0)
395 			" ((unsigned long)(REC->dfd)),");  in sys_enter_openat_print_fmt()
397 			" ((unsigned long)(REC->filename)),");  in sys_enter_openat_print_fmt()
401 			" (REC->flags & ~3) && !(REC->flags & 3) ? \"O_RDONLY|\" : \"\", ");  in sys_enter_openat_print_fmt()
403 			" REC->flags ? __print_flags(REC->flags, \"|\", ");  in sys_enter_openat_print_fmt()
436 			" ((unsigned long)(REC->mode))");  in sys_enter_openat_print_fmt()
443 	bool is_string = entry->user_arg_is_str;  in __set_enter_print_fmt()
447 	switch (entry->syscall_nr) {  in __set_enter_print_fmt()
455 	for (i = 0; i < entry->nb_args; i++) {  in __set_enter_print_fmt()
459 				entry->args[i], sizeof(unsigned long));  in __set_enter_print_fmt()
461 		if (!(BIT(i) & entry->user_mask))  in __set_enter_print_fmt()
465 		if (entry->user_arg_size < 0 || is_string)  in __set_enter_print_fmt()
472 	for (i = 0; i < entry->nb_args; i++) {  in __set_enter_print_fmt()
474 				", ((unsigned long)(REC->%s))", entry->args[i]);  in __set_enter_print_fmt()
475 		if (!(BIT(i) & entry->user_mask))  in __set_enter_print_fmt()
478 		if (entry->user_arg_size < 0 || is_string) {  in __set_enter_print_fmt()
480 					entry->args[i]);  in __set_enter_print_fmt()
483 					entry->args[i]);  in __set_enter_print_fmt()
497 	struct syscall_metadata *entry = call->data;  in set_syscall_print_fmt()
499 	if (entry->enter_event != call) {  in set_syscall_print_fmt()
500 		call->print_fmt = "\"0x%lx\", REC->ret";  in set_syscall_print_fmt()
509 		return -ENOMEM;  in set_syscall_print_fmt()
513 	call->print_fmt = print_fmt;  in set_syscall_print_fmt()
520 	struct syscall_metadata *entry = call->data;  in free_syscall_print_fmt()
522 	if (entry->enter_event == call)  in free_syscall_print_fmt()
523 		kfree(call->print_fmt);  in free_syscall_print_fmt()
529 	struct syscall_metadata *meta = call->data;  in syscall_enter_define_fields()
537 	for (i = 0; i < meta->nb_args; i++) {  in syscall_enter_define_fields()
538 		ret = trace_define_field(call, meta->types[i],  in syscall_enter_define_fields()
539 					 meta->args[i], offset,  in syscall_enter_define_fields()
547 	if (ret || !meta->user_mask)  in syscall_enter_define_fields()
550 	mask = meta->user_mask;  in syscall_enter_define_fields()
553 		int idx = ffs(mask) - 1;  in syscall_enter_define_fields()
562 		len = strlen(meta->args[idx]) + sizeof("___val");  in syscall_enter_define_fields()
565 			meta->user_mask = 0;  in syscall_enter_define_fields()
566 			return -ENOMEM;  in syscall_enter_define_fields()
569 		snprintf(arg, len, "__%s_val", meta->args[idx]);  in syscall_enter_define_fields()
616 		trace_user_fault_get(&syscall_buffer->buf);  in syscall_fault_buffer_enable()
622 		return -ENOMEM;  in syscall_fault_buffer_enable()
624 	ret = trace_user_fault_init(&sbuf->buf, SYSCALL_FAULT_BUF_SZ);  in syscall_fault_buffer_enable()
640 	trace_user_fault_destroy(&sbuf->buf);  in rcu_free_syscall_buffer()
651 	if (trace_user_fault_put(&sbuf->buf))  in syscall_fault_buffer_disable()
655 	call_rcu_tasks_trace(&sbuf->rcu, rcu_free_syscall_buffer);  in syscall_fault_buffer_disable()
670 	for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) {  in syscall_copy_user()
671 		ptr = (char __user *)args->ptr_array[i];  in syscall_copy_user()
673 		args->read[i] = ret;  in syscall_copy_user()
684 	for (int i = 0; i < args->uargs; i++, buf += SYSCALL_FAULT_ARG_SZ) {  in syscall_copy_user_array()
685 		ptr = (char __user *)args->ptr_array[i];  in syscall_copy_user_array()
687 		args->read[i] = ret ? -1 : size;  in syscall_copy_user_array()
699 	unsigned long mask = sys_data->user_mask;  in sys_fault_user()
700 	unsigned long size = SYSCALL_FAULT_ARG_SZ - 1;  in sys_fault_user()
717 	if (sys_data->user_arg_size >= 0) {  in sys_fault_user()
719 		size = args[sys_data->user_arg_size];  in sys_fault_user()
720 		if (size > SYSCALL_FAULT_ARG_SZ - 1)  in sys_fault_user()
721 			size = SYSCALL_FAULT_ARG_SZ - 1;  in sys_fault_user()
726 		int idx = ffs(mask) - 1;  in sys_fault_user()
740 		data_size[i] = -1; /* Denotes no pointer */  in sys_fault_user()
747 	buffer = trace_user_fault_read(&sbuf->buf, NULL, size,  in sys_fault_user()
760 		/* For strings, replace any non-printable characters with '.' */  in sys_fault_user()
799 		return -1;  in syscall_get_data()
830 	ptr = (void *)entry->args + sizeof(unsigned long) * sys_data->nb_args;  in syscall_put_data()
837 	val = (ptr - (void *)entry) + 4 * uargs;  in syscall_put_data()
842 			val += user_sizes[i - 1];  in syscall_put_data()
878 	 * buffer and per-cpu data require preemption to be disabled.  in ftrace_syscall_enter()
886 	trace_file = READ_ONCE(tr->enter_syscall_files[syscall_nr]);  in ftrace_syscall_enter()
898 	mayfault = sys_data->user_mask != 0;  in ftrace_syscall_enter()
906 				     &size, user_sizes, &uargs, tr->syscall_buf_sz) < 0)  in ftrace_syscall_enter()
910 	size += sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;  in ftrace_syscall_enter()
917 	entry->nr = syscall_nr;  in ftrace_syscall_enter()
919 	memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args);  in ftrace_syscall_enter()
938 	 * buffer and per-cpu data require preemption to be disabled.  in ftrace_syscall_exit()
947 	trace_file = READ_ONCE(tr->exit_syscall_files[syscall_nr]);  in ftrace_syscall_exit()
963 	entry->nr = syscall_nr;  in ftrace_syscall_exit()
964 	entry->ret = syscall_get_return_value(current, regs);  in ftrace_syscall_exit()
972 	struct syscall_metadata *sys_data = call->data;  in reg_event_syscall_enter()
973 	struct trace_array *tr = file->tr;  in reg_event_syscall_enter()
975 	int num;  in reg_event_syscall_enter()  local
977 	num = sys_data->syscall_nr;  in reg_event_syscall_enter()
978 	if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))  in reg_event_syscall_enter()
979 		return -ENOSYS;  in reg_event_syscall_enter()
981 	if (sys_data->user_mask) {  in reg_event_syscall_enter()
986 	if (!tr->sys_refcount_enter) {  in reg_event_syscall_enter()
989 			if (sys_data->user_mask)  in reg_event_syscall_enter()
994 	WRITE_ONCE(tr->enter_syscall_files[num], file);  in reg_event_syscall_enter()
995 	tr->sys_refcount_enter++;  in reg_event_syscall_enter()
1002 	struct syscall_metadata *sys_data = call->data;  in unreg_event_syscall_enter()
1003 	struct trace_array *tr = file->tr;  in unreg_event_syscall_enter()
1004 	int num;  in unreg_event_syscall_enter()  local
1006 	num = sys_data->syscall_nr;  in unreg_event_syscall_enter()
1007 	if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))  in unreg_event_syscall_enter()
1010 	tr->sys_refcount_enter--;  in unreg_event_syscall_enter()
1011 	WRITE_ONCE(tr->enter_syscall_files[num], NULL);  in unreg_event_syscall_enter()
1012 	if (!tr->sys_refcount_enter)  in unreg_event_syscall_enter()
1014 	if (sys_data->user_mask)  in unreg_event_syscall_enter()
1021 	struct trace_array *tr = file->tr;  in reg_event_syscall_exit()
1023 	int num;  in reg_event_syscall_exit()  local
1025 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in reg_event_syscall_exit()
1026 	if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))  in reg_event_syscall_exit()
1027 		return -ENOSYS;  in reg_event_syscall_exit()
1029 	if (!tr->sys_refcount_exit)  in reg_event_syscall_exit()
1032 		WRITE_ONCE(tr->exit_syscall_files[num], file);  in reg_event_syscall_exit()
1033 		tr->sys_refcount_exit++;  in reg_event_syscall_exit()
1042 	struct trace_array *tr = file->tr;  in unreg_event_syscall_exit()
1043 	int num;  in unreg_event_syscall_exit()  local
1045 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in unreg_event_syscall_exit()
1046 	if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))  in unreg_event_syscall_exit()
1049 	tr->sys_refcount_exit--;  in unreg_event_syscall_exit()
1050 	WRITE_ONCE(tr->exit_syscall_files[num], NULL);  in unreg_event_syscall_exit()
1051 	if (!tr->sys_refcount_exit)  in unreg_event_syscall_exit()
1063 	struct syscall_metadata *sys_data = call->data;  in check_faultable_syscall()
1067 	if (sys_data->enter_event != call)  in check_faultable_syscall()
1070 	sys_data->user_arg_size = -1;  in check_faultable_syscall()
1079 		sys_data->user_mask = BIT(1);  in check_faultable_syscall()
1080 		sys_data->user_arg_size = 2;  in check_faultable_syscall()
1085 		sys_data->user_mask = BIT(0);  in check_faultable_syscall()
1086 		sys_data->user_arg_size = 1;  in check_faultable_syscall()
1087 		sys_data->user_arg_is_str = 1;  in check_faultable_syscall()
1092 		sys_data->user_mask = BIT(4);  in check_faultable_syscall()
1093 		sys_data->user_arg_size = 3;  in check_faultable_syscall()
1094 		sys_data->user_arg_is_str = 1;  in check_faultable_syscall()
1156 		sys_data->user_mask = BIT(0);  in check_faultable_syscall()
1194 		sys_data->user_mask = BIT(1);  in check_faultable_syscall()
1199 		sys_data->user_mask = BIT(2);  in check_faultable_syscall()
1203 		sys_data->user_mask = BIT(4);  in check_faultable_syscall()
1226 		sys_data->user_mask = BIT(0) | BIT(1);  in check_faultable_syscall()
1230 		sys_data->user_mask = BIT(0) | BIT(2);  in check_faultable_syscall()
1243 		sys_data->user_mask = BIT(1) | BIT(3);  in check_faultable_syscall()
1246 		sys_data->user_mask = BIT(0) | BIT(1) | BIT(2);  in check_faultable_syscall()
1249 		sys_data->user_mask = 0;  in check_faultable_syscall()
1253 	if (sys_data->user_arg_size < 0)  in check_faultable_syscall()
1260 	mask = sys_data->user_mask;  in check_faultable_syscall()
1261 	if (WARN_ON(mask & (mask - 1)))  in check_faultable_syscall()
1262 		sys_data->user_arg_size = -1;  in check_faultable_syscall()
1268 	int num;  in init_syscall_trace()  local
1270 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in init_syscall_trace()
1271 	if (num < 0 || num >= NR_syscalls) {  in init_syscall_trace()
1273 				((struct syscall_metadata *)call->data)->name);  in init_syscall_trace()
1274 		return -ENOSYS;  in init_syscall_trace()
1277 	check_faultable_syscall(call, num);  in init_syscall_trace()
1280 		return -ENOMEM;  in init_syscall_trace()
1371 		meta->syscall_nr = i;  in init_ftrace_syscalls()
1410 	for (i = 0; i < sys_data->nb_args; i++)  in perf_call_bpf_enter()
1446 	 * Run BPF program in faultable context before per-cpu buffer  in perf_syscall_enter()
1449 	valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);  in perf_syscall_enter()
1451 	    !perf_call_bpf_enter(sys_data->enter_event, sys_data,  in perf_syscall_enter()
1456 	 * Per-cpu ring buffer and perf event list operations require  in perf_syscall_enter()
1461 	head = this_cpu_ptr(sys_data->enter_event->perf_events);  in perf_syscall_enter()
1466 	mayfault = sys_data->user_mask != 0;  in perf_syscall_enter()
1474 		head = this_cpu_ptr(sys_data->enter_event->perf_events);  in perf_syscall_enter()
1480 	size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);  in perf_syscall_enter()
1482 	size -= sizeof(u32);  in perf_syscall_enter()
1488 	rec->nr = syscall_nr;  in perf_syscall_enter()
1489 	memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args);  in perf_syscall_enter()
1495 			      sys_data->enter_event->event.type, 1, regs,  in perf_syscall_enter()
1501 	struct syscall_metadata *sys_data = call->data;  in perf_sysenter_enable()
1502 	int num;  in perf_sysenter_enable()  local
1505 	num = sys_data->syscall_nr;  in perf_sysenter_enable()
1508 	if (sys_data->user_mask) {  in perf_sysenter_enable()
1517 			if (sys_data->user_mask)  in perf_sysenter_enable()
1522 	set_bit(num, enabled_perf_enter_syscalls);  in perf_sysenter_enable()
1529 	struct syscall_metadata *sys_data = call->data;  in perf_sysenter_disable()
1530 	int num;  in perf_sysenter_disable()  local
1532 	num = sys_data->syscall_nr;  in perf_sysenter_disable()
1535 	sys_perf_refcount_enter--;  in perf_sysenter_disable()
1536 	clear_bit(num, enabled_perf_enter_syscalls);  in perf_sysenter_disable()
1539 	if (sys_data->user_mask)  in perf_sysenter_disable()
1584 	 * Run BPF program in faultable context before per-cpu buffer  in perf_syscall_exit()
1587 	valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);  in perf_syscall_exit()
1589 	    !perf_call_bpf_exit(sys_data->exit_event, syscall_nr,  in perf_syscall_exit()
1594 	 * Per-cpu ring buffer and perf event list operations require  in perf_syscall_exit()
1599 	head = this_cpu_ptr(sys_data->exit_event->perf_events);  in perf_syscall_exit()
1605 	size -= sizeof(u32);  in perf_syscall_exit()
1611 	rec->nr = syscall_nr;  in perf_syscall_exit()
1612 	rec->ret = syscall_get_return_value(current, regs);  in perf_syscall_exit()
1614 	perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,  in perf_syscall_exit()
1620 	int num;  in perf_sysexit_enable()  local
1622 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in perf_sysexit_enable()
1632 	set_bit(num, enabled_perf_exit_syscalls);  in perf_sysexit_enable()
1639 	int num;  in perf_sysexit_disable()  local
1641 	num = ((struct syscall_metadata *)call->data)->syscall_nr;  in perf_sysexit_disable()
1644 	sys_perf_refcount_exit--;  in perf_sysexit_disable()
1645 	clear_bit(num, enabled_perf_exit_syscalls);  in perf_sysexit_disable()