201 typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
209 static struct user_event_mm *user_event_mm_get_all(struct user_event *user);
211 static int destroy_user_event(struct user_event *user);
212 static bool user_fields_match(struct user_event *user, int argc,
231 static struct user_event *user_event_get(struct user_event *user)  in user_event_get()  argument
233 	refcount_inc(&user->refcnt);  in user_event_get()
235 	return user;  in user_event_get()
240 	struct user_event *user = container_of(  in delayed_destroy_user_event()  local
245 	if (!refcount_dec_and_test(&user->refcnt))  in delayed_destroy_user_event()
248 	if (destroy_user_event(user)) {  in delayed_destroy_user_event()
256 		refcount_set(&user->refcnt, 1);  in delayed_destroy_user_event()
262 static void user_event_put(struct user_event *user, bool locked)  in user_event_put()  argument
266 	if (unlikely(!user))  in user_event_put()
282 		delete = refcount_dec_and_mutex_lock(&user->refcnt, &event_mutex);  in user_event_put()
285 		delete = refcount_dec_and_test(&user->refcnt);  in user_event_put()
298 	if (user->reg_flags & USER_EVENT_REG_PERSIST) {  in user_event_put()
309 	INIT_WORK(&user->put_work, delayed_destroy_user_event);  in user_event_put()
322 	refcount_set(&user->refcnt, 1);  in user_event_put()
324 	if (WARN_ON_ONCE(!schedule_work(&user->put_work))) {  in user_event_put()
456 		struct user_event *user = enabler->event;  in user_event_enabler_fault_fixup()  local
459 			mm->mm, (unsigned long long)uaddr, EVENT_NAME(user));  in user_event_enabler_fault_fixup()
592 static void user_event_enabler_update(struct user_event *user)  in user_event_enabler_update()  argument
612 	mm = user_event_mm_get_all(user);  in user_event_enabler_update()
619 			if (enabler->event == user) {  in user_event_enabler_update()
664 static struct user_event_mm *user_event_mm_get_all(struct user_event *user)  in user_event_mm_get_all()  argument
692 			if (enabler->event == user) {  in user_event_mm_get_all()
882 *user_event_enabler_create(struct user_reg *reg, struct user_event *user,  in user_event_enabler_create()  argument
900 	enabler->event = user;  in user_event_enabler_create()
928 		user_event_get(user);  in user_event_enabler_create()
949 bool user_event_last_ref(struct user_event *user)  in user_event_last_ref()  argument
953 	if (user->reg_flags & USER_EVENT_REG_PERSIST)  in user_event_last_ref()
956 	return refcount_read(&user->refcnt) == last;  in user_event_last_ref()
975 	struct user_event *user = (struct user_event *)call->data;  in user_event_get_fields()  local
977 	return &user->fields;  in user_event_get_fields()
1086 static void user_event_destroy_validators(struct user_event *user)  in user_event_destroy_validators()  argument
1089 	struct list_head *head = &user->validators;  in user_event_destroy_validators()
1097 static void user_event_destroy_fields(struct user_event *user)  in user_event_destroy_fields()  argument
1100 	struct list_head *head = &user->fields;  in user_event_destroy_fields()
1108 static int user_event_add_field(struct user_event *user, const char *type,  in user_event_add_field()  argument
1146 	list_add_tail(&validator->user_event_link, &user->validators);  in user_event_add_field()
1159 	list_add(&field->link, &user->fields);  in user_event_add_field()
1165 	user->min_size = (offset + size) - sizeof(struct trace_entry);  in user_event_add_field()
1174 static int user_event_parse_field(char *field, struct user_event *user,  in user_event_parse_field()  argument
1261 	return user_event_add_field(user, type, name, saved_offset, size,  in user_event_parse_field()
1265 static int user_event_parse_fields(struct user_event *user, char *args)  in user_event_parse_fields()  argument
1275 		ret = user_event_parse_field(field, user, &offset);  in user_event_parse_fields()
1386 static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)  in user_event_set_print_fmt()  argument
1389 	struct list_head *head = &user->fields;  in user_event_set_print_fmt()
1420 static int user_event_create_print_fmt(struct user_event *user)  in user_event_create_print_fmt()  argument
1425 	len = user_event_set_print_fmt(user, NULL, 0);  in user_event_create_print_fmt()
1432 	user_event_set_print_fmt(user, print_fmt, len);  in user_event_create_print_fmt()
1434 	user->call.print_fmt = print_fmt;  in user_event_create_print_fmt()
1450 static int user_event_set_call_visible(struct user_event *user, bool visible)  in user_event_set_call_visible()  argument
1475 		ret = trace_add_event_call(&user->call);  in user_event_set_call_visible()
1477 		ret = trace_remove_event_call(&user->call);  in user_event_set_call_visible()
1485 static int destroy_user_event(struct user_event *user)  in destroy_user_event()  argument
1492 	user_event_destroy_fields(user);  in destroy_user_event()
1494 	ret = user_event_set_call_visible(user, false);  in destroy_user_event()
1499 	dyn_event_remove(&user->devent);  in destroy_user_event()
1500 	hash_del(&user->node);  in destroy_user_event()
1502 	user_event_destroy_validators(user);  in destroy_user_event()
1505 	if (EVENT_NAME(user) != EVENT_TP_NAME(user))  in destroy_user_event()
1506 		kfree(EVENT_TP_NAME(user));  in destroy_user_event()
1508 	kfree(user->call.print_fmt);  in destroy_user_event()
1509 	kfree(EVENT_NAME(user));  in destroy_user_event()
1510 	kfree(user);  in destroy_user_event()
1524 	struct user_event *user;  in find_user_event()  local
1529 	hash_for_each_possible(group->register_table, user, node, key) {  in find_user_event()
1535 		if (EVENT_MULTI_FORMAT(flags) != EVENT_MULTI_FORMAT(user->reg_flags))  in find_user_event()
1538 		if (strcmp(EVENT_NAME(user), name))  in find_user_event()
1541 		if (user_fields_match(user, argc, argv))  in find_user_event()
1542 			return user_event_get(user);  in find_user_event()
1554 static int user_event_validate(struct user_event *user, void *data, int len)  in user_event_validate()  argument
1556 	struct list_head *head = &user->validators;  in user_event_validate()
1590 static void user_event_ftrace(struct user_event *user, struct iov_iter *i,  in user_event_ftrace()  argument
1614 	if (!list_empty(&user->validators) &&  in user_event_ftrace()
1615 	    unlikely(user_event_validate(user, entry, size)))  in user_event_ftrace()
1631 static void user_event_perf(struct user_event *user, struct iov_iter *i,  in user_event_perf()  argument
1636 	perf_head = this_cpu_ptr(user->call.perf_events);  in user_event_perf()
1655 		if (!list_empty(&user->validators) &&  in user_event_perf()
1656 		    unlikely(user_event_validate(user, perf_entry, size)))  in user_event_perf()
1660 				      user->call.event.type, 1, regs,  in user_event_perf()
1674 static void update_enable_bit_for(struct user_event *user)  in update_enable_bit_for()  argument
1676 	struct tracepoint *tp = &user->tracepoint;  in update_enable_bit_for()
1705 	user->status = status;  in update_enable_bit_for()
1707 	user_event_enabler_update(user);  in update_enable_bit_for()
1717 	struct user_event *user = (struct user_event *)call->data;  in user_event_reg()  local
1720 	if (!user)  in user_event_reg()
1763 	user_event_get(user);  in user_event_reg()
1764 	update_enable_bit_for(user);  in user_event_reg()
1767 	update_enable_bit_for(user);  in user_event_reg()
1768 	user_event_put(user, true);  in user_event_reg()
1775 	struct user_event *user;  in user_event_create()  local
1800 	ret = user_event_parse_cmd(group, name, &user, USER_EVENT_REG_PERSIST);  in user_event_create()
1803 		user_event_put(user, false);  in user_event_create()
1815 	struct user_event *user = container_of(ev, struct user_event, devent);  in user_event_show()  local
1820 	seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));  in user_event_show()
1822 	head = trace_get_fields(&user->call);  in user_event_show()
1845 	struct user_event *user = container_of(ev, struct user_event, devent);  in user_event_is_busy()  local
1847 	return !user_event_last_ref(user);  in user_event_is_busy()
1852 	struct user_event *user = container_of(ev, struct user_event, devent);  in user_event_free()  local
1854 	if (!user_event_last_ref(user))  in user_event_free()
1857 	if (!user_event_capable(user->reg_flags))  in user_event_free()
1860 	return destroy_user_event(user);  in user_event_free()
1900 static bool user_fields_match(struct user_event *user, int argc,  in user_fields_match()  argument
1904 	struct list_head *head = &user->fields;  in user_fields_match()
1924 	struct user_event *user = container_of(ev, struct user_event, devent);  in user_event_match()  local
1927 	match = strcmp(EVENT_NAME(user), event) == 0;  in user_event_match()
1930 		match = strcmp(system, user->group->system_name) == 0 ||  in user_event_match()
1931 			strcmp(system, user->group->system_multi_name) == 0;  in user_event_match()
1935 		match = user_fields_match(user, argc, argv);  in user_event_match()
1948 static int user_event_trace_register(struct user_event *user)  in user_event_trace_register()  argument
1952 	ret = register_trace_event(&user->call.event);  in user_event_trace_register()
1957 	ret = user_event_set_call_visible(user, true);  in user_event_trace_register()
1960 		unregister_trace_event(&user->call.event);  in user_event_trace_register()
1965 static int user_event_set_tp_name(struct user_event *user)  in user_event_set_tp_name()  argument
1967 	lockdep_assert_held(&user->group->reg_mutex);  in user_event_set_tp_name()
1969 	if (EVENT_MULTI_FORMAT(user->reg_flags)) {  in user_event_set_tp_name()
1973 				       user->reg_name, user->group->multi_id);  in user_event_set_tp_name()
1978 		user->call.name = multi_name;  in user_event_set_tp_name()
1979 		user->tracepoint.name = multi_name;  in user_event_set_tp_name()
1982 		user->group->multi_id++;  in user_event_set_tp_name()
1985 		user->call.name = user->reg_name;  in user_event_set_tp_name()
1986 		user->tracepoint.name = user->reg_name;  in user_event_set_tp_name()
2075 	struct user_event *user;  in user_event_parse()  local
2097 	user = find_user_event(group, name, argc, (const char **)argv,  in user_event_parse()
2104 	if (IS_ERR(user))  in user_event_parse()
2105 		return PTR_ERR(user);  in user_event_parse()
2107 	if (user) {  in user_event_parse()
2108 		*newuser = user;  in user_event_parse()
2118 	user = kzalloc(sizeof(*user), GFP_KERNEL_ACCOUNT);  in user_event_parse()
2120 	if (!user)  in user_event_parse()
2123 	INIT_LIST_HEAD(&user->class.fields);  in user_event_parse()
2124 	INIT_LIST_HEAD(&user->fields);  in user_event_parse()
2125 	INIT_LIST_HEAD(&user->validators);  in user_event_parse()
2127 	user->group = group;  in user_event_parse()
2128 	user->reg_name = name;  in user_event_parse()
2129 	user->reg_flags = reg_flags;  in user_event_parse()
2131 	ret = user_event_set_tp_name(user);  in user_event_parse()
2136 	ret = user_event_parse_fields(user, args);  in user_event_parse()
2141 	ret = user_event_create_print_fmt(user);  in user_event_parse()
2146 	user->call.data = user;  in user_event_parse()
2147 	user->call.class = &user->class;  in user_event_parse()
2148 	user->call.flags = TRACE_EVENT_FL_TRACEPOINT;  in user_event_parse()
2149 	user->call.tp = &user->tracepoint;  in user_event_parse()
2150 	user->call.event.funcs = &user_event_funcs;  in user_event_parse()
2152 	if (EVENT_MULTI_FORMAT(user->reg_flags))  in user_event_parse()
2153 		user->class.system = group->system_multi_name;  in user_event_parse()
2155 		user->class.system = group->system_name;  in user_event_parse()
2157 	user->class.fields_array = user_event_fields_array;  in user_event_parse()
2158 	user->class.get_fields = user_event_get_fields;  in user_event_parse()
2159 	user->class.reg = user_event_reg;  in user_event_parse()
2160 	user->class.probe = user_event_ftrace;  in user_event_parse()
2162 	user->class.perf_probe = user_event_perf;  in user_event_parse()
2172 	ret = user_event_trace_register(user);  in user_event_parse()
2177 	if (user->reg_flags & USER_EVENT_REG_PERSIST) {  in user_event_parse()
2179 		refcount_set(&user->refcnt, 2);  in user_event_parse()
2182 		refcount_set(&user->refcnt, 1);  in user_event_parse()
2185 	dyn_event_init(&user->devent, &user_event_dops);  in user_event_parse()
2186 	dyn_event_add(&user->devent, &user->call);  in user_event_parse()
2187 	hash_add(group->register_table, &user->node, key);  in user_event_parse()
2192 	*newuser = user;  in user_event_parse()
2197 	user_event_destroy_fields(user);  in user_event_parse()
2198 	user_event_destroy_validators(user);  in user_event_parse()
2199 	kfree(user->call.print_fmt);  in user_event_parse()
2202 	if (EVENT_NAME(user) != EVENT_TP_NAME(user))  in user_event_parse()
2203 		kfree(EVENT_TP_NAME(user));  in user_event_parse()
2205 	kfree(user);  in user_event_parse()
2214 	struct user_event *user;  in delete_user_event()  local
2220 	hash_for_each_possible_safe(group->register_table, user, tmp, node, key) {  in delete_user_event()
2221 		if (strcmp(EVENT_NAME(user), name))  in delete_user_event()
2224 		if (!user_event_last_ref(user))  in delete_user_event()
2227 		if (!user_event_capable(user->reg_flags))  in delete_user_event()
2230 		ret = destroy_user_event(user);  in delete_user_event()
2246 	struct user_event *user = NULL;  in user_events_write_core()  local
2267 		user = refs->events[idx];  in user_events_write_core()
2271 	if (unlikely(user == NULL))  in user_events_write_core()
2274 	if (unlikely(i->count < user->min_size))  in user_events_write_core()
2277 	tp = &user->tracepoint;  in user_events_write_core()
2304 				probe_func(user, &copy, tpdata, &faulted);  in user_events_write_core()
2360 			       struct user_event *user)  in user_events_ref_add()  argument
2373 			if (refs->events[i] == user)  in user_events_ref_add()
2389 	new_refs->events[i] = user_event_get(user);  in user_events_ref_add()
2464 	struct user_event *user;  in user_events_ioctl_reg()  local
2493 	ret = user_event_parse_cmd(info->group, name, &user, reg.flags);  in user_events_ioctl_reg()
2500 	ret = user_events_ref_add(info, user);  in user_events_ioctl_reg()
2503 	user_event_put(user, false);  in user_events_ioctl_reg()
2521 	enabler = user_event_enabler_create(&reg, user, &write_result);  in user_events_ioctl_reg()
2782 	struct user_event *user;  in user_seq_show()  local
2791 	hash_for_each(group->register_table, i, user, node) {  in user_seq_show()
2792 		status = user->status;  in user_seq_show()
2794 		seq_printf(m, "%s", EVENT_TP_NAME(user));  in user_seq_show()