37  * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp
82 static int __insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)  in __insert_fprobe_node()  argument
91 		WRITE_ONCE(node->fp, fp);  in __insert_fprobe_node()
100 	if (READ_ONCE(node->fp) != NULL) {  in __delete_fprobe_node()
101 		WRITE_ONCE(node->fp, NULL);  in __delete_fprobe_node()
108 static bool fprobe_registered(struct fprobe *fp)  in fprobe_registered()  argument
113 	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];  in fprobe_registered()
116 		if (fph->fp == fp)  in fprobe_registered()
123 static int add_fprobe_hash(struct fprobe *fp)  in add_fprobe_hash()  argument
125 	struct fprobe_hlist *fph = fp->hlist_array;  in add_fprobe_hash()
133 	head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)];  in add_fprobe_hash()
134 	hlist_add_head_rcu(&fp->hlist_array->hlist, head);  in add_fprobe_hash()
138 static int del_fprobe_hash(struct fprobe *fp)  in del_fprobe_hash()  argument
140 	struct fprobe_hlist *fph = fp->hlist_array;  in del_fprobe_hash()
147 	if (!fprobe_registered(fp))  in del_fprobe_hash()
150 	fph->fp = NULL;  in del_fprobe_hash()
161 					struct fprobe *fp, unsigned int size_words)  in write_fprobe_header()  argument
164 			 !arch_fprobe_header_encodable(fp)))  in write_fprobe_header()
167 	*stack = arch_encode_fprobe_header(fp, size_words);  in write_fprobe_header()
172 					struct fprobe **fp, unsigned int *size_words)  in read_fprobe_header()  argument
174 	*fp = arch_decode_fprobe_header_fp(*stack);  in read_fprobe_header()
182 	struct fprobe *fp;  member
189 					struct fprobe *fp, unsigned int size_words)  in write_fprobe_header()  argument
196 	fph->fp = fp;  in write_fprobe_header()
202 					struct fprobe **fp, unsigned int *size_words)  in read_fprobe_header()  argument
206 	*fp = fph->fp;  in read_fprobe_header()
222 				   struct fprobe *fp, struct ftrace_regs *fregs,  in __fprobe_handler()  argument
225 	if (!fp->entry_handler)  in __fprobe_handler()
228 	return fp->entry_handler(fp, ip, parent_ip, fregs, data);  in __fprobe_handler()
232 					  struct fprobe *fp, struct ftrace_regs *fregs,  in __fprobe_kprobe_handler()  argument
243 		fp->nmissed++;  in __fprobe_kprobe_handler()
248 	ret = __fprobe_handler(ip, parent_ip, fp, fregs, data);  in __fprobe_kprobe_handler()
318 	struct fprobe *fp;  in fprobe_ftrace_entry()  local
336 		fp = READ_ONCE(node->fp);  in fprobe_ftrace_entry()
337 		if (unlikely(!fp || fprobe_disabled(fp) || fp->exit_handler))  in fprobe_ftrace_entry()
340 		if (fprobe_shared_with_kprobes(fp))  in fprobe_ftrace_entry()
341 			__fprobe_kprobe_handler(ip, parent_ip, fp, fregs, NULL);  in fprobe_ftrace_entry()
343 			__fprobe_handler(ip, parent_ip, fp, fregs, NULL);  in fprobe_ftrace_entry()
399 static bool fprobe_is_ftrace(struct fprobe *fp)  in fprobe_is_ftrace()  argument
401 	return !fp->exit_handler;  in fprobe_is_ftrace()
405 static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)  in insert_fprobe_node()  argument
411 	ret = __insert_fprobe_node(node, fp);  in insert_fprobe_node()
413 		if (fprobe_is_ftrace(fp))  in insert_fprobe_node()
424 	struct fprobe *fp;  in delete_fprobe_node()  local
428 	fp = READ_ONCE(node->fp);  in delete_fprobe_node()
429 	if (fp) {  in delete_fprobe_node()
430 		if (fprobe_is_ftrace(fp))  in delete_fprobe_node()
442 	struct fprobe *fp;  in fprobe_exists_on_hash()  local
453 		fp = READ_ONCE(node->fp);  in fprobe_exists_on_hash()
454 		if (likely(fp)) {  in fprobe_exists_on_hash()
455 			if ((!ftrace && fp->exit_handler) ||  in fprobe_exists_on_hash()
456 			    (ftrace && !fp->exit_handler))  in fprobe_exists_on_hash()
488 static bool fprobe_is_ftrace(struct fprobe *fp)  in fprobe_is_ftrace()  argument
494 static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp)  in insert_fprobe_node()  argument
500 	ret = __insert_fprobe_node(node, fp);  in insert_fprobe_node()
509 	struct fprobe *fp;  in delete_fprobe_node()  local
513 	fp = READ_ONCE(node->fp);  in delete_fprobe_node()
514 	if (fp)  in delete_fprobe_node()
523 	struct fprobe *fp;  in fprobe_exists_on_hash()  local
530 	/* We only need to check fp is there. */  in fprobe_exists_on_hash()
534 		fp = READ_ONCE(node->fp);  in fprobe_exists_on_hash()
535 		if (likely(fp))  in fprobe_exists_on_hash()
563 	struct fprobe *fp;  in fprobe_fgraph_entry()  local
575 		fp = READ_ONCE(node->fp);  in fprobe_fgraph_entry()
576 		if (!fp || !fp->exit_handler)  in fprobe_fgraph_entry()
583 			FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);  in fprobe_fgraph_entry()
591 				fp = READ_ONCE(node->fp);  in fprobe_fgraph_entry()
592 				if (fp && !fprobe_disabled(fp) && !fprobe_is_ftrace(fp))  in fprobe_fgraph_entry()
593 					fp->nmissed++;  in fprobe_fgraph_entry()
611 		fp = READ_ONCE(node->fp);  in fprobe_fgraph_entry()
612 		if (unlikely(!fp || fprobe_disabled(fp) || fprobe_is_ftrace(fp)))  in fprobe_fgraph_entry()
615 		data_size = fp->entry_data_size;  in fprobe_fgraph_entry()
616 		if (data_size && fp->exit_handler)  in fprobe_fgraph_entry()
621 		if (fprobe_shared_with_kprobes(fp))  in fprobe_fgraph_entry()
622 			ret = __fprobe_kprobe_handler(func, ret_ip, fp, fregs, data);  in fprobe_fgraph_entry()
624 			ret = __fprobe_handler(func, ret_ip, fp, fregs, data);  in fprobe_fgraph_entry()
627 		if (!ret && fp->exit_handler) {  in fprobe_fgraph_entry()
630 			if (write_fprobe_header(&fgraph_data[used], fp, size_words))  in fprobe_fgraph_entry()
646 	struct fprobe *fp;  in fprobe_return()  local
660 		read_fprobe_header(&fgraph_data[curr], &fp, &size);  in fprobe_return()
661 		if (!fp)  in fprobe_return()
664 		if (fprobe_registered(fp) && !fprobe_disabled(fp)) {  in fprobe_return()
667 			fp->exit_handler(fp, trace->func, ret_ip, fregs,  in fprobe_return()
873 static void fprobe_fail_cleanup(struct fprobe *fp)  in fprobe_fail_cleanup()  argument
875 	kfree(fp->hlist_array);  in fprobe_fail_cleanup()
876 	fp->hlist_array = NULL;  in fprobe_fail_cleanup()
880 static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num)  in fprobe_init()  argument
886 	if (!fp || !addrs || num <= 0)  in fprobe_init()
889 	size = ALIGN(fp->entry_data_size, sizeof(long));  in fprobe_init()
892 	fp->entry_data_size = size;  in fprobe_init()
898 	fp->nmissed = 0;  in fprobe_init()
901 	fp->hlist_array = hlist_array;  in fprobe_init()
902 	hlist_array->fp = fp;  in fprobe_init()
906 			fprobe_fail_cleanup(fp);  in fprobe_init()
923  * @fp: A fprobe data structure to be registered.
927  * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
930  * Return 0 if @fp is registered successfully, -errno if not.
932 int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter)  in register_fprobe()  argument
938 	if (!fp || !filter)  in register_fprobe()
957 	ret = register_fprobe_ips(fp, addrs, ret);  in register_fprobe()
967 static int unregister_fprobe_nolock(struct fprobe *fp);
971  * @fp: A fprobe data structure to be registered.
975  * Register @fp to ftrace for enabling the probe on the address given by @addrs.
980  * Return 0 if @fp is registered successfully, -errno if not.
982 int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num)  in register_fprobe_ips()  argument
988 	if (fprobe_registered(fp))  in register_fprobe_ips()
991 	ret = fprobe_init(fp, addrs, num);  in register_fprobe_ips()
995 	if (fprobe_is_ftrace(fp))  in register_fprobe_ips()
1000 		fprobe_fail_cleanup(fp);  in register_fprobe_ips()
1004 	hlist_array = fp->hlist_array;  in register_fprobe_ips()
1005 	ret = add_fprobe_hash(fp);  in register_fprobe_ips()
1007 		ret = insert_fprobe_node(&hlist_array->array[i], fp);  in register_fprobe_ips()
1010 		unregister_fprobe_nolock(fp);  in register_fprobe_ips()
1021  * @fp: A fprobe data structure to be registered.
1025  * Register @fp to the symbols given by @syms array. This will be useful if
1028  * Return 0 if @fp is registered successfully, -errno if not.
1030 int register_fprobe_syms(struct fprobe *fp, const char **syms, int num)  in register_fprobe_syms()  argument
1035 	if (!fp || !syms || num <= 0)  in register_fprobe_syms()
1042 	ret = register_fprobe_ips(fp, addrs, num);  in register_fprobe_syms()
1050 bool fprobe_is_registered(struct fprobe *fp)  in fprobe_is_registered()  argument
1052 	if (!fp || !fp->hlist_array)  in fprobe_is_registered()
1057 static int unregister_fprobe_nolock(struct fprobe *fp)  in unregister_fprobe_nolock()  argument
1059 	struct fprobe_hlist *hlist_array = fp->hlist_array;  in unregister_fprobe_nolock()
1078 						    fprobe_is_ftrace(fp)))  in unregister_fprobe_nolock()
1081 	del_fprobe_hash(fp);  in unregister_fprobe_nolock()
1083 	if (fprobe_is_ftrace(fp))  in unregister_fprobe_nolock()
1089 	fp->hlist_array = NULL;  in unregister_fprobe_nolock()
1097  * @fp: A fprobe data structure to be unregistered.
1101  * Return 0 if @fp is unregistered successfully, -errno if not.
1103 int unregister_fprobe(struct fprobe *fp)  in unregister_fprobe()  argument
1106 	if (!fp || !fprobe_registered(fp))  in unregister_fprobe()
1109 	return unregister_fprobe_nolock(fp);  in unregister_fprobe()