xref: /linux/kernel/trace/ftrace.c (revision fc6dfd5547794b0bf10790576a9d97443d975439)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Infrastructure for profiling code inserted by 'gcc -pg'.
4  *
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7  *
8  * Originally ported from the -rt patch by:
9  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10  *
11  * Based on code in the latency_tracer, that is:
12  *
13  *  Copyright (C) 2004-2006 Ingo Molnar
14  *  Copyright (C) 2004 Nadia Yvette Chambers
15  */
16 
17 #include <linux/stop_machine.h>
18 #include <linux/clocksource.h>
19 #include <linux/sched/task.h>
20 #include <linux/kallsyms.h>
21 #include <linux/security.h>
22 #include <linux/seq_file.h>
23 #include <linux/tracefs.h>
24 #include <linux/hardirq.h>
25 #include <linux/kthread.h>
26 #include <linux/uaccess.h>
27 #include <linux/bsearch.h>
28 #include <linux/module.h>
29 #include <linux/ftrace.h>
30 #include <linux/sysctl.h>
31 #include <linux/slab.h>
32 #include <linux/ctype.h>
33 #include <linux/sort.h>
34 #include <linux/list.h>
35 #include <linux/hash.h>
36 #include <linux/rcupdate.h>
37 #include <linux/kprobes.h>
38 
39 #include <trace/events/sched.h>
40 
41 #include <asm/sections.h>
42 #include <asm/setup.h>
43 
44 #include "ftrace_internal.h"
45 #include "trace_output.h"
46 #include "trace_stat.h"
47 
48 #define FTRACE_INVALID_FUNCTION		"__ftrace_invalid_address__"
49 
50 #define FTRACE_WARN_ON(cond)			\
51 	({					\
52 		int ___r = cond;		\
53 		if (WARN_ON(___r))		\
54 			ftrace_kill();		\
55 		___r;				\
56 	})
57 
58 #define FTRACE_WARN_ON_ONCE(cond)		\
59 	({					\
60 		int ___r = cond;		\
61 		if (WARN_ON_ONCE(___r))		\
62 			ftrace_kill();		\
63 		___r;				\
64 	})
65 
66 /* hash bits for specific function selection */
67 #define FTRACE_HASH_DEFAULT_BITS 10
68 #define FTRACE_HASH_MAX_BITS 12
69 
70 #ifdef CONFIG_DYNAMIC_FTRACE
71 #define INIT_OPS_HASH(opsname)	\
72 	.func_hash		= &opsname.local_hash,			\
73 	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
74 #else
75 #define INIT_OPS_HASH(opsname)
76 #endif
77 
78 enum {
79 	FTRACE_MODIFY_ENABLE_FL		= (1 << 0),
80 	FTRACE_MODIFY_MAY_SLEEP_FL	= (1 << 1),
81 };
82 
83 struct ftrace_ops ftrace_list_end __read_mostly = {
84 	.func		= ftrace_stub,
85 	.flags		= FTRACE_OPS_FL_STUB,
86 	INIT_OPS_HASH(ftrace_list_end)
87 };
88 
89 /* ftrace_enabled is a method to turn ftrace on or off */
90 int ftrace_enabled __read_mostly;
91 static int __maybe_unused last_ftrace_enabled;
92 
93 /* Current function tracing op */
94 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
95 /* What to set function_trace_op to */
96 static struct ftrace_ops *set_function_trace_op;
97 
98 static bool ftrace_pids_enabled(struct ftrace_ops *ops)
99 {
100 	struct trace_array *tr;
101 
102 	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
103 		return false;
104 
105 	tr = ops->private;
106 
107 	return tr->function_pids != NULL || tr->function_no_pids != NULL;
108 }
109 
110 static void ftrace_update_trampoline(struct ftrace_ops *ops);
111 
112 /*
113  * ftrace_disabled is set when an anomaly is discovered.
114  * ftrace_disabled is much stronger than ftrace_enabled.
115  */
116 static int ftrace_disabled __read_mostly;
117 
118 DEFINE_MUTEX(ftrace_lock);
119 
120 struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
121 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
122 struct ftrace_ops global_ops;
123 
124 /* Defined by vmlinux.lds.h see the comment above arch_ftrace_ops_list_func for details */
125 void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
126 			  struct ftrace_ops *op, struct ftrace_regs *fregs);
127 
128 static inline void ftrace_ops_init(struct ftrace_ops *ops)
129 {
130 #ifdef CONFIG_DYNAMIC_FTRACE
131 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
132 		mutex_init(&ops->local_hash.regex_lock);
133 		ops->func_hash = &ops->local_hash;
134 		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
135 	}
136 #endif
137 }
138 
139 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
140 			    struct ftrace_ops *op, struct ftrace_regs *fregs)
141 {
142 	struct trace_array *tr = op->private;
143 	int pid;
144 
145 	if (tr) {
146 		pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
147 		if (pid == FTRACE_PID_IGNORE)
148 			return;
149 		if (pid != FTRACE_PID_TRACE &&
150 		    pid != current->pid)
151 			return;
152 	}
153 
154 	op->saved_func(ip, parent_ip, op, fregs);
155 }
156 
157 static void ftrace_sync_ipi(void *data)
158 {
159 	/* Probably not needed, but do it anyway */
160 	smp_rmb();
161 }
162 
163 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
164 {
165 	/*
166 	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
167 	 * then it needs to call the list anyway.
168 	 */
169 	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
170 	    FTRACE_FORCE_LIST_FUNC)
171 		return ftrace_ops_list_func;
172 
173 	return ftrace_ops_get_func(ops);
174 }
175 
176 static void update_ftrace_function(void)
177 {
178 	ftrace_func_t func;
179 
180 	/*
181 	 * Prepare the ftrace_ops that the arch callback will use.
182 	 * If there's only one ftrace_ops registered, the ftrace_ops_list
183 	 * will point to the ops we want.
184 	 */
185 	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
186 						lockdep_is_held(&ftrace_lock));
187 
188 	/* If there's no ftrace_ops registered, just call the stub function */
189 	if (set_function_trace_op == &ftrace_list_end) {
190 		func = ftrace_stub;
191 
192 	/*
193 	 * If we are at the end of the list and this ops is
194 	 * recursion safe and not dynamic and the arch supports passing ops,
195 	 * then have the mcount trampoline call the function directly.
196 	 */
197 	} else if (rcu_dereference_protected(ftrace_ops_list->next,
198 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
199 		func = ftrace_ops_get_list_func(ftrace_ops_list);
200 
201 	} else {
202 		/* Just use the default ftrace_ops */
203 		set_function_trace_op = &ftrace_list_end;
204 		func = ftrace_ops_list_func;
205 	}
206 
207 	update_function_graph_func();
208 
209 	/* If there's no change, then do nothing more here */
210 	if (ftrace_trace_function == func)
211 		return;
212 
213 	/*
214 	 * If we are using the list function, it doesn't care
215 	 * about the function_trace_ops.
216 	 */
217 	if (func == ftrace_ops_list_func) {
218 		ftrace_trace_function = func;
219 		/*
220 		 * Don't even bother setting function_trace_ops,
221 		 * it would be racy to do so anyway.
222 		 */
223 		return;
224 	}
225 
226 #ifndef CONFIG_DYNAMIC_FTRACE
227 	/*
228 	 * For static tracing, we need to be a bit more careful.
229 	 * The function change takes affect immediately. Thus,
230 	 * we need to coordinate the setting of the function_trace_ops
231 	 * with the setting of the ftrace_trace_function.
232 	 *
233 	 * Set the function to the list ops, which will call the
234 	 * function we want, albeit indirectly, but it handles the
235 	 * ftrace_ops and doesn't depend on function_trace_op.
236 	 */
237 	ftrace_trace_function = ftrace_ops_list_func;
238 	/*
239 	 * Make sure all CPUs see this. Yes this is slow, but static
240 	 * tracing is slow and nasty to have enabled.
241 	 */
242 	synchronize_rcu_tasks_rude();
243 	/* Now all cpus are using the list ops. */
244 	function_trace_op = set_function_trace_op;
245 	/* Make sure the function_trace_op is visible on all CPUs */
246 	smp_wmb();
247 	/* Nasty way to force a rmb on all cpus */
248 	smp_call_function(ftrace_sync_ipi, NULL, 1);
249 	/* OK, we are all set to update the ftrace_trace_function now! */
250 #endif /* !CONFIG_DYNAMIC_FTRACE */
251 
252 	ftrace_trace_function = func;
253 }
254 
255 static void add_ftrace_ops(struct ftrace_ops __rcu **list,
256 			   struct ftrace_ops *ops)
257 {
258 	rcu_assign_pointer(ops->next, *list);
259 
260 	/*
261 	 * We are entering ops into the list but another
262 	 * CPU might be walking that list. We need to make sure
263 	 * the ops->next pointer is valid before another CPU sees
264 	 * the ops pointer included into the list.
265 	 */
266 	rcu_assign_pointer(*list, ops);
267 }
268 
269 static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
270 			     struct ftrace_ops *ops)
271 {
272 	struct ftrace_ops **p;
273 
274 	/*
275 	 * If we are removing the last function, then simply point
276 	 * to the ftrace_stub.
277 	 */
278 	if (rcu_dereference_protected(*list,
279 			lockdep_is_held(&ftrace_lock)) == ops &&
280 	    rcu_dereference_protected(ops->next,
281 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
282 		*list = &ftrace_list_end;
283 		return 0;
284 	}
285 
286 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
287 		if (*p == ops)
288 			break;
289 
290 	if (*p != ops)
291 		return -1;
292 
293 	*p = (*p)->next;
294 	return 0;
295 }
296 
297 static void ftrace_update_trampoline(struct ftrace_ops *ops);
298 
299 int __register_ftrace_function(struct ftrace_ops *ops)
300 {
301 	if (ops->flags & FTRACE_OPS_FL_DELETED)
302 		return -EINVAL;
303 
304 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
305 		return -EBUSY;
306 
307 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
308 	/*
309 	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
310 	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
311 	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
312 	 */
313 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
314 	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
315 		return -EINVAL;
316 
317 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
318 		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
319 #endif
320 	if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT))
321 		return -EBUSY;
322 
323 	if (!is_kernel_core_data((unsigned long)ops))
324 		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
325 
326 	add_ftrace_ops(&ftrace_ops_list, ops);
327 
328 	/* Always save the function, and reset at unregistering */
329 	ops->saved_func = ops->func;
330 
331 	if (ftrace_pids_enabled(ops))
332 		ops->func = ftrace_pid_func;
333 
334 	ftrace_update_trampoline(ops);
335 
336 	if (ftrace_enabled)
337 		update_ftrace_function();
338 
339 	return 0;
340 }
341 
342 int __unregister_ftrace_function(struct ftrace_ops *ops)
343 {
344 	int ret;
345 
346 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
347 		return -EBUSY;
348 
349 	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
350 
351 	if (ret < 0)
352 		return ret;
353 
354 	if (ftrace_enabled)
355 		update_ftrace_function();
356 
357 	ops->func = ops->saved_func;
358 
359 	return 0;
360 }
361 
362 static void ftrace_update_pid_func(void)
363 {
364 	struct ftrace_ops *op;
365 
366 	/* Only do something if we are tracing something */
367 	if (ftrace_trace_function == ftrace_stub)
368 		return;
369 
370 	do_for_each_ftrace_op(op, ftrace_ops_list) {
371 		if (op->flags & FTRACE_OPS_FL_PID) {
372 			op->func = ftrace_pids_enabled(op) ?
373 				ftrace_pid_func : op->saved_func;
374 			ftrace_update_trampoline(op);
375 		}
376 	} while_for_each_ftrace_op(op);
377 
378 	update_ftrace_function();
379 }
380 
381 #ifdef CONFIG_FUNCTION_PROFILER
382 struct ftrace_profile {
383 	struct hlist_node		node;
384 	unsigned long			ip;
385 	unsigned long			counter;
386 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
387 	unsigned long long		time;
388 	unsigned long long		time_squared;
389 #endif
390 };
391 
392 struct ftrace_profile_page {
393 	struct ftrace_profile_page	*next;
394 	unsigned long			index;
395 	struct ftrace_profile		records[];
396 };
397 
398 struct ftrace_profile_stat {
399 	atomic_t			disabled;
400 	struct hlist_head		*hash;
401 	struct ftrace_profile_page	*pages;
402 	struct ftrace_profile_page	*start;
403 	struct tracer_stat		stat;
404 };
405 
406 #define PROFILE_RECORDS_SIZE						\
407 	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
408 
409 #define PROFILES_PER_PAGE					\
410 	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
411 
412 static int ftrace_profile_enabled __read_mostly;
413 
414 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
415 static DEFINE_MUTEX(ftrace_profile_lock);
416 
417 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
418 
419 #define FTRACE_PROFILE_HASH_BITS 10
420 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
421 
422 static void *
423 function_stat_next(void *v, int idx)
424 {
425 	struct ftrace_profile *rec = v;
426 	struct ftrace_profile_page *pg;
427 
428 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
429 
430  again:
431 	if (idx != 0)
432 		rec++;
433 
434 	if ((void *)rec >= (void *)&pg->records[pg->index]) {
435 		pg = pg->next;
436 		if (!pg)
437 			return NULL;
438 		rec = &pg->records[0];
439 		if (!rec->counter)
440 			goto again;
441 	}
442 
443 	return rec;
444 }
445 
446 static void *function_stat_start(struct tracer_stat *trace)
447 {
448 	struct ftrace_profile_stat *stat =
449 		container_of(trace, struct ftrace_profile_stat, stat);
450 
451 	if (!stat || !stat->start)
452 		return NULL;
453 
454 	return function_stat_next(&stat->start->records[0], 0);
455 }
456 
457 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
458 /* function graph compares on total time */
459 static int function_stat_cmp(const void *p1, const void *p2)
460 {
461 	const struct ftrace_profile *a = p1;
462 	const struct ftrace_profile *b = p2;
463 
464 	if (a->time < b->time)
465 		return -1;
466 	if (a->time > b->time)
467 		return 1;
468 	else
469 		return 0;
470 }
471 #else
472 /* not function graph compares against hits */
473 static int function_stat_cmp(const void *p1, const void *p2)
474 {
475 	const struct ftrace_profile *a = p1;
476 	const struct ftrace_profile *b = p2;
477 
478 	if (a->counter < b->counter)
479 		return -1;
480 	if (a->counter > b->counter)
481 		return 1;
482 	else
483 		return 0;
484 }
485 #endif
486 
487 static int function_stat_headers(struct seq_file *m)
488 {
489 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
490 	seq_puts(m, "  Function                               "
491 		 "Hit    Time            Avg             s^2\n"
492 		    "  --------                               "
493 		 "---    ----            ---             ---\n");
494 #else
495 	seq_puts(m, "  Function                               Hit\n"
496 		    "  --------                               ---\n");
497 #endif
498 	return 0;
499 }
500 
501 static int function_stat_show(struct seq_file *m, void *v)
502 {
503 	struct ftrace_profile *rec = v;
504 	char str[KSYM_SYMBOL_LEN];
505 	int ret = 0;
506 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
507 	static struct trace_seq s;
508 	unsigned long long avg;
509 	unsigned long long stddev;
510 #endif
511 	mutex_lock(&ftrace_profile_lock);
512 
513 	/* we raced with function_profile_reset() */
514 	if (unlikely(rec->counter == 0)) {
515 		ret = -EBUSY;
516 		goto out;
517 	}
518 
519 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
520 	avg = div64_ul(rec->time, rec->counter);
521 	if (tracing_thresh && (avg < tracing_thresh))
522 		goto out;
523 #endif
524 
525 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
526 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
527 
528 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
529 	seq_puts(m, "    ");
530 
531 	/* Sample standard deviation (s^2) */
532 	if (rec->counter <= 1)
533 		stddev = 0;
534 	else {
535 		/*
536 		 * Apply Welford's method:
537 		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
538 		 */
539 		stddev = rec->counter * rec->time_squared -
540 			 rec->time * rec->time;
541 
542 		/*
543 		 * Divide only 1000 for ns^2 -> us^2 conversion.
544 		 * trace_print_graph_duration will divide 1000 again.
545 		 */
546 		stddev = div64_ul(stddev,
547 				  rec->counter * (rec->counter - 1) * 1000);
548 	}
549 
550 	trace_seq_init(&s);
551 	trace_print_graph_duration(rec->time, &s);
552 	trace_seq_puts(&s, "    ");
553 	trace_print_graph_duration(avg, &s);
554 	trace_seq_puts(&s, "    ");
555 	trace_print_graph_duration(stddev, &s);
556 	trace_print_seq(m, &s);
557 #endif
558 	seq_putc(m, '\n');
559 out:
560 	mutex_unlock(&ftrace_profile_lock);
561 
562 	return ret;
563 }
564 
565 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
566 {
567 	struct ftrace_profile_page *pg;
568 
569 	pg = stat->pages = stat->start;
570 
571 	while (pg) {
572 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
573 		pg->index = 0;
574 		pg = pg->next;
575 	}
576 
577 	memset(stat->hash, 0,
578 	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
579 }
580 
581 static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
582 {
583 	struct ftrace_profile_page *pg;
584 	int functions;
585 	int pages;
586 	int i;
587 
588 	/* If we already allocated, do nothing */
589 	if (stat->pages)
590 		return 0;
591 
592 	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
593 	if (!stat->pages)
594 		return -ENOMEM;
595 
596 #ifdef CONFIG_DYNAMIC_FTRACE
597 	functions = ftrace_update_tot_cnt;
598 #else
599 	/*
600 	 * We do not know the number of functions that exist because
601 	 * dynamic tracing is what counts them. With past experience
602 	 * we have around 20K functions. That should be more than enough.
603 	 * It is highly unlikely we will execute every function in
604 	 * the kernel.
605 	 */
606 	functions = 20000;
607 #endif
608 
609 	pg = stat->start = stat->pages;
610 
611 	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
612 
613 	for (i = 1; i < pages; i++) {
614 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
615 		if (!pg->next)
616 			goto out_free;
617 		pg = pg->next;
618 	}
619 
620 	return 0;
621 
622  out_free:
623 	pg = stat->start;
624 	while (pg) {
625 		unsigned long tmp = (unsigned long)pg;
626 
627 		pg = pg->next;
628 		free_page(tmp);
629 	}
630 
631 	stat->pages = NULL;
632 	stat->start = NULL;
633 
634 	return -ENOMEM;
635 }
636 
637 static int ftrace_profile_init_cpu(int cpu)
638 {
639 	struct ftrace_profile_stat *stat;
640 	int size;
641 
642 	stat = &per_cpu(ftrace_profile_stats, cpu);
643 
644 	if (stat->hash) {
645 		/* If the profile is already created, simply reset it */
646 		ftrace_profile_reset(stat);
647 		return 0;
648 	}
649 
650 	/*
651 	 * We are profiling all functions, but usually only a few thousand
652 	 * functions are hit. We'll make a hash of 1024 items.
653 	 */
654 	size = FTRACE_PROFILE_HASH_SIZE;
655 
656 	stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
657 
658 	if (!stat->hash)
659 		return -ENOMEM;
660 
661 	/* Preallocate the function profiling pages */
662 	if (ftrace_profile_pages_init(stat) < 0) {
663 		kfree(stat->hash);
664 		stat->hash = NULL;
665 		return -ENOMEM;
666 	}
667 
668 	return 0;
669 }
670 
671 static int ftrace_profile_init(void)
672 {
673 	int cpu;
674 	int ret = 0;
675 
676 	for_each_possible_cpu(cpu) {
677 		ret = ftrace_profile_init_cpu(cpu);
678 		if (ret)
679 			break;
680 	}
681 
682 	return ret;
683 }
684 
685 /* interrupts must be disabled */
686 static struct ftrace_profile *
687 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
688 {
689 	struct ftrace_profile *rec;
690 	struct hlist_head *hhd;
691 	unsigned long key;
692 
693 	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
694 	hhd = &stat->hash[key];
695 
696 	if (hlist_empty(hhd))
697 		return NULL;
698 
699 	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
700 		if (rec->ip == ip)
701 			return rec;
702 	}
703 
704 	return NULL;
705 }
706 
707 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
708 			       struct ftrace_profile *rec)
709 {
710 	unsigned long key;
711 
712 	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
713 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
714 }
715 
716 /*
717  * The memory is already allocated, this simply finds a new record to use.
718  */
719 static struct ftrace_profile *
720 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
721 {
722 	struct ftrace_profile *rec = NULL;
723 
724 	/* prevent recursion (from NMIs) */
725 	if (atomic_inc_return(&stat->disabled) != 1)
726 		goto out;
727 
728 	/*
729 	 * Try to find the function again since an NMI
730 	 * could have added it
731 	 */
732 	rec = ftrace_find_profiled_func(stat, ip);
733 	if (rec)
734 		goto out;
735 
736 	if (stat->pages->index == PROFILES_PER_PAGE) {
737 		if (!stat->pages->next)
738 			goto out;
739 		stat->pages = stat->pages->next;
740 	}
741 
742 	rec = &stat->pages->records[stat->pages->index++];
743 	rec->ip = ip;
744 	ftrace_add_profile(stat, rec);
745 
746  out:
747 	atomic_dec(&stat->disabled);
748 
749 	return rec;
750 }
751 
752 static void
753 function_profile_call(unsigned long ip, unsigned long parent_ip,
754 		      struct ftrace_ops *ops, struct ftrace_regs *fregs)
755 {
756 	struct ftrace_profile_stat *stat;
757 	struct ftrace_profile *rec;
758 	unsigned long flags;
759 
760 	if (!ftrace_profile_enabled)
761 		return;
762 
763 	local_irq_save(flags);
764 
765 	stat = this_cpu_ptr(&ftrace_profile_stats);
766 	if (!stat->hash || !ftrace_profile_enabled)
767 		goto out;
768 
769 	rec = ftrace_find_profiled_func(stat, ip);
770 	if (!rec) {
771 		rec = ftrace_profile_alloc(stat, ip);
772 		if (!rec)
773 			goto out;
774 	}
775 
776 	rec->counter++;
777  out:
778 	local_irq_restore(flags);
779 }
780 
781 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
782 static bool fgraph_graph_time = true;
783 
784 void ftrace_graph_graph_time_control(bool enable)
785 {
786 	fgraph_graph_time = enable;
787 }
788 
789 static int profile_graph_entry(struct ftrace_graph_ent *trace)
790 {
791 	struct ftrace_ret_stack *ret_stack;
792 
793 	function_profile_call(trace->func, 0, NULL, NULL);
794 
795 	/* If function graph is shutting down, ret_stack can be NULL */
796 	if (!current->ret_stack)
797 		return 0;
798 
799 	ret_stack = ftrace_graph_get_ret_stack(current, 0);
800 	if (ret_stack)
801 		ret_stack->subtime = 0;
802 
803 	return 1;
804 }
805 
806 static void profile_graph_return(struct ftrace_graph_ret *trace)
807 {
808 	struct ftrace_ret_stack *ret_stack;
809 	struct ftrace_profile_stat *stat;
810 	unsigned long long calltime;
811 	struct ftrace_profile *rec;
812 	unsigned long flags;
813 
814 	local_irq_save(flags);
815 	stat = this_cpu_ptr(&ftrace_profile_stats);
816 	if (!stat->hash || !ftrace_profile_enabled)
817 		goto out;
818 
819 	/* If the calltime was zero'd ignore it */
820 	if (!trace->calltime)
821 		goto out;
822 
823 	calltime = trace->rettime - trace->calltime;
824 
825 	if (!fgraph_graph_time) {
826 
827 		/* Append this call time to the parent time to subtract */
828 		ret_stack = ftrace_graph_get_ret_stack(current, 1);
829 		if (ret_stack)
830 			ret_stack->subtime += calltime;
831 
832 		ret_stack = ftrace_graph_get_ret_stack(current, 0);
833 		if (ret_stack && ret_stack->subtime < calltime)
834 			calltime -= ret_stack->subtime;
835 		else
836 			calltime = 0;
837 	}
838 
839 	rec = ftrace_find_profiled_func(stat, trace->func);
840 	if (rec) {
841 		rec->time += calltime;
842 		rec->time_squared += calltime * calltime;
843 	}
844 
845  out:
846 	local_irq_restore(flags);
847 }
848 
849 static struct fgraph_ops fprofiler_ops = {
850 	.entryfunc = &profile_graph_entry,
851 	.retfunc = &profile_graph_return,
852 };
853 
854 static int register_ftrace_profiler(void)
855 {
856 	return register_ftrace_graph(&fprofiler_ops);
857 }
858 
859 static void unregister_ftrace_profiler(void)
860 {
861 	unregister_ftrace_graph(&fprofiler_ops);
862 }
863 #else
864 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
865 	.func		= function_profile_call,
866 	.flags		= FTRACE_OPS_FL_INITIALIZED,
867 	INIT_OPS_HASH(ftrace_profile_ops)
868 };
869 
870 static int register_ftrace_profiler(void)
871 {
872 	return register_ftrace_function(&ftrace_profile_ops);
873 }
874 
875 static void unregister_ftrace_profiler(void)
876 {
877 	unregister_ftrace_function(&ftrace_profile_ops);
878 }
879 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
880 
881 static ssize_t
882 ftrace_profile_write(struct file *filp, const char __user *ubuf,
883 		     size_t cnt, loff_t *ppos)
884 {
885 	unsigned long val;
886 	int ret;
887 
888 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
889 	if (ret)
890 		return ret;
891 
892 	val = !!val;
893 
894 	mutex_lock(&ftrace_profile_lock);
895 	if (ftrace_profile_enabled ^ val) {
896 		if (val) {
897 			ret = ftrace_profile_init();
898 			if (ret < 0) {
899 				cnt = ret;
900 				goto out;
901 			}
902 
903 			ret = register_ftrace_profiler();
904 			if (ret < 0) {
905 				cnt = ret;
906 				goto out;
907 			}
908 			ftrace_profile_enabled = 1;
909 		} else {
910 			ftrace_profile_enabled = 0;
911 			/*
912 			 * unregister_ftrace_profiler calls stop_machine
913 			 * so this acts like an synchronize_rcu.
914 			 */
915 			unregister_ftrace_profiler();
916 		}
917 	}
918  out:
919 	mutex_unlock(&ftrace_profile_lock);
920 
921 	*ppos += cnt;
922 
923 	return cnt;
924 }
925 
926 static ssize_t
927 ftrace_profile_read(struct file *filp, char __user *ubuf,
928 		     size_t cnt, loff_t *ppos)
929 {
930 	char buf[64];		/* big enough to hold a number */
931 	int r;
932 
933 	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
934 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
935 }
936 
937 static const struct file_operations ftrace_profile_fops = {
938 	.open		= tracing_open_generic,
939 	.read		= ftrace_profile_read,
940 	.write		= ftrace_profile_write,
941 	.llseek		= default_llseek,
942 };
943 
944 /* used to initialize the real stat files */
945 static struct tracer_stat function_stats __initdata = {
946 	.name		= "functions",
947 	.stat_start	= function_stat_start,
948 	.stat_next	= function_stat_next,
949 	.stat_cmp	= function_stat_cmp,
950 	.stat_headers	= function_stat_headers,
951 	.stat_show	= function_stat_show
952 };
953 
954 static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
955 {
956 	struct ftrace_profile_stat *stat;
957 	char *name;
958 	int ret;
959 	int cpu;
960 
961 	for_each_possible_cpu(cpu) {
962 		stat = &per_cpu(ftrace_profile_stats, cpu);
963 
964 		name = kasprintf(GFP_KERNEL, "function%d", cpu);
965 		if (!name) {
966 			/*
967 			 * The files created are permanent, if something happens
968 			 * we still do not free memory.
969 			 */
970 			WARN(1,
971 			     "Could not allocate stat file for cpu %d\n",
972 			     cpu);
973 			return;
974 		}
975 		stat->stat = function_stats;
976 		stat->stat.name = name;
977 		ret = register_stat_tracer(&stat->stat);
978 		if (ret) {
979 			WARN(1,
980 			     "Could not register function stat for cpu %d\n",
981 			     cpu);
982 			kfree(name);
983 			return;
984 		}
985 	}
986 
987 	trace_create_file("function_profile_enabled",
988 			  TRACE_MODE_WRITE, d_tracer, NULL,
989 			  &ftrace_profile_fops);
990 }
991 
992 #else /* CONFIG_FUNCTION_PROFILER */
993 static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
994 {
995 }
996 #endif /* CONFIG_FUNCTION_PROFILER */
997 
998 #ifdef CONFIG_DYNAMIC_FTRACE
999 
1000 static struct ftrace_ops *removed_ops;
1001 
1002 /*
1003  * Set when doing a global update, like enabling all recs or disabling them.
1004  * It is not set when just updating a single ftrace_ops.
1005  */
1006 static bool update_all_ops;
1007 
1008 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
1009 # error Dynamic ftrace depends on MCOUNT_RECORD
1010 #endif
1011 
1012 struct ftrace_func_probe {
1013 	struct ftrace_probe_ops	*probe_ops;
1014 	struct ftrace_ops	ops;
1015 	struct trace_array	*tr;
1016 	struct list_head	list;
1017 	void			*data;
1018 	int			ref;
1019 };
1020 
1021 /*
1022  * We make these constant because no one should touch them,
1023  * but they are used as the default "empty hash", to avoid allocating
1024  * it all the time. These are in a read only section such that if
1025  * anyone does try to modify it, it will cause an exception.
1026  */
1027 static const struct hlist_head empty_buckets[1];
1028 static const struct ftrace_hash empty_hash = {
1029 	.buckets = (struct hlist_head *)empty_buckets,
1030 };
1031 #define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1032 
1033 struct ftrace_ops global_ops = {
1034 	.func				= ftrace_stub,
1035 	.local_hash.notrace_hash	= EMPTY_HASH,
1036 	.local_hash.filter_hash		= EMPTY_HASH,
1037 	INIT_OPS_HASH(global_ops)
1038 	.flags				= FTRACE_OPS_FL_INITIALIZED |
1039 					  FTRACE_OPS_FL_PID,
1040 };
1041 
1042 /*
1043  * Used by the stack unwinder to know about dynamic ftrace trampolines.
1044  */
1045 struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
1046 {
1047 	struct ftrace_ops *op = NULL;
1048 
1049 	/*
1050 	 * Some of the ops may be dynamically allocated,
1051 	 * they are freed after a synchronize_rcu().
1052 	 */
1053 	preempt_disable_notrace();
1054 
1055 	do_for_each_ftrace_op(op, ftrace_ops_list) {
1056 		/*
1057 		 * This is to check for dynamically allocated trampolines.
1058 		 * Trampolines that are in kernel text will have
1059 		 * core_kernel_text() return true.
1060 		 */
1061 		if (op->trampoline && op->trampoline_size)
1062 			if (addr >= op->trampoline &&
1063 			    addr < op->trampoline + op->trampoline_size) {
1064 				preempt_enable_notrace();
1065 				return op;
1066 			}
1067 	} while_for_each_ftrace_op(op);
1068 	preempt_enable_notrace();
1069 
1070 	return NULL;
1071 }
1072 
1073 /*
1074  * This is used by __kernel_text_address() to return true if the
1075  * address is on a dynamically allocated trampoline that would
1076  * not return true for either core_kernel_text() or
1077  * is_module_text_address().
1078  */
1079 bool is_ftrace_trampoline(unsigned long addr)
1080 {
1081 	return ftrace_ops_trampoline(addr) != NULL;
1082 }
1083 
1084 struct ftrace_page {
1085 	struct ftrace_page	*next;
1086 	struct dyn_ftrace	*records;
1087 	int			index;
1088 	int			order;
1089 };
1090 
1091 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
1092 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1093 
1094 static struct ftrace_page	*ftrace_pages_start;
1095 static struct ftrace_page	*ftrace_pages;
1096 
1097 static __always_inline unsigned long
1098 ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
1099 {
1100 	if (hash->size_bits > 0)
1101 		return hash_long(ip, hash->size_bits);
1102 
1103 	return 0;
1104 }
1105 
1106 /* Only use this function if ftrace_hash_empty() has already been tested */
1107 static __always_inline struct ftrace_func_entry *
1108 __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1109 {
1110 	unsigned long key;
1111 	struct ftrace_func_entry *entry;
1112 	struct hlist_head *hhd;
1113 
1114 	key = ftrace_hash_key(hash, ip);
1115 	hhd = &hash->buckets[key];
1116 
1117 	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1118 		if (entry->ip == ip)
1119 			return entry;
1120 	}
1121 	return NULL;
1122 }
1123 
1124 /**
1125  * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1126  * @hash: The hash to look at
1127  * @ip: The instruction pointer to test
1128  *
1129  * Search a given @hash to see if a given instruction pointer (@ip)
1130  * exists in it.
1131  *
1132  * Returns the entry that holds the @ip if found. NULL otherwise.
1133  */
1134 struct ftrace_func_entry *
1135 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1136 {
1137 	if (ftrace_hash_empty(hash))
1138 		return NULL;
1139 
1140 	return __ftrace_lookup_ip(hash, ip);
1141 }
1142 
1143 static void __add_hash_entry(struct ftrace_hash *hash,
1144 			     struct ftrace_func_entry *entry)
1145 {
1146 	struct hlist_head *hhd;
1147 	unsigned long key;
1148 
1149 	key = ftrace_hash_key(hash, entry->ip);
1150 	hhd = &hash->buckets[key];
1151 	hlist_add_head(&entry->hlist, hhd);
1152 	hash->count++;
1153 }
1154 
1155 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1156 {
1157 	struct ftrace_func_entry *entry;
1158 
1159 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1160 	if (!entry)
1161 		return -ENOMEM;
1162 
1163 	entry->ip = ip;
1164 	__add_hash_entry(hash, entry);
1165 
1166 	return 0;
1167 }
1168 
1169 static void
1170 free_hash_entry(struct ftrace_hash *hash,
1171 		  struct ftrace_func_entry *entry)
1172 {
1173 	hlist_del(&entry->hlist);
1174 	kfree(entry);
1175 	hash->count--;
1176 }
1177 
1178 static void
1179 remove_hash_entry(struct ftrace_hash *hash,
1180 		  struct ftrace_func_entry *entry)
1181 {
1182 	hlist_del_rcu(&entry->hlist);
1183 	hash->count--;
1184 }
1185 
1186 static void ftrace_hash_clear(struct ftrace_hash *hash)
1187 {
1188 	struct hlist_head *hhd;
1189 	struct hlist_node *tn;
1190 	struct ftrace_func_entry *entry;
1191 	int size = 1 << hash->size_bits;
1192 	int i;
1193 
1194 	if (!hash->count)
1195 		return;
1196 
1197 	for (i = 0; i < size; i++) {
1198 		hhd = &hash->buckets[i];
1199 		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1200 			free_hash_entry(hash, entry);
1201 	}
1202 	FTRACE_WARN_ON(hash->count);
1203 }
1204 
1205 static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
1206 {
1207 	list_del(&ftrace_mod->list);
1208 	kfree(ftrace_mod->module);
1209 	kfree(ftrace_mod->func);
1210 	kfree(ftrace_mod);
1211 }
1212 
1213 static void clear_ftrace_mod_list(struct list_head *head)
1214 {
1215 	struct ftrace_mod_load *p, *n;
1216 
1217 	/* stack tracer isn't supported yet */
1218 	if (!head)
1219 		return;
1220 
1221 	mutex_lock(&ftrace_lock);
1222 	list_for_each_entry_safe(p, n, head, list)
1223 		free_ftrace_mod(p);
1224 	mutex_unlock(&ftrace_lock);
1225 }
1226 
1227 static void free_ftrace_hash(struct ftrace_hash *hash)
1228 {
1229 	if (!hash || hash == EMPTY_HASH)
1230 		return;
1231 	ftrace_hash_clear(hash);
1232 	kfree(hash->buckets);
1233 	kfree(hash);
1234 }
1235 
1236 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1237 {
1238 	struct ftrace_hash *hash;
1239 
1240 	hash = container_of(rcu, struct ftrace_hash, rcu);
1241 	free_ftrace_hash(hash);
1242 }
1243 
1244 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1245 {
1246 	if (!hash || hash == EMPTY_HASH)
1247 		return;
1248 	call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
1249 }
1250 
1251 void ftrace_free_filter(struct ftrace_ops *ops)
1252 {
1253 	ftrace_ops_init(ops);
1254 	free_ftrace_hash(ops->func_hash->filter_hash);
1255 	free_ftrace_hash(ops->func_hash->notrace_hash);
1256 }
1257 
1258 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1259 {
1260 	struct ftrace_hash *hash;
1261 	int size;
1262 
1263 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1264 	if (!hash)
1265 		return NULL;
1266 
1267 	size = 1 << size_bits;
1268 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1269 
1270 	if (!hash->buckets) {
1271 		kfree(hash);
1272 		return NULL;
1273 	}
1274 
1275 	hash->size_bits = size_bits;
1276 
1277 	return hash;
1278 }
1279 
1280 
1281 static int ftrace_add_mod(struct trace_array *tr,
1282 			  const char *func, const char *module,
1283 			  int enable)
1284 {
1285 	struct ftrace_mod_load *ftrace_mod;
1286 	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
1287 
1288 	ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
1289 	if (!ftrace_mod)
1290 		return -ENOMEM;
1291 
1292 	ftrace_mod->func = kstrdup(func, GFP_KERNEL);
1293 	ftrace_mod->module = kstrdup(module, GFP_KERNEL);
1294 	ftrace_mod->enable = enable;
1295 
1296 	if (!ftrace_mod->func || !ftrace_mod->module)
1297 		goto out_free;
1298 
1299 	list_add(&ftrace_mod->list, mod_head);
1300 
1301 	return 0;
1302 
1303  out_free:
1304 	free_ftrace_mod(ftrace_mod);
1305 
1306 	return -ENOMEM;
1307 }
1308 
1309 static struct ftrace_hash *
1310 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1311 {
1312 	struct ftrace_func_entry *entry;
1313 	struct ftrace_hash *new_hash;
1314 	int size;
1315 	int ret;
1316 	int i;
1317 
1318 	new_hash = alloc_ftrace_hash(size_bits);
1319 	if (!new_hash)
1320 		return NULL;
1321 
1322 	if (hash)
1323 		new_hash->flags = hash->flags;
1324 
1325 	/* Empty hash? */
1326 	if (ftrace_hash_empty(hash))
1327 		return new_hash;
1328 
1329 	size = 1 << hash->size_bits;
1330 	for (i = 0; i < size; i++) {
1331 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1332 			ret = add_hash_entry(new_hash, entry->ip);
1333 			if (ret < 0)
1334 				goto free_hash;
1335 		}
1336 	}
1337 
1338 	FTRACE_WARN_ON(new_hash->count != hash->count);
1339 
1340 	return new_hash;
1341 
1342  free_hash:
1343 	free_ftrace_hash(new_hash);
1344 	return NULL;
1345 }
1346 
1347 static void
1348 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1349 static void
1350 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1351 
1352 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1353 				       struct ftrace_hash *new_hash);
1354 
1355 static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size)
1356 {
1357 	struct ftrace_func_entry *entry;
1358 	struct ftrace_hash *new_hash;
1359 	struct hlist_head *hhd;
1360 	struct hlist_node *tn;
1361 	int bits = 0;
1362 	int i;
1363 
1364 	/*
1365 	 * Use around half the size (max bit of it), but
1366 	 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits).
1367 	 */
1368 	bits = fls(size / 2);
1369 
1370 	/* Don't allocate too much */
1371 	if (bits > FTRACE_HASH_MAX_BITS)
1372 		bits = FTRACE_HASH_MAX_BITS;
1373 
1374 	new_hash = alloc_ftrace_hash(bits);
1375 	if (!new_hash)
1376 		return NULL;
1377 
1378 	new_hash->flags = src->flags;
1379 
1380 	size = 1 << src->size_bits;
1381 	for (i = 0; i < size; i++) {
1382 		hhd = &src->buckets[i];
1383 		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1384 			remove_hash_entry(src, entry);
1385 			__add_hash_entry(new_hash, entry);
1386 		}
1387 	}
1388 	return new_hash;
1389 }
1390 
1391 static struct ftrace_hash *
1392 __ftrace_hash_move(struct ftrace_hash *src)
1393 {
1394 	int size = src->count;
1395 
1396 	/*
1397 	 * If the new source is empty, just return the empty_hash.
1398 	 */
1399 	if (ftrace_hash_empty(src))
1400 		return EMPTY_HASH;
1401 
1402 	return dup_hash(src, size);
1403 }
1404 
1405 static int
1406 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1407 		 struct ftrace_hash **dst, struct ftrace_hash *src)
1408 {
1409 	struct ftrace_hash *new_hash;
1410 	int ret;
1411 
1412 	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
1413 	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1414 		return -EINVAL;
1415 
1416 	new_hash = __ftrace_hash_move(src);
1417 	if (!new_hash)
1418 		return -ENOMEM;
1419 
1420 	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1421 	if (enable) {
1422 		/* IPMODIFY should be updated only when filter_hash updating */
1423 		ret = ftrace_hash_ipmodify_update(ops, new_hash);
1424 		if (ret < 0) {
1425 			free_ftrace_hash(new_hash);
1426 			return ret;
1427 		}
1428 	}
1429 
1430 	/*
1431 	 * Remove the current set, update the hash and add
1432 	 * them back.
1433 	 */
1434 	ftrace_hash_rec_disable_modify(ops, enable);
1435 
1436 	rcu_assign_pointer(*dst, new_hash);
1437 
1438 	ftrace_hash_rec_enable_modify(ops, enable);
1439 
1440 	return 0;
1441 }
1442 
1443 static bool hash_contains_ip(unsigned long ip,
1444 			     struct ftrace_ops_hash *hash)
1445 {
1446 	/*
1447 	 * The function record is a match if it exists in the filter
1448 	 * hash and not in the notrace hash. Note, an empty hash is
1449 	 * considered a match for the filter hash, but an empty
1450 	 * notrace hash is considered not in the notrace hash.
1451 	 */
1452 	return (ftrace_hash_empty(hash->filter_hash) ||
1453 		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1454 		(ftrace_hash_empty(hash->notrace_hash) ||
1455 		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1456 }
1457 
1458 /*
1459  * Test the hashes for this ops to see if we want to call
1460  * the ops->func or not.
1461  *
1462  * It's a match if the ip is in the ops->filter_hash or
1463  * the filter_hash does not exist or is empty,
1464  *  AND
1465  * the ip is not in the ops->notrace_hash.
1466  *
1467  * This needs to be called with preemption disabled as
1468  * the hashes are freed with call_rcu().
1469  */
1470 int
1471 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1472 {
1473 	struct ftrace_ops_hash hash;
1474 	int ret;
1475 
1476 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1477 	/*
1478 	 * There's a small race when adding ops that the ftrace handler
1479 	 * that wants regs, may be called without them. We can not
1480 	 * allow that handler to be called if regs is NULL.
1481 	 */
1482 	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1483 		return 0;
1484 #endif
1485 
1486 	rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
1487 	rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1488 
1489 	if (hash_contains_ip(ip, &hash))
1490 		ret = 1;
1491 	else
1492 		ret = 0;
1493 
1494 	return ret;
1495 }
1496 
1497 /*
1498  * This is a double for. Do not use 'break' to break out of the loop,
1499  * you must use a goto.
1500  */
1501 #define do_for_each_ftrace_rec(pg, rec)					\
1502 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1503 		int _____i;						\
1504 		for (_____i = 0; _____i < pg->index; _____i++) {	\
1505 			rec = &pg->records[_____i];
1506 
1507 #define while_for_each_ftrace_rec()		\
1508 		}				\
1509 	}
1510 
1511 
1512 static int ftrace_cmp_recs(const void *a, const void *b)
1513 {
1514 	const struct dyn_ftrace *key = a;
1515 	const struct dyn_ftrace *rec = b;
1516 
1517 	if (key->flags < rec->ip)
1518 		return -1;
1519 	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1520 		return 1;
1521 	return 0;
1522 }
1523 
1524 static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end)
1525 {
1526 	struct ftrace_page *pg;
1527 	struct dyn_ftrace *rec = NULL;
1528 	struct dyn_ftrace key;
1529 
1530 	key.ip = start;
1531 	key.flags = end;	/* overload flags, as it is unsigned long */
1532 
1533 	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1534 		if (end < pg->records[0].ip ||
1535 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1536 			continue;
1537 		rec = bsearch(&key, pg->records, pg->index,
1538 			      sizeof(struct dyn_ftrace),
1539 			      ftrace_cmp_recs);
1540 		if (rec)
1541 			break;
1542 	}
1543 	return rec;
1544 }
1545 
1546 /**
1547  * ftrace_location_range - return the first address of a traced location
1548  *	if it touches the given ip range
1549  * @start: start of range to search.
1550  * @end: end of range to search (inclusive). @end points to the last byte
1551  *	to check.
1552  *
1553  * Returns rec->ip if the related ftrace location is a least partly within
1554  * the given address range. That is, the first address of the instruction
1555  * that is either a NOP or call to the function tracer. It checks the ftrace
1556  * internal tables to determine if the address belongs or not.
1557  */
1558 unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1559 {
1560 	struct dyn_ftrace *rec;
1561 
1562 	rec = lookup_rec(start, end);
1563 	if (rec)
1564 		return rec->ip;
1565 
1566 	return 0;
1567 }
1568 
1569 /**
1570  * ftrace_location - return the ftrace location
1571  * @ip: the instruction pointer to check
1572  *
1573  * If @ip matches the ftrace location, return @ip.
1574  * If @ip matches sym+0, return sym's ftrace location.
1575  * Otherwise, return 0.
1576  */
1577 unsigned long ftrace_location(unsigned long ip)
1578 {
1579 	struct dyn_ftrace *rec;
1580 	unsigned long offset;
1581 	unsigned long size;
1582 
1583 	rec = lookup_rec(ip, ip);
1584 	if (!rec) {
1585 		if (!kallsyms_lookup_size_offset(ip, &size, &offset))
1586 			goto out;
1587 
1588 		/* map sym+0 to __fentry__ */
1589 		if (!offset)
1590 			rec = lookup_rec(ip, ip + size - 1);
1591 	}
1592 
1593 	if (rec)
1594 		return rec->ip;
1595 
1596 out:
1597 	return 0;
1598 }
1599 
1600 /**
1601  * ftrace_text_reserved - return true if range contains an ftrace location
1602  * @start: start of range to search
1603  * @end: end of range to search (inclusive). @end points to the last byte to check.
1604  *
1605  * Returns 1 if @start and @end contains a ftrace location.
1606  * That is, the instruction that is either a NOP or call to
1607  * the function tracer. It checks the ftrace internal tables to
1608  * determine if the address belongs or not.
1609  */
1610 int ftrace_text_reserved(const void *start, const void *end)
1611 {
1612 	unsigned long ret;
1613 
1614 	ret = ftrace_location_range((unsigned long)start,
1615 				    (unsigned long)end);
1616 
1617 	return (int)!!ret;
1618 }
1619 
1620 /* Test if ops registered to this rec needs regs */
1621 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1622 {
1623 	struct ftrace_ops *ops;
1624 	bool keep_regs = false;
1625 
1626 	for (ops = ftrace_ops_list;
1627 	     ops != &ftrace_list_end; ops = ops->next) {
1628 		/* pass rec in as regs to have non-NULL val */
1629 		if (ftrace_ops_test(ops, rec->ip, rec)) {
1630 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1631 				keep_regs = true;
1632 				break;
1633 			}
1634 		}
1635 	}
1636 
1637 	return  keep_regs;
1638 }
1639 
1640 static struct ftrace_ops *
1641 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1642 static struct ftrace_ops *
1643 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
1644 static struct ftrace_ops *
1645 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1646 
1647 static bool skip_record(struct dyn_ftrace *rec)
1648 {
1649 	/*
1650 	 * At boot up, weak functions are set to disable. Function tracing
1651 	 * can be enabled before they are, and they still need to be disabled now.
1652 	 * If the record is disabled, still continue if it is marked as already
1653 	 * enabled (this is needed to keep the accounting working).
1654 	 */
1655 	return rec->flags & FTRACE_FL_DISABLED &&
1656 		!(rec->flags & FTRACE_FL_ENABLED);
1657 }
1658 
1659 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1660 				     int filter_hash,
1661 				     bool inc)
1662 {
1663 	struct ftrace_hash *hash;
1664 	struct ftrace_hash *other_hash;
1665 	struct ftrace_page *pg;
1666 	struct dyn_ftrace *rec;
1667 	bool update = false;
1668 	int count = 0;
1669 	int all = false;
1670 
1671 	/* Only update if the ops has been registered */
1672 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1673 		return false;
1674 
1675 	/*
1676 	 * In the filter_hash case:
1677 	 *   If the count is zero, we update all records.
1678 	 *   Otherwise we just update the items in the hash.
1679 	 *
1680 	 * In the notrace_hash case:
1681 	 *   We enable the update in the hash.
1682 	 *   As disabling notrace means enabling the tracing,
1683 	 *   and enabling notrace means disabling, the inc variable
1684 	 *   gets inversed.
1685 	 */
1686 	if (filter_hash) {
1687 		hash = ops->func_hash->filter_hash;
1688 		other_hash = ops->func_hash->notrace_hash;
1689 		if (ftrace_hash_empty(hash))
1690 			all = true;
1691 	} else {
1692 		inc = !inc;
1693 		hash = ops->func_hash->notrace_hash;
1694 		other_hash = ops->func_hash->filter_hash;
1695 		/*
1696 		 * If the notrace hash has no items,
1697 		 * then there's nothing to do.
1698 		 */
1699 		if (ftrace_hash_empty(hash))
1700 			return false;
1701 	}
1702 
1703 	do_for_each_ftrace_rec(pg, rec) {
1704 		int in_other_hash = 0;
1705 		int in_hash = 0;
1706 		int match = 0;
1707 
1708 		if (skip_record(rec))
1709 			continue;
1710 
1711 		if (all) {
1712 			/*
1713 			 * Only the filter_hash affects all records.
1714 			 * Update if the record is not in the notrace hash.
1715 			 */
1716 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1717 				match = 1;
1718 		} else {
1719 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1720 			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1721 
1722 			/*
1723 			 * If filter_hash is set, we want to match all functions
1724 			 * that are in the hash but not in the other hash.
1725 			 *
1726 			 * If filter_hash is not set, then we are decrementing.
1727 			 * That means we match anything that is in the hash
1728 			 * and also in the other_hash. That is, we need to turn
1729 			 * off functions in the other hash because they are disabled
1730 			 * by this hash.
1731 			 */
1732 			if (filter_hash && in_hash && !in_other_hash)
1733 				match = 1;
1734 			else if (!filter_hash && in_hash &&
1735 				 (in_other_hash || ftrace_hash_empty(other_hash)))
1736 				match = 1;
1737 		}
1738 		if (!match)
1739 			continue;
1740 
1741 		if (inc) {
1742 			rec->flags++;
1743 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1744 				return false;
1745 
1746 			if (ops->flags & FTRACE_OPS_FL_DIRECT)
1747 				rec->flags |= FTRACE_FL_DIRECT;
1748 
1749 			/*
1750 			 * If there's only a single callback registered to a
1751 			 * function, and the ops has a trampoline registered
1752 			 * for it, then we can call it directly.
1753 			 */
1754 			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1755 				rec->flags |= FTRACE_FL_TRAMP;
1756 			else
1757 				/*
1758 				 * If we are adding another function callback
1759 				 * to this function, and the previous had a
1760 				 * custom trampoline in use, then we need to go
1761 				 * back to the default trampoline.
1762 				 */
1763 				rec->flags &= ~FTRACE_FL_TRAMP;
1764 
1765 			/*
1766 			 * If any ops wants regs saved for this function
1767 			 * then all ops will get saved regs.
1768 			 */
1769 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1770 				rec->flags |= FTRACE_FL_REGS;
1771 		} else {
1772 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1773 				return false;
1774 			rec->flags--;
1775 
1776 			/*
1777 			 * Only the internal direct_ops should have the
1778 			 * DIRECT flag set. Thus, if it is removing a
1779 			 * function, then that function should no longer
1780 			 * be direct.
1781 			 */
1782 			if (ops->flags & FTRACE_OPS_FL_DIRECT)
1783 				rec->flags &= ~FTRACE_FL_DIRECT;
1784 
1785 			/*
1786 			 * If the rec had REGS enabled and the ops that is
1787 			 * being removed had REGS set, then see if there is
1788 			 * still any ops for this record that wants regs.
1789 			 * If not, we can stop recording them.
1790 			 */
1791 			if (ftrace_rec_count(rec) > 0 &&
1792 			    rec->flags & FTRACE_FL_REGS &&
1793 			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1794 				if (!test_rec_ops_needs_regs(rec))
1795 					rec->flags &= ~FTRACE_FL_REGS;
1796 			}
1797 
1798 			/*
1799 			 * The TRAMP needs to be set only if rec count
1800 			 * is decremented to one, and the ops that is
1801 			 * left has a trampoline. As TRAMP can only be
1802 			 * enabled if there is only a single ops attached
1803 			 * to it.
1804 			 */
1805 			if (ftrace_rec_count(rec) == 1 &&
1806 			    ftrace_find_tramp_ops_any_other(rec, ops))
1807 				rec->flags |= FTRACE_FL_TRAMP;
1808 			else
1809 				rec->flags &= ~FTRACE_FL_TRAMP;
1810 
1811 			/*
1812 			 * flags will be cleared in ftrace_check_record()
1813 			 * if rec count is zero.
1814 			 */
1815 		}
1816 		count++;
1817 
1818 		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1819 		update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
1820 
1821 		/* Shortcut, if we handled all records, we are done. */
1822 		if (!all && count == hash->count)
1823 			return update;
1824 	} while_for_each_ftrace_rec();
1825 
1826 	return update;
1827 }
1828 
1829 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1830 				    int filter_hash)
1831 {
1832 	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1833 }
1834 
1835 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1836 				   int filter_hash)
1837 {
1838 	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1839 }
1840 
1841 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
1842 					  int filter_hash, int inc)
1843 {
1844 	struct ftrace_ops *op;
1845 
1846 	__ftrace_hash_rec_update(ops, filter_hash, inc);
1847 
1848 	if (ops->func_hash != &global_ops.local_hash)
1849 		return;
1850 
1851 	/*
1852 	 * If the ops shares the global_ops hash, then we need to update
1853 	 * all ops that are enabled and use this hash.
1854 	 */
1855 	do_for_each_ftrace_op(op, ftrace_ops_list) {
1856 		/* Already done */
1857 		if (op == ops)
1858 			continue;
1859 		if (op->func_hash == &global_ops.local_hash)
1860 			__ftrace_hash_rec_update(op, filter_hash, inc);
1861 	} while_for_each_ftrace_op(op);
1862 }
1863 
1864 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
1865 					   int filter_hash)
1866 {
1867 	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
1868 }
1869 
1870 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
1871 					  int filter_hash)
1872 {
1873 	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
1874 }
1875 
1876 /*
1877  * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
1878  * or no-needed to update, -EBUSY if it detects a conflict of the flag
1879  * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1880  * Note that old_hash and new_hash has below meanings
1881  *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1882  *  - If the hash is EMPTY_HASH, it hits nothing
1883  *  - Anything else hits the recs which match the hash entries.
1884  *
1885  * DIRECT ops does not have IPMODIFY flag, but we still need to check it
1886  * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call
1887  * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with
1888  * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate
1889  * the return value to the caller and eventually to the owner of the DIRECT
1890  * ops.
1891  */
1892 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
1893 					 struct ftrace_hash *old_hash,
1894 					 struct ftrace_hash *new_hash)
1895 {
1896 	struct ftrace_page *pg;
1897 	struct dyn_ftrace *rec, *end = NULL;
1898 	int in_old, in_new;
1899 	bool is_ipmodify, is_direct;
1900 
1901 	/* Only update if the ops has been registered */
1902 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1903 		return 0;
1904 
1905 	is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY;
1906 	is_direct = ops->flags & FTRACE_OPS_FL_DIRECT;
1907 
1908 	/* neither IPMODIFY nor DIRECT, skip */
1909 	if (!is_ipmodify && !is_direct)
1910 		return 0;
1911 
1912 	if (WARN_ON_ONCE(is_ipmodify && is_direct))
1913 		return 0;
1914 
1915 	/*
1916 	 * Since the IPMODIFY and DIRECT are very address sensitive
1917 	 * actions, we do not allow ftrace_ops to set all functions to new
1918 	 * hash.
1919 	 */
1920 	if (!new_hash || !old_hash)
1921 		return -EINVAL;
1922 
1923 	/* Update rec->flags */
1924 	do_for_each_ftrace_rec(pg, rec) {
1925 
1926 		if (rec->flags & FTRACE_FL_DISABLED)
1927 			continue;
1928 
1929 		/* We need to update only differences of filter_hash */
1930 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1931 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1932 		if (in_old == in_new)
1933 			continue;
1934 
1935 		if (in_new) {
1936 			if (rec->flags & FTRACE_FL_IPMODIFY) {
1937 				int ret;
1938 
1939 				/* Cannot have two ipmodify on same rec */
1940 				if (is_ipmodify)
1941 					goto rollback;
1942 
1943 				FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT);
1944 
1945 				/*
1946 				 * Another ops with IPMODIFY is already
1947 				 * attached. We are now attaching a direct
1948 				 * ops. Run SHARE_IPMODIFY_SELF, to check
1949 				 * whether sharing is supported.
1950 				 */
1951 				if (!ops->ops_func)
1952 					return -EBUSY;
1953 				ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF);
1954 				if (ret)
1955 					return ret;
1956 			} else if (is_ipmodify) {
1957 				rec->flags |= FTRACE_FL_IPMODIFY;
1958 			}
1959 		} else if (is_ipmodify) {
1960 			rec->flags &= ~FTRACE_FL_IPMODIFY;
1961 		}
1962 	} while_for_each_ftrace_rec();
1963 
1964 	return 0;
1965 
1966 rollback:
1967 	end = rec;
1968 
1969 	/* Roll back what we did above */
1970 	do_for_each_ftrace_rec(pg, rec) {
1971 
1972 		if (rec->flags & FTRACE_FL_DISABLED)
1973 			continue;
1974 
1975 		if (rec == end)
1976 			goto err_out;
1977 
1978 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1979 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1980 		if (in_old == in_new)
1981 			continue;
1982 
1983 		if (in_new)
1984 			rec->flags &= ~FTRACE_FL_IPMODIFY;
1985 		else
1986 			rec->flags |= FTRACE_FL_IPMODIFY;
1987 	} while_for_each_ftrace_rec();
1988 
1989 err_out:
1990 	return -EBUSY;
1991 }
1992 
1993 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
1994 {
1995 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1996 
1997 	if (ftrace_hash_empty(hash))
1998 		hash = NULL;
1999 
2000 	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
2001 }
2002 
2003 /* Disabling always succeeds */
2004 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
2005 {
2006 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
2007 
2008 	if (ftrace_hash_empty(hash))
2009 		hash = NULL;
2010 
2011 	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
2012 }
2013 
2014 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
2015 				       struct ftrace_hash *new_hash)
2016 {
2017 	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
2018 
2019 	if (ftrace_hash_empty(old_hash))
2020 		old_hash = NULL;
2021 
2022 	if (ftrace_hash_empty(new_hash))
2023 		new_hash = NULL;
2024 
2025 	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
2026 }
2027 
2028 static void print_ip_ins(const char *fmt, const unsigned char *p)
2029 {
2030 	char ins[MCOUNT_INSN_SIZE];
2031 
2032 	if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) {
2033 		printk(KERN_CONT "%s[FAULT] %px\n", fmt, p);
2034 		return;
2035 	}
2036 
2037 	printk(KERN_CONT "%s", fmt);
2038 	pr_cont("%*phC", MCOUNT_INSN_SIZE, ins);
2039 }
2040 
2041 enum ftrace_bug_type ftrace_bug_type;
2042 const void *ftrace_expected;
2043 
2044 static void print_bug_type(void)
2045 {
2046 	switch (ftrace_bug_type) {
2047 	case FTRACE_BUG_UNKNOWN:
2048 		break;
2049 	case FTRACE_BUG_INIT:
2050 		pr_info("Initializing ftrace call sites\n");
2051 		break;
2052 	case FTRACE_BUG_NOP:
2053 		pr_info("Setting ftrace call site to NOP\n");
2054 		break;
2055 	case FTRACE_BUG_CALL:
2056 		pr_info("Setting ftrace call site to call ftrace function\n");
2057 		break;
2058 	case FTRACE_BUG_UPDATE:
2059 		pr_info("Updating ftrace call site to call a different ftrace function\n");
2060 		break;
2061 	}
2062 }
2063 
2064 /**
2065  * ftrace_bug - report and shutdown function tracer
2066  * @failed: The failed type (EFAULT, EINVAL, EPERM)
2067  * @rec: The record that failed
2068  *
2069  * The arch code that enables or disables the function tracing
2070  * can call ftrace_bug() when it has detected a problem in
2071  * modifying the code. @failed should be one of either:
2072  * EFAULT - if the problem happens on reading the @ip address
2073  * EINVAL - if what is read at @ip is not what was expected
2074  * EPERM - if the problem happens on writing to the @ip address
2075  */
2076 void ftrace_bug(int failed, struct dyn_ftrace *rec)
2077 {
2078 	unsigned long ip = rec ? rec->ip : 0;
2079 
2080 	pr_info("------------[ ftrace bug ]------------\n");
2081 
2082 	switch (failed) {
2083 	case -EFAULT:
2084 		pr_info("ftrace faulted on modifying ");
2085 		print_ip_sym(KERN_INFO, ip);
2086 		break;
2087 	case -EINVAL:
2088 		pr_info("ftrace failed to modify ");
2089 		print_ip_sym(KERN_INFO, ip);
2090 		print_ip_ins(" actual:   ", (unsigned char *)ip);
2091 		pr_cont("\n");
2092 		if (ftrace_expected) {
2093 			print_ip_ins(" expected: ", ftrace_expected);
2094 			pr_cont("\n");
2095 		}
2096 		break;
2097 	case -EPERM:
2098 		pr_info("ftrace faulted on writing ");
2099 		print_ip_sym(KERN_INFO, ip);
2100 		break;
2101 	default:
2102 		pr_info("ftrace faulted on unknown error ");
2103 		print_ip_sym(KERN_INFO, ip);
2104 	}
2105 	print_bug_type();
2106 	if (rec) {
2107 		struct ftrace_ops *ops = NULL;
2108 
2109 		pr_info("ftrace record flags: %lx\n", rec->flags);
2110 		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
2111 			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
2112 		if (rec->flags & FTRACE_FL_TRAMP_EN) {
2113 			ops = ftrace_find_tramp_ops_any(rec);
2114 			if (ops) {
2115 				do {
2116 					pr_cont("\ttramp: %pS (%pS)",
2117 						(void *)ops->trampoline,
2118 						(void *)ops->func);
2119 					ops = ftrace_find_tramp_ops_next(rec, ops);
2120 				} while (ops);
2121 			} else
2122 				pr_cont("\ttramp: ERROR!");
2123 
2124 		}
2125 		ip = ftrace_get_addr_curr(rec);
2126 		pr_cont("\n expected tramp: %lx\n", ip);
2127 	}
2128 
2129 	FTRACE_WARN_ON_ONCE(1);
2130 }
2131 
2132 static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
2133 {
2134 	unsigned long flag = 0UL;
2135 
2136 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2137 
2138 	if (skip_record(rec))
2139 		return FTRACE_UPDATE_IGNORE;
2140 
2141 	/*
2142 	 * If we are updating calls:
2143 	 *
2144 	 *   If the record has a ref count, then we need to enable it
2145 	 *   because someone is using it.
2146 	 *
2147 	 *   Otherwise we make sure its disabled.
2148 	 *
2149 	 * If we are disabling calls, then disable all records that
2150 	 * are enabled.
2151 	 */
2152 	if (enable && ftrace_rec_count(rec))
2153 		flag = FTRACE_FL_ENABLED;
2154 
2155 	/*
2156 	 * If enabling and the REGS flag does not match the REGS_EN, or
2157 	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2158 	 * this record. Set flags to fail the compare against ENABLED.
2159 	 * Same for direct calls.
2160 	 */
2161 	if (flag) {
2162 		if (!(rec->flags & FTRACE_FL_REGS) !=
2163 		    !(rec->flags & FTRACE_FL_REGS_EN))
2164 			flag |= FTRACE_FL_REGS;
2165 
2166 		if (!(rec->flags & FTRACE_FL_TRAMP) !=
2167 		    !(rec->flags & FTRACE_FL_TRAMP_EN))
2168 			flag |= FTRACE_FL_TRAMP;
2169 
2170 		/*
2171 		 * Direct calls are special, as count matters.
2172 		 * We must test the record for direct, if the
2173 		 * DIRECT and DIRECT_EN do not match, but only
2174 		 * if the count is 1. That's because, if the
2175 		 * count is something other than one, we do not
2176 		 * want the direct enabled (it will be done via the
2177 		 * direct helper). But if DIRECT_EN is set, and
2178 		 * the count is not one, we need to clear it.
2179 		 */
2180 		if (ftrace_rec_count(rec) == 1) {
2181 			if (!(rec->flags & FTRACE_FL_DIRECT) !=
2182 			    !(rec->flags & FTRACE_FL_DIRECT_EN))
2183 				flag |= FTRACE_FL_DIRECT;
2184 		} else if (rec->flags & FTRACE_FL_DIRECT_EN) {
2185 			flag |= FTRACE_FL_DIRECT;
2186 		}
2187 	}
2188 
2189 	/* If the state of this record hasn't changed, then do nothing */
2190 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2191 		return FTRACE_UPDATE_IGNORE;
2192 
2193 	if (flag) {
2194 		/* Save off if rec is being enabled (for return value) */
2195 		flag ^= rec->flags & FTRACE_FL_ENABLED;
2196 
2197 		if (update) {
2198 			rec->flags |= FTRACE_FL_ENABLED;
2199 			if (flag & FTRACE_FL_REGS) {
2200 				if (rec->flags & FTRACE_FL_REGS)
2201 					rec->flags |= FTRACE_FL_REGS_EN;
2202 				else
2203 					rec->flags &= ~FTRACE_FL_REGS_EN;
2204 			}
2205 			if (flag & FTRACE_FL_TRAMP) {
2206 				if (rec->flags & FTRACE_FL_TRAMP)
2207 					rec->flags |= FTRACE_FL_TRAMP_EN;
2208 				else
2209 					rec->flags &= ~FTRACE_FL_TRAMP_EN;
2210 			}
2211 
2212 			if (flag & FTRACE_FL_DIRECT) {
2213 				/*
2214 				 * If there's only one user (direct_ops helper)
2215 				 * then we can call the direct function
2216 				 * directly (no ftrace trampoline).
2217 				 */
2218 				if (ftrace_rec_count(rec) == 1) {
2219 					if (rec->flags & FTRACE_FL_DIRECT)
2220 						rec->flags |= FTRACE_FL_DIRECT_EN;
2221 					else
2222 						rec->flags &= ~FTRACE_FL_DIRECT_EN;
2223 				} else {
2224 					/*
2225 					 * Can only call directly if there's
2226 					 * only one callback to the function.
2227 					 */
2228 					rec->flags &= ~FTRACE_FL_DIRECT_EN;
2229 				}
2230 			}
2231 		}
2232 
2233 		/*
2234 		 * If this record is being updated from a nop, then
2235 		 *   return UPDATE_MAKE_CALL.
2236 		 * Otherwise,
2237 		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2238 		 *   from the save regs, to a non-save regs function or
2239 		 *   vice versa, or from a trampoline call.
2240 		 */
2241 		if (flag & FTRACE_FL_ENABLED) {
2242 			ftrace_bug_type = FTRACE_BUG_CALL;
2243 			return FTRACE_UPDATE_MAKE_CALL;
2244 		}
2245 
2246 		ftrace_bug_type = FTRACE_BUG_UPDATE;
2247 		return FTRACE_UPDATE_MODIFY_CALL;
2248 	}
2249 
2250 	if (update) {
2251 		/* If there's no more users, clear all flags */
2252 		if (!ftrace_rec_count(rec))
2253 			rec->flags &= FTRACE_FL_DISABLED;
2254 		else
2255 			/*
2256 			 * Just disable the record, but keep the ops TRAMP
2257 			 * and REGS states. The _EN flags must be disabled though.
2258 			 */
2259 			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2260 					FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN);
2261 	}
2262 
2263 	ftrace_bug_type = FTRACE_BUG_NOP;
2264 	return FTRACE_UPDATE_MAKE_NOP;
2265 }
2266 
2267 /**
2268  * ftrace_update_record - set a record that now is tracing or not
2269  * @rec: the record to update
2270  * @enable: set to true if the record is tracing, false to force disable
2271  *
2272  * The records that represent all functions that can be traced need
2273  * to be updated when tracing has been enabled.
2274  */
2275 int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
2276 {
2277 	return ftrace_check_record(rec, enable, true);
2278 }
2279 
2280 /**
2281  * ftrace_test_record - check if the record has been enabled or not
2282  * @rec: the record to test
2283  * @enable: set to true to check if enabled, false if it is disabled
2284  *
2285  * The arch code may need to test if a record is already set to
2286  * tracing to determine how to modify the function code that it
2287  * represents.
2288  */
2289 int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
2290 {
2291 	return ftrace_check_record(rec, enable, false);
2292 }
2293 
2294 static struct ftrace_ops *
2295 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2296 {
2297 	struct ftrace_ops *op;
2298 	unsigned long ip = rec->ip;
2299 
2300 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2301 
2302 		if (!op->trampoline)
2303 			continue;
2304 
2305 		if (hash_contains_ip(ip, op->func_hash))
2306 			return op;
2307 	} while_for_each_ftrace_op(op);
2308 
2309 	return NULL;
2310 }
2311 
2312 static struct ftrace_ops *
2313 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
2314 {
2315 	struct ftrace_ops *op;
2316 	unsigned long ip = rec->ip;
2317 
2318 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2319 
2320 		if (op == op_exclude || !op->trampoline)
2321 			continue;
2322 
2323 		if (hash_contains_ip(ip, op->func_hash))
2324 			return op;
2325 	} while_for_each_ftrace_op(op);
2326 
2327 	return NULL;
2328 }
2329 
2330 static struct ftrace_ops *
2331 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2332 			   struct ftrace_ops *op)
2333 {
2334 	unsigned long ip = rec->ip;
2335 
2336 	while_for_each_ftrace_op(op) {
2337 
2338 		if (!op->trampoline)
2339 			continue;
2340 
2341 		if (hash_contains_ip(ip, op->func_hash))
2342 			return op;
2343 	}
2344 
2345 	return NULL;
2346 }
2347 
2348 static struct ftrace_ops *
2349 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2350 {
2351 	struct ftrace_ops *op;
2352 	unsigned long ip = rec->ip;
2353 
2354 	/*
2355 	 * Need to check removed ops first.
2356 	 * If they are being removed, and this rec has a tramp,
2357 	 * and this rec is in the ops list, then it would be the
2358 	 * one with the tramp.
2359 	 */
2360 	if (removed_ops) {
2361 		if (hash_contains_ip(ip, &removed_ops->old_hash))
2362 			return removed_ops;
2363 	}
2364 
2365 	/*
2366 	 * Need to find the current trampoline for a rec.
2367 	 * Now, a trampoline is only attached to a rec if there
2368 	 * was a single 'ops' attached to it. But this can be called
2369 	 * when we are adding another op to the rec or removing the
2370 	 * current one. Thus, if the op is being added, we can
2371 	 * ignore it because it hasn't attached itself to the rec
2372 	 * yet.
2373 	 *
2374 	 * If an ops is being modified (hooking to different functions)
2375 	 * then we don't care about the new functions that are being
2376 	 * added, just the old ones (that are probably being removed).
2377 	 *
2378 	 * If we are adding an ops to a function that already is using
2379 	 * a trampoline, it needs to be removed (trampolines are only
2380 	 * for single ops connected), then an ops that is not being
2381 	 * modified also needs to be checked.
2382 	 */
2383 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2384 
2385 		if (!op->trampoline)
2386 			continue;
2387 
2388 		/*
2389 		 * If the ops is being added, it hasn't gotten to
2390 		 * the point to be removed from this tree yet.
2391 		 */
2392 		if (op->flags & FTRACE_OPS_FL_ADDING)
2393 			continue;
2394 
2395 
2396 		/*
2397 		 * If the ops is being modified and is in the old
2398 		 * hash, then it is probably being removed from this
2399 		 * function.
2400 		 */
2401 		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2402 		    hash_contains_ip(ip, &op->old_hash))
2403 			return op;
2404 		/*
2405 		 * If the ops is not being added or modified, and it's
2406 		 * in its normal filter hash, then this must be the one
2407 		 * we want!
2408 		 */
2409 		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2410 		    hash_contains_ip(ip, op->func_hash))
2411 			return op;
2412 
2413 	} while_for_each_ftrace_op(op);
2414 
2415 	return NULL;
2416 }
2417 
2418 static struct ftrace_ops *
2419 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2420 {
2421 	struct ftrace_ops *op;
2422 	unsigned long ip = rec->ip;
2423 
2424 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2425 		/* pass rec in as regs to have non-NULL val */
2426 		if (hash_contains_ip(ip, op->func_hash))
2427 			return op;
2428 	} while_for_each_ftrace_op(op);
2429 
2430 	return NULL;
2431 }
2432 
2433 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
2434 /* Protected by rcu_tasks for reading, and direct_mutex for writing */
2435 static struct ftrace_hash *direct_functions = EMPTY_HASH;
2436 static DEFINE_MUTEX(direct_mutex);
2437 int ftrace_direct_func_count;
2438 
2439 /*
2440  * Search the direct_functions hash to see if the given instruction pointer
2441  * has a direct caller attached to it.
2442  */
2443 unsigned long ftrace_find_rec_direct(unsigned long ip)
2444 {
2445 	struct ftrace_func_entry *entry;
2446 
2447 	entry = __ftrace_lookup_ip(direct_functions, ip);
2448 	if (!entry)
2449 		return 0;
2450 
2451 	return entry->direct;
2452 }
2453 
2454 static struct ftrace_func_entry*
2455 ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
2456 		      struct ftrace_hash **free_hash)
2457 {
2458 	struct ftrace_func_entry *entry;
2459 
2460 	if (ftrace_hash_empty(direct_functions) ||
2461 	    direct_functions->count > 2 * (1 << direct_functions->size_bits)) {
2462 		struct ftrace_hash *new_hash;
2463 		int size = ftrace_hash_empty(direct_functions) ? 0 :
2464 			direct_functions->count + 1;
2465 
2466 		if (size < 32)
2467 			size = 32;
2468 
2469 		new_hash = dup_hash(direct_functions, size);
2470 		if (!new_hash)
2471 			return NULL;
2472 
2473 		*free_hash = direct_functions;
2474 		direct_functions = new_hash;
2475 	}
2476 
2477 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2478 	if (!entry)
2479 		return NULL;
2480 
2481 	entry->ip = ip;
2482 	entry->direct = addr;
2483 	__add_hash_entry(direct_functions, entry);
2484 	return entry;
2485 }
2486 
2487 static void call_direct_funcs(unsigned long ip, unsigned long pip,
2488 			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
2489 {
2490 	struct pt_regs *regs = ftrace_get_regs(fregs);
2491 	unsigned long addr;
2492 
2493 	addr = ftrace_find_rec_direct(ip);
2494 	if (!addr)
2495 		return;
2496 
2497 	arch_ftrace_set_direct_caller(regs, addr);
2498 }
2499 
2500 struct ftrace_ops direct_ops = {
2501 	.func		= call_direct_funcs,
2502 	.flags		= FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS
2503 			  | FTRACE_OPS_FL_PERMANENT,
2504 	/*
2505 	 * By declaring the main trampoline as this trampoline
2506 	 * it will never have one allocated for it. Allocated
2507 	 * trampolines should not call direct functions.
2508 	 * The direct_ops should only be called by the builtin
2509 	 * ftrace_regs_caller trampoline.
2510 	 */
2511 	.trampoline	= FTRACE_REGS_ADDR,
2512 };
2513 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
2514 
2515 /**
2516  * ftrace_get_addr_new - Get the call address to set to
2517  * @rec:  The ftrace record descriptor
2518  *
2519  * If the record has the FTRACE_FL_REGS set, that means that it
2520  * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2521  * is not set, then it wants to convert to the normal callback.
2522  *
2523  * Returns the address of the trampoline to set to
2524  */
2525 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2526 {
2527 	struct ftrace_ops *ops;
2528 	unsigned long addr;
2529 
2530 	if ((rec->flags & FTRACE_FL_DIRECT) &&
2531 	    (ftrace_rec_count(rec) == 1)) {
2532 		addr = ftrace_find_rec_direct(rec->ip);
2533 		if (addr)
2534 			return addr;
2535 		WARN_ON_ONCE(1);
2536 	}
2537 
2538 	/* Trampolines take precedence over regs */
2539 	if (rec->flags & FTRACE_FL_TRAMP) {
2540 		ops = ftrace_find_tramp_ops_new(rec);
2541 		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2542 			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2543 				(void *)rec->ip, (void *)rec->ip, rec->flags);
2544 			/* Ftrace is shutting down, return anything */
2545 			return (unsigned long)FTRACE_ADDR;
2546 		}
2547 		return ops->trampoline;
2548 	}
2549 
2550 	if (rec->flags & FTRACE_FL_REGS)
2551 		return (unsigned long)FTRACE_REGS_ADDR;
2552 	else
2553 		return (unsigned long)FTRACE_ADDR;
2554 }
2555 
2556 /**
2557  * ftrace_get_addr_curr - Get the call address that is already there
2558  * @rec:  The ftrace record descriptor
2559  *
2560  * The FTRACE_FL_REGS_EN is set when the record already points to
2561  * a function that saves all the regs. Basically the '_EN' version
2562  * represents the current state of the function.
2563  *
2564  * Returns the address of the trampoline that is currently being called
2565  */
2566 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2567 {
2568 	struct ftrace_ops *ops;
2569 	unsigned long addr;
2570 
2571 	/* Direct calls take precedence over trampolines */
2572 	if (rec->flags & FTRACE_FL_DIRECT_EN) {
2573 		addr = ftrace_find_rec_direct(rec->ip);
2574 		if (addr)
2575 			return addr;
2576 		WARN_ON_ONCE(1);
2577 	}
2578 
2579 	/* Trampolines take precedence over regs */
2580 	if (rec->flags & FTRACE_FL_TRAMP_EN) {
2581 		ops = ftrace_find_tramp_ops_curr(rec);
2582 		if (FTRACE_WARN_ON(!ops)) {
2583 			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
2584 				(void *)rec->ip, (void *)rec->ip);
2585 			/* Ftrace is shutting down, return anything */
2586 			return (unsigned long)FTRACE_ADDR;
2587 		}
2588 		return ops->trampoline;
2589 	}
2590 
2591 	if (rec->flags & FTRACE_FL_REGS_EN)
2592 		return (unsigned long)FTRACE_REGS_ADDR;
2593 	else
2594 		return (unsigned long)FTRACE_ADDR;
2595 }
2596 
2597 static int
2598 __ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
2599 {
2600 	unsigned long ftrace_old_addr;
2601 	unsigned long ftrace_addr;
2602 	int ret;
2603 
2604 	ftrace_addr = ftrace_get_addr_new(rec);
2605 
2606 	/* This needs to be done before we call ftrace_update_record */
2607 	ftrace_old_addr = ftrace_get_addr_curr(rec);
2608 
2609 	ret = ftrace_update_record(rec, enable);
2610 
2611 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2612 
2613 	switch (ret) {
2614 	case FTRACE_UPDATE_IGNORE:
2615 		return 0;
2616 
2617 	case FTRACE_UPDATE_MAKE_CALL:
2618 		ftrace_bug_type = FTRACE_BUG_CALL;
2619 		return ftrace_make_call(rec, ftrace_addr);
2620 
2621 	case FTRACE_UPDATE_MAKE_NOP:
2622 		ftrace_bug_type = FTRACE_BUG_NOP;
2623 		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2624 
2625 	case FTRACE_UPDATE_MODIFY_CALL:
2626 		ftrace_bug_type = FTRACE_BUG_UPDATE;
2627 		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2628 	}
2629 
2630 	return -1; /* unknown ftrace bug */
2631 }
2632 
2633 void __weak ftrace_replace_code(int mod_flags)
2634 {
2635 	struct dyn_ftrace *rec;
2636 	struct ftrace_page *pg;
2637 	bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
2638 	int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
2639 	int failed;
2640 
2641 	if (unlikely(ftrace_disabled))
2642 		return;
2643 
2644 	do_for_each_ftrace_rec(pg, rec) {
2645 
2646 		if (skip_record(rec))
2647 			continue;
2648 
2649 		failed = __ftrace_replace_code(rec, enable);
2650 		if (failed) {
2651 			ftrace_bug(failed, rec);
2652 			/* Stop processing */
2653 			return;
2654 		}
2655 		if (schedulable)
2656 			cond_resched();
2657 	} while_for_each_ftrace_rec();
2658 }
2659 
2660 struct ftrace_rec_iter {
2661 	struct ftrace_page	*pg;
2662 	int			index;
2663 };
2664 
2665 /**
2666  * ftrace_rec_iter_start - start up iterating over traced functions
2667  *
2668  * Returns an iterator handle that is used to iterate over all
2669  * the records that represent address locations where functions
2670  * are traced.
2671  *
2672  * May return NULL if no records are available.
2673  */
2674 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2675 {
2676 	/*
2677 	 * We only use a single iterator.
2678 	 * Protected by the ftrace_lock mutex.
2679 	 */
2680 	static struct ftrace_rec_iter ftrace_rec_iter;
2681 	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2682 
2683 	iter->pg = ftrace_pages_start;
2684 	iter->index = 0;
2685 
2686 	/* Could have empty pages */
2687 	while (iter->pg && !iter->pg->index)
2688 		iter->pg = iter->pg->next;
2689 
2690 	if (!iter->pg)
2691 		return NULL;
2692 
2693 	return iter;
2694 }
2695 
2696 /**
2697  * ftrace_rec_iter_next - get the next record to process.
2698  * @iter: The handle to the iterator.
2699  *
2700  * Returns the next iterator after the given iterator @iter.
2701  */
2702 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2703 {
2704 	iter->index++;
2705 
2706 	if (iter->index >= iter->pg->index) {
2707 		iter->pg = iter->pg->next;
2708 		iter->index = 0;
2709 
2710 		/* Could have empty pages */
2711 		while (iter->pg && !iter->pg->index)
2712 			iter->pg = iter->pg->next;
2713 	}
2714 
2715 	if (!iter->pg)
2716 		return NULL;
2717 
2718 	return iter;
2719 }
2720 
2721 /**
2722  * ftrace_rec_iter_record - get the record at the iterator location
2723  * @iter: The current iterator location
2724  *
2725  * Returns the record that the current @iter is at.
2726  */
2727 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2728 {
2729 	return &iter->pg->records[iter->index];
2730 }
2731 
2732 static int
2733 ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
2734 {
2735 	int ret;
2736 
2737 	if (unlikely(ftrace_disabled))
2738 		return 0;
2739 
2740 	ret = ftrace_init_nop(mod, rec);
2741 	if (ret) {
2742 		ftrace_bug_type = FTRACE_BUG_INIT;
2743 		ftrace_bug(ret, rec);
2744 		return 0;
2745 	}
2746 	return 1;
2747 }
2748 
2749 /*
2750  * archs can override this function if they must do something
2751  * before the modifying code is performed.
2752  */
2753 void __weak ftrace_arch_code_modify_prepare(void)
2754 {
2755 }
2756 
2757 /*
2758  * archs can override this function if they must do something
2759  * after the modifying code is performed.
2760  */
2761 void __weak ftrace_arch_code_modify_post_process(void)
2762 {
2763 }
2764 
2765 void ftrace_modify_all_code(int command)
2766 {
2767 	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2768 	int mod_flags = 0;
2769 	int err = 0;
2770 
2771 	if (command & FTRACE_MAY_SLEEP)
2772 		mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
2773 
2774 	/*
2775 	 * If the ftrace_caller calls a ftrace_ops func directly,
2776 	 * we need to make sure that it only traces functions it
2777 	 * expects to trace. When doing the switch of functions,
2778 	 * we need to update to the ftrace_ops_list_func first
2779 	 * before the transition between old and new calls are set,
2780 	 * as the ftrace_ops_list_func will check the ops hashes
2781 	 * to make sure the ops are having the right functions
2782 	 * traced.
2783 	 */
2784 	if (update) {
2785 		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2786 		if (FTRACE_WARN_ON(err))
2787 			return;
2788 	}
2789 
2790 	if (command & FTRACE_UPDATE_CALLS)
2791 		ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
2792 	else if (command & FTRACE_DISABLE_CALLS)
2793 		ftrace_replace_code(mod_flags);
2794 
2795 	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2796 		function_trace_op = set_function_trace_op;
2797 		smp_wmb();
2798 		/* If irqs are disabled, we are in stop machine */
2799 		if (!irqs_disabled())
2800 			smp_call_function(ftrace_sync_ipi, NULL, 1);
2801 		err = ftrace_update_ftrace_func(ftrace_trace_function);
2802 		if (FTRACE_WARN_ON(err))
2803 			return;
2804 	}
2805 
2806 	if (command & FTRACE_START_FUNC_RET)
2807 		err = ftrace_enable_ftrace_graph_caller();
2808 	else if (command & FTRACE_STOP_FUNC_RET)
2809 		err = ftrace_disable_ftrace_graph_caller();
2810 	FTRACE_WARN_ON(err);
2811 }
2812 
2813 static int __ftrace_modify_code(void *data)
2814 {
2815 	int *command = data;
2816 
2817 	ftrace_modify_all_code(*command);
2818 
2819 	return 0;
2820 }
2821 
2822 /**
2823  * ftrace_run_stop_machine - go back to the stop machine method
2824  * @command: The command to tell ftrace what to do
2825  *
2826  * If an arch needs to fall back to the stop machine method, the
2827  * it can call this function.
2828  */
2829 void ftrace_run_stop_machine(int command)
2830 {
2831 	stop_machine(__ftrace_modify_code, &command, NULL);
2832 }
2833 
2834 /**
2835  * arch_ftrace_update_code - modify the code to trace or not trace
2836  * @command: The command that needs to be done
2837  *
2838  * Archs can override this function if it does not need to
2839  * run stop_machine() to modify code.
2840  */
2841 void __weak arch_ftrace_update_code(int command)
2842 {
2843 	ftrace_run_stop_machine(command);
2844 }
2845 
2846 static void ftrace_run_update_code(int command)
2847 {
2848 	ftrace_arch_code_modify_prepare();
2849 
2850 	/*
2851 	 * By default we use stop_machine() to modify the code.
2852 	 * But archs can do what ever they want as long as it
2853 	 * is safe. The stop_machine() is the safest, but also
2854 	 * produces the most overhead.
2855 	 */
2856 	arch_ftrace_update_code(command);
2857 
2858 	ftrace_arch_code_modify_post_process();
2859 }
2860 
2861 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2862 				   struct ftrace_ops_hash *old_hash)
2863 {
2864 	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2865 	ops->old_hash.filter_hash = old_hash->filter_hash;
2866 	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2867 	ftrace_run_update_code(command);
2868 	ops->old_hash.filter_hash = NULL;
2869 	ops->old_hash.notrace_hash = NULL;
2870 	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
2871 }
2872 
2873 static ftrace_func_t saved_ftrace_func;
2874 static int ftrace_start_up;
2875 
2876 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
2877 {
2878 }
2879 
2880 /* List of trace_ops that have allocated trampolines */
2881 static LIST_HEAD(ftrace_ops_trampoline_list);
2882 
2883 static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops)
2884 {
2885 	lockdep_assert_held(&ftrace_lock);
2886 	list_add_rcu(&ops->list, &ftrace_ops_trampoline_list);
2887 }
2888 
2889 static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops)
2890 {
2891 	lockdep_assert_held(&ftrace_lock);
2892 	list_del_rcu(&ops->list);
2893 	synchronize_rcu();
2894 }
2895 
2896 /*
2897  * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols
2898  * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
2899  * not a module.
2900  */
2901 #define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace"
2902 #define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline"
2903 
2904 static void ftrace_trampoline_free(struct ftrace_ops *ops)
2905 {
2906 	if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) &&
2907 	    ops->trampoline) {
2908 		/*
2909 		 * Record the text poke event before the ksymbol unregister
2910 		 * event.
2911 		 */
2912 		perf_event_text_poke((void *)ops->trampoline,
2913 				     (void *)ops->trampoline,
2914 				     ops->trampoline_size, NULL, 0);
2915 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
2916 				   ops->trampoline, ops->trampoline_size,
2917 				   true, FTRACE_TRAMPOLINE_SYM);
2918 		/* Remove from kallsyms after the perf events */
2919 		ftrace_remove_trampoline_from_kallsyms(ops);
2920 	}
2921 
2922 	arch_ftrace_trampoline_free(ops);
2923 }
2924 
2925 static void ftrace_startup_enable(int command)
2926 {
2927 	if (saved_ftrace_func != ftrace_trace_function) {
2928 		saved_ftrace_func = ftrace_trace_function;
2929 		command |= FTRACE_UPDATE_TRACE_FUNC;
2930 	}
2931 
2932 	if (!command || !ftrace_enabled)
2933 		return;
2934 
2935 	ftrace_run_update_code(command);
2936 }
2937 
2938 static void ftrace_startup_all(int command)
2939 {
2940 	update_all_ops = true;
2941 	ftrace_startup_enable(command);
2942 	update_all_ops = false;
2943 }
2944 
2945 int ftrace_startup(struct ftrace_ops *ops, int command)
2946 {
2947 	int ret;
2948 
2949 	if (unlikely(ftrace_disabled))
2950 		return -ENODEV;
2951 
2952 	ret = __register_ftrace_function(ops);
2953 	if (ret)
2954 		return ret;
2955 
2956 	ftrace_start_up++;
2957 
2958 	/*
2959 	 * Note that ftrace probes uses this to start up
2960 	 * and modify functions it will probe. But we still
2961 	 * set the ADDING flag for modification, as probes
2962 	 * do not have trampolines. If they add them in the
2963 	 * future, then the probes will need to distinguish
2964 	 * between adding and updating probes.
2965 	 */
2966 	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2967 
2968 	ret = ftrace_hash_ipmodify_enable(ops);
2969 	if (ret < 0) {
2970 		/* Rollback registration process */
2971 		__unregister_ftrace_function(ops);
2972 		ftrace_start_up--;
2973 		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2974 		if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
2975 			ftrace_trampoline_free(ops);
2976 		return ret;
2977 	}
2978 
2979 	if (ftrace_hash_rec_enable(ops, 1))
2980 		command |= FTRACE_UPDATE_CALLS;
2981 
2982 	ftrace_startup_enable(command);
2983 
2984 	/*
2985 	 * If ftrace is in an undefined state, we just remove ops from list
2986 	 * to prevent the NULL pointer, instead of totally rolling it back and
2987 	 * free trampoline, because those actions could cause further damage.
2988 	 */
2989 	if (unlikely(ftrace_disabled)) {
2990 		__unregister_ftrace_function(ops);
2991 		return -ENODEV;
2992 	}
2993 
2994 	ops->flags &= ~FTRACE_OPS_FL_ADDING;
2995 
2996 	return 0;
2997 }
2998 
2999 int ftrace_shutdown(struct ftrace_ops *ops, int command)
3000 {
3001 	int ret;
3002 
3003 	if (unlikely(ftrace_disabled))
3004 		return -ENODEV;
3005 
3006 	ret = __unregister_ftrace_function(ops);
3007 	if (ret)
3008 		return ret;
3009 
3010 	ftrace_start_up--;
3011 	/*
3012 	 * Just warn in case of unbalance, no need to kill ftrace, it's not
3013 	 * critical but the ftrace_call callers may be never nopped again after
3014 	 * further ftrace uses.
3015 	 */
3016 	WARN_ON_ONCE(ftrace_start_up < 0);
3017 
3018 	/* Disabling ipmodify never fails */
3019 	ftrace_hash_ipmodify_disable(ops);
3020 
3021 	if (ftrace_hash_rec_disable(ops, 1))
3022 		command |= FTRACE_UPDATE_CALLS;
3023 
3024 	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
3025 
3026 	if (saved_ftrace_func != ftrace_trace_function) {
3027 		saved_ftrace_func = ftrace_trace_function;
3028 		command |= FTRACE_UPDATE_TRACE_FUNC;
3029 	}
3030 
3031 	if (!command || !ftrace_enabled) {
3032 		/*
3033 		 * If these are dynamic or per_cpu ops, they still
3034 		 * need their data freed. Since, function tracing is
3035 		 * not currently active, we can just free them
3036 		 * without synchronizing all CPUs.
3037 		 */
3038 		if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
3039 			goto free_ops;
3040 
3041 		return 0;
3042 	}
3043 
3044 	/*
3045 	 * If the ops uses a trampoline, then it needs to be
3046 	 * tested first on update.
3047 	 */
3048 	ops->flags |= FTRACE_OPS_FL_REMOVING;
3049 	removed_ops = ops;
3050 
3051 	/* The trampoline logic checks the old hashes */
3052 	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
3053 	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
3054 
3055 	ftrace_run_update_code(command);
3056 
3057 	/*
3058 	 * If there's no more ops registered with ftrace, run a
3059 	 * sanity check to make sure all rec flags are cleared.
3060 	 */
3061 	if (rcu_dereference_protected(ftrace_ops_list,
3062 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
3063 		struct ftrace_page *pg;
3064 		struct dyn_ftrace *rec;
3065 
3066 		do_for_each_ftrace_rec(pg, rec) {
3067 			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
3068 				pr_warn("  %pS flags:%lx\n",
3069 					(void *)rec->ip, rec->flags);
3070 		} while_for_each_ftrace_rec();
3071 	}
3072 
3073 	ops->old_hash.filter_hash = NULL;
3074 	ops->old_hash.notrace_hash = NULL;
3075 
3076 	removed_ops = NULL;
3077 	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
3078 
3079 	/*
3080 	 * Dynamic ops may be freed, we must make sure that all
3081 	 * callers are done before leaving this function.
3082 	 * The same goes for freeing the per_cpu data of the per_cpu
3083 	 * ops.
3084 	 */
3085 	if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
3086 		/*
3087 		 * We need to do a hard force of sched synchronization.
3088 		 * This is because we use preempt_disable() to do RCU, but
3089 		 * the function tracers can be called where RCU is not watching
3090 		 * (like before user_exit()). We can not rely on the RCU
3091 		 * infrastructure to do the synchronization, thus we must do it
3092 		 * ourselves.
3093 		 */
3094 		synchronize_rcu_tasks_rude();
3095 
3096 		/*
3097 		 * When the kernel is preemptive, tasks can be preempted
3098 		 * while on a ftrace trampoline. Just scheduling a task on
3099 		 * a CPU is not good enough to flush them. Calling
3100 		 * synchronize_rcu_tasks() will wait for those tasks to
3101 		 * execute and either schedule voluntarily or enter user space.
3102 		 */
3103 		if (IS_ENABLED(CONFIG_PREEMPTION))
3104 			synchronize_rcu_tasks();
3105 
3106  free_ops:
3107 		ftrace_trampoline_free(ops);
3108 	}
3109 
3110 	return 0;
3111 }
3112 
3113 static u64		ftrace_update_time;
3114 unsigned long		ftrace_update_tot_cnt;
3115 unsigned long		ftrace_number_of_pages;
3116 unsigned long		ftrace_number_of_groups;
3117 
3118 static inline int ops_traces_mod(struct ftrace_ops *ops)
3119 {
3120 	/*
3121 	 * Filter_hash being empty will default to trace module.
3122 	 * But notrace hash requires a test of individual module functions.
3123 	 */
3124 	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
3125 		ftrace_hash_empty(ops->func_hash->notrace_hash);
3126 }
3127 
3128 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
3129 {
3130 	bool init_nop = ftrace_need_init_nop();
3131 	struct ftrace_page *pg;
3132 	struct dyn_ftrace *p;
3133 	u64 start, stop;
3134 	unsigned long update_cnt = 0;
3135 	unsigned long rec_flags = 0;
3136 	int i;
3137 
3138 	start = ftrace_now(raw_smp_processor_id());
3139 
3140 	/*
3141 	 * When a module is loaded, this function is called to convert
3142 	 * the calls to mcount in its text to nops, and also to create
3143 	 * an entry in the ftrace data. Now, if ftrace is activated
3144 	 * after this call, but before the module sets its text to
3145 	 * read-only, the modification of enabling ftrace can fail if
3146 	 * the read-only is done while ftrace is converting the calls.
3147 	 * To prevent this, the module's records are set as disabled
3148 	 * and will be enabled after the call to set the module's text
3149 	 * to read-only.
3150 	 */
3151 	if (mod)
3152 		rec_flags |= FTRACE_FL_DISABLED;
3153 
3154 	for (pg = new_pgs; pg; pg = pg->next) {
3155 
3156 		for (i = 0; i < pg->index; i++) {
3157 
3158 			/* If something went wrong, bail without enabling anything */
3159 			if (unlikely(ftrace_disabled))
3160 				return -1;
3161 
3162 			p = &pg->records[i];
3163 			p->flags = rec_flags;
3164 
3165 			/*
3166 			 * Do the initial record conversion from mcount jump
3167 			 * to the NOP instructions.
3168 			 */
3169 			if (init_nop && !ftrace_nop_initialize(mod, p))
3170 				break;
3171 
3172 			update_cnt++;
3173 		}
3174 	}
3175 
3176 	stop = ftrace_now(raw_smp_processor_id());
3177 	ftrace_update_time = stop - start;
3178 	ftrace_update_tot_cnt += update_cnt;
3179 
3180 	return 0;
3181 }
3182 
3183 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
3184 {
3185 	int order;
3186 	int pages;
3187 	int cnt;
3188 
3189 	if (WARN_ON(!count))
3190 		return -EINVAL;
3191 
3192 	/* We want to fill as much as possible, with no empty pages */
3193 	pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE);
3194 	order = fls(pages) - 1;
3195 
3196  again:
3197 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
3198 
3199 	if (!pg->records) {
3200 		/* if we can't allocate this size, try something smaller */
3201 		if (!order)
3202 			return -ENOMEM;
3203 		order >>= 1;
3204 		goto again;
3205 	}
3206 
3207 	ftrace_number_of_pages += 1 << order;
3208 	ftrace_number_of_groups++;
3209 
3210 	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
3211 	pg->order = order;
3212 
3213 	if (cnt > count)
3214 		cnt = count;
3215 
3216 	return cnt;
3217 }
3218 
3219 static struct ftrace_page *
3220 ftrace_allocate_pages(unsigned long num_to_init)
3221 {
3222 	struct ftrace_page *start_pg;
3223 	struct ftrace_page *pg;
3224 	int cnt;
3225 
3226 	if (!num_to_init)
3227 		return NULL;
3228 
3229 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
3230 	if (!pg)
3231 		return NULL;
3232 
3233 	/*
3234 	 * Try to allocate as much as possible in one continues
3235 	 * location that fills in all of the space. We want to
3236 	 * waste as little space as possible.
3237 	 */
3238 	for (;;) {
3239 		cnt = ftrace_allocate_records(pg, num_to_init);
3240 		if (cnt < 0)
3241 			goto free_pages;
3242 
3243 		num_to_init -= cnt;
3244 		if (!num_to_init)
3245 			break;
3246 
3247 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
3248 		if (!pg->next)
3249 			goto free_pages;
3250 
3251 		pg = pg->next;
3252 	}
3253 
3254 	return start_pg;
3255 
3256  free_pages:
3257 	pg = start_pg;
3258 	while (pg) {
3259 		if (pg->records) {
3260 			free_pages((unsigned long)pg->records, pg->order);
3261 			ftrace_number_of_pages -= 1 << pg->order;
3262 		}
3263 		start_pg = pg->next;
3264 		kfree(pg);
3265 		pg = start_pg;
3266 		ftrace_number_of_groups--;
3267 	}
3268 	pr_info("ftrace: FAILED to allocate memory for functions\n");
3269 	return NULL;
3270 }
3271 
3272 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
3273 
3274 struct ftrace_iterator {
3275 	loff_t				pos;
3276 	loff_t				func_pos;
3277 	loff_t				mod_pos;
3278 	struct ftrace_page		*pg;
3279 	struct dyn_ftrace		*func;
3280 	struct ftrace_func_probe	*probe;
3281 	struct ftrace_func_entry	*probe_entry;
3282 	struct trace_parser		parser;
3283 	struct ftrace_hash		*hash;
3284 	struct ftrace_ops		*ops;
3285 	struct trace_array		*tr;
3286 	struct list_head		*mod_list;
3287 	int				pidx;
3288 	int				idx;
3289 	unsigned			flags;
3290 };
3291 
3292 static void *
3293 t_probe_next(struct seq_file *m, loff_t *pos)
3294 {
3295 	struct ftrace_iterator *iter = m->private;
3296 	struct trace_array *tr = iter->ops->private;
3297 	struct list_head *func_probes;
3298 	struct ftrace_hash *hash;
3299 	struct list_head *next;
3300 	struct hlist_node *hnd = NULL;
3301 	struct hlist_head *hhd;
3302 	int size;
3303 
3304 	(*pos)++;
3305 	iter->pos = *pos;
3306 
3307 	if (!tr)
3308 		return NULL;
3309 
3310 	func_probes = &tr->func_probes;
3311 	if (list_empty(func_probes))
3312 		return NULL;
3313 
3314 	if (!iter->probe) {
3315 		next = func_probes->next;
3316 		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3317 	}
3318 
3319 	if (iter->probe_entry)
3320 		hnd = &iter->probe_entry->hlist;
3321 
3322 	hash = iter->probe->ops.func_hash->filter_hash;
3323 
3324 	/*
3325 	 * A probe being registered may temporarily have an empty hash
3326 	 * and it's at the end of the func_probes list.
3327 	 */
3328 	if (!hash || hash == EMPTY_HASH)
3329 		return NULL;
3330 
3331 	size = 1 << hash->size_bits;
3332 
3333  retry:
3334 	if (iter->pidx >= size) {
3335 		if (iter->probe->list.next == func_probes)
3336 			return NULL;
3337 		next = iter->probe->list.next;
3338 		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3339 		hash = iter->probe->ops.func_hash->filter_hash;
3340 		size = 1 << hash->size_bits;
3341 		iter->pidx = 0;
3342 	}
3343 
3344 	hhd = &hash->buckets[iter->pidx];
3345 
3346 	if (hlist_empty(hhd)) {
3347 		iter->pidx++;
3348 		hnd = NULL;
3349 		goto retry;
3350 	}
3351 
3352 	if (!hnd)
3353 		hnd = hhd->first;
3354 	else {
3355 		hnd = hnd->next;
3356 		if (!hnd) {
3357 			iter->pidx++;
3358 			goto retry;
3359 		}
3360 	}
3361 
3362 	if (WARN_ON_ONCE(!hnd))
3363 		return NULL;
3364 
3365 	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3366 
3367 	return iter;
3368 }
3369 
3370 static void *t_probe_start(struct seq_file *m, loff_t *pos)
3371 {
3372 	struct ftrace_iterator *iter = m->private;
3373 	void *p = NULL;
3374 	loff_t l;
3375 
3376 	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3377 		return NULL;
3378 
3379 	if (iter->mod_pos > *pos)
3380 		return NULL;
3381 
3382 	iter->probe = NULL;
3383 	iter->probe_entry = NULL;
3384 	iter->pidx = 0;
3385 	for (l = 0; l <= (*pos - iter->mod_pos); ) {
3386 		p = t_probe_next(m, &l);
3387 		if (!p)
3388 			break;
3389 	}
3390 	if (!p)
3391 		return NULL;
3392 
3393 	/* Only set this if we have an item */
3394 	iter->flags |= FTRACE_ITER_PROBE;
3395 
3396 	return iter;
3397 }
3398 
3399 static int
3400 t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3401 {
3402 	struct ftrace_func_entry *probe_entry;
3403 	struct ftrace_probe_ops *probe_ops;
3404 	struct ftrace_func_probe *probe;
3405 
3406 	probe = iter->probe;
3407 	probe_entry = iter->probe_entry;
3408 
3409 	if (WARN_ON_ONCE(!probe || !probe_entry))
3410 		return -EIO;
3411 
3412 	probe_ops = probe->probe_ops;
3413 
3414 	if (probe_ops->print)
3415 		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3416 
3417 	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
3418 		   (void *)probe_ops->func);
3419 
3420 	return 0;
3421 }
3422 
3423 static void *
3424 t_mod_next(struct seq_file *m, loff_t *pos)
3425 {
3426 	struct ftrace_iterator *iter = m->private;
3427 	struct trace_array *tr = iter->tr;
3428 
3429 	(*pos)++;
3430 	iter->pos = *pos;
3431 
3432 	iter->mod_list = iter->mod_list->next;
3433 
3434 	if (iter->mod_list == &tr->mod_trace ||
3435 	    iter->mod_list == &tr->mod_notrace) {
3436 		iter->flags &= ~FTRACE_ITER_MOD;
3437 		return NULL;
3438 	}
3439 
3440 	iter->mod_pos = *pos;
3441 
3442 	return iter;
3443 }
3444 
3445 static void *t_mod_start(struct seq_file *m, loff_t *pos)
3446 {
3447 	struct ftrace_iterator *iter = m->private;
3448 	void *p = NULL;
3449 	loff_t l;
3450 
3451 	if (iter->func_pos > *pos)
3452 		return NULL;
3453 
3454 	iter->mod_pos = iter->func_pos;
3455 
3456 	/* probes are only available if tr is set */
3457 	if (!iter->tr)
3458 		return NULL;
3459 
3460 	for (l = 0; l <= (*pos - iter->func_pos); ) {
3461 		p = t_mod_next(m, &l);
3462 		if (!p)
3463 			break;
3464 	}
3465 	if (!p) {
3466 		iter->flags &= ~FTRACE_ITER_MOD;
3467 		return t_probe_start(m, pos);
3468 	}
3469 
3470 	/* Only set this if we have an item */
3471 	iter->flags |= FTRACE_ITER_MOD;
3472 
3473 	return iter;
3474 }
3475 
3476 static int
3477 t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
3478 {
3479 	struct ftrace_mod_load *ftrace_mod;
3480 	struct trace_array *tr = iter->tr;
3481 
3482 	if (WARN_ON_ONCE(!iter->mod_list) ||
3483 			 iter->mod_list == &tr->mod_trace ||
3484 			 iter->mod_list == &tr->mod_notrace)
3485 		return -EIO;
3486 
3487 	ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
3488 
3489 	if (ftrace_mod->func)
3490 		seq_printf(m, "%s", ftrace_mod->func);
3491 	else
3492 		seq_putc(m, '*');
3493 
3494 	seq_printf(m, ":mod:%s\n", ftrace_mod->module);
3495 
3496 	return 0;
3497 }
3498 
3499 static void *
3500 t_func_next(struct seq_file *m, loff_t *pos)
3501 {
3502 	struct ftrace_iterator *iter = m->private;
3503 	struct dyn_ftrace *rec = NULL;
3504 
3505 	(*pos)++;
3506 
3507  retry:
3508 	if (iter->idx >= iter->pg->index) {
3509 		if (iter->pg->next) {
3510 			iter->pg = iter->pg->next;
3511 			iter->idx = 0;
3512 			goto retry;
3513 		}
3514 	} else {
3515 		rec = &iter->pg->records[iter->idx++];
3516 		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3517 		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3518 
3519 		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3520 		     !(rec->flags & FTRACE_FL_ENABLED))) {
3521 
3522 			rec = NULL;
3523 			goto retry;
3524 		}
3525 	}
3526 
3527 	if (!rec)
3528 		return NULL;
3529 
3530 	iter->pos = iter->func_pos = *pos;
3531 	iter->func = rec;
3532 
3533 	return iter;
3534 }
3535 
3536 static void *
3537 t_next(struct seq_file *m, void *v, loff_t *pos)
3538 {
3539 	struct ftrace_iterator *iter = m->private;
3540 	loff_t l = *pos; /* t_probe_start() must use original pos */
3541 	void *ret;
3542 
3543 	if (unlikely(ftrace_disabled))
3544 		return NULL;
3545 
3546 	if (iter->flags & FTRACE_ITER_PROBE)
3547 		return t_probe_next(m, pos);
3548 
3549 	if (iter->flags & FTRACE_ITER_MOD)
3550 		return t_mod_next(m, pos);
3551 
3552 	if (iter->flags & FTRACE_ITER_PRINTALL) {
3553 		/* next must increment pos, and t_probe_start does not */
3554 		(*pos)++;
3555 		return t_mod_start(m, &l);
3556 	}
3557 
3558 	ret = t_func_next(m, pos);
3559 
3560 	if (!ret)
3561 		return t_mod_start(m, &l);
3562 
3563 	return ret;
3564 }
3565 
3566 static void reset_iter_read(struct ftrace_iterator *iter)
3567 {
3568 	iter->pos = 0;
3569 	iter->func_pos = 0;
3570 	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
3571 }
3572 
3573 static void *t_start(struct seq_file *m, loff_t *pos)
3574 {
3575 	struct ftrace_iterator *iter = m->private;
3576 	void *p = NULL;
3577 	loff_t l;
3578 
3579 	mutex_lock(&ftrace_lock);
3580 
3581 	if (unlikely(ftrace_disabled))
3582 		return NULL;
3583 
3584 	/*
3585 	 * If an lseek was done, then reset and start from beginning.
3586 	 */
3587 	if (*pos < iter->pos)
3588 		reset_iter_read(iter);
3589 
3590 	/*
3591 	 * For set_ftrace_filter reading, if we have the filter
3592 	 * off, we can short cut and just print out that all
3593 	 * functions are enabled.
3594 	 */
3595 	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3596 	    ftrace_hash_empty(iter->hash)) {
3597 		iter->func_pos = 1; /* Account for the message */
3598 		if (*pos > 0)
3599 			return t_mod_start(m, pos);
3600 		iter->flags |= FTRACE_ITER_PRINTALL;
3601 		/* reset in case of seek/pread */
3602 		iter->flags &= ~FTRACE_ITER_PROBE;
3603 		return iter;
3604 	}
3605 
3606 	if (iter->flags & FTRACE_ITER_MOD)
3607 		return t_mod_start(m, pos);
3608 
3609 	/*
3610 	 * Unfortunately, we need to restart at ftrace_pages_start
3611 	 * every time we let go of the ftrace_mutex. This is because
3612 	 * those pointers can change without the lock.
3613 	 */
3614 	iter->pg = ftrace_pages_start;
3615 	iter->idx = 0;
3616 	for (l = 0; l <= *pos; ) {
3617 		p = t_func_next(m, &l);
3618 		if (!p)
3619 			break;
3620 	}
3621 
3622 	if (!p)
3623 		return t_mod_start(m, pos);
3624 
3625 	return iter;
3626 }
3627 
3628 static void t_stop(struct seq_file *m, void *p)
3629 {
3630 	mutex_unlock(&ftrace_lock);
3631 }
3632 
3633 void * __weak
3634 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
3635 {
3636 	return NULL;
3637 }
3638 
3639 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
3640 				struct dyn_ftrace *rec)
3641 {
3642 	void *ptr;
3643 
3644 	ptr = arch_ftrace_trampoline_func(ops, rec);
3645 	if (ptr)
3646 		seq_printf(m, " ->%pS", ptr);
3647 }
3648 
3649 #ifdef FTRACE_MCOUNT_MAX_OFFSET
3650 /*
3651  * Weak functions can still have an mcount/fentry that is saved in
3652  * the __mcount_loc section. These can be detected by having a
3653  * symbol offset of greater than FTRACE_MCOUNT_MAX_OFFSET, as the
3654  * symbol found by kallsyms is not the function that the mcount/fentry
3655  * is part of. The offset is much greater in these cases.
3656  *
3657  * Test the record to make sure that the ip points to a valid kallsyms
3658  * and if not, mark it disabled.
3659  */
3660 static int test_for_valid_rec(struct dyn_ftrace *rec)
3661 {
3662 	char str[KSYM_SYMBOL_LEN];
3663 	unsigned long offset;
3664 	const char *ret;
3665 
3666 	ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str);
3667 
3668 	/* Weak functions can cause invalid addresses */
3669 	if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
3670 		rec->flags |= FTRACE_FL_DISABLED;
3671 		return 0;
3672 	}
3673 	return 1;
3674 }
3675 
3676 static struct workqueue_struct *ftrace_check_wq __initdata;
3677 static struct work_struct ftrace_check_work __initdata;
3678 
3679 /*
3680  * Scan all the mcount/fentry entries to make sure they are valid.
3681  */
3682 static __init void ftrace_check_work_func(struct work_struct *work)
3683 {
3684 	struct ftrace_page *pg;
3685 	struct dyn_ftrace *rec;
3686 
3687 	mutex_lock(&ftrace_lock);
3688 	do_for_each_ftrace_rec(pg, rec) {
3689 		test_for_valid_rec(rec);
3690 	} while_for_each_ftrace_rec();
3691 	mutex_unlock(&ftrace_lock);
3692 }
3693 
3694 static int __init ftrace_check_for_weak_functions(void)
3695 {
3696 	INIT_WORK(&ftrace_check_work, ftrace_check_work_func);
3697 
3698 	ftrace_check_wq = alloc_workqueue("ftrace_check_wq", WQ_UNBOUND, 0);
3699 
3700 	queue_work(ftrace_check_wq, &ftrace_check_work);
3701 	return 0;
3702 }
3703 
3704 static int __init ftrace_check_sync(void)
3705 {
3706 	/* Make sure the ftrace_check updates are finished */
3707 	if (ftrace_check_wq)
3708 		destroy_workqueue(ftrace_check_wq);
3709 	return 0;
3710 }
3711 
3712 late_initcall_sync(ftrace_check_sync);
3713 subsys_initcall(ftrace_check_for_weak_functions);
3714 
3715 static int print_rec(struct seq_file *m, unsigned long ip)
3716 {
3717 	unsigned long offset;
3718 	char str[KSYM_SYMBOL_LEN];
3719 	char *modname;
3720 	const char *ret;
3721 
3722 	ret = kallsyms_lookup(ip, NULL, &offset, &modname, str);
3723 	/* Weak functions can cause invalid addresses */
3724 	if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
3725 		snprintf(str, KSYM_SYMBOL_LEN, "%s_%ld",
3726 			 FTRACE_INVALID_FUNCTION, offset);
3727 		ret = NULL;
3728 	}
3729 
3730 	seq_puts(m, str);
3731 	if (modname)
3732 		seq_printf(m, " [%s]", modname);
3733 	return ret == NULL ? -1 : 0;
3734 }
3735 #else
3736 static inline int test_for_valid_rec(struct dyn_ftrace *rec)
3737 {
3738 	return 1;
3739 }
3740 
3741 static inline int print_rec(struct seq_file *m, unsigned long ip)
3742 {
3743 	seq_printf(m, "%ps", (void *)ip);
3744 	return 0;
3745 }
3746 #endif
3747 
3748 static int t_show(struct seq_file *m, void *v)
3749 {
3750 	struct ftrace_iterator *iter = m->private;
3751 	struct dyn_ftrace *rec;
3752 
3753 	if (iter->flags & FTRACE_ITER_PROBE)
3754 		return t_probe_show(m, iter);
3755 
3756 	if (iter->flags & FTRACE_ITER_MOD)
3757 		return t_mod_show(m, iter);
3758 
3759 	if (iter->flags & FTRACE_ITER_PRINTALL) {
3760 		if (iter->flags & FTRACE_ITER_NOTRACE)
3761 			seq_puts(m, "#### no functions disabled ####\n");
3762 		else
3763 			seq_puts(m, "#### all functions enabled ####\n");
3764 		return 0;
3765 	}
3766 
3767 	rec = iter->func;
3768 
3769 	if (!rec)
3770 		return 0;
3771 
3772 	if (print_rec(m, rec->ip)) {
3773 		/* This should only happen when a rec is disabled */
3774 		WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED));
3775 		seq_putc(m, '\n');
3776 		return 0;
3777 	}
3778 
3779 	if (iter->flags & FTRACE_ITER_ENABLED) {
3780 		struct ftrace_ops *ops;
3781 
3782 		seq_printf(m, " (%ld)%s%s%s",
3783 			   ftrace_rec_count(rec),
3784 			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
3785 			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ",
3786 			   rec->flags & FTRACE_FL_DIRECT ? " D" : "  ");
3787 		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3788 			ops = ftrace_find_tramp_ops_any(rec);
3789 			if (ops) {
3790 				do {
3791 					seq_printf(m, "\ttramp: %pS (%pS)",
3792 						   (void *)ops->trampoline,
3793 						   (void *)ops->func);
3794 					add_trampoline_func(m, ops, rec);
3795 					ops = ftrace_find_tramp_ops_next(rec, ops);
3796 				} while (ops);
3797 			} else
3798 				seq_puts(m, "\ttramp: ERROR!");
3799 		} else {
3800 			add_trampoline_func(m, NULL, rec);
3801 		}
3802 		if (rec->flags & FTRACE_FL_DIRECT) {
3803 			unsigned long direct;
3804 
3805 			direct = ftrace_find_rec_direct(rec->ip);
3806 			if (direct)
3807 				seq_printf(m, "\n\tdirect-->%pS", (void *)direct);
3808 		}
3809 	}
3810 
3811 	seq_putc(m, '\n');
3812 
3813 	return 0;
3814 }
3815 
3816 static const struct seq_operations show_ftrace_seq_ops = {
3817 	.start = t_start,
3818 	.next = t_next,
3819 	.stop = t_stop,
3820 	.show = t_show,
3821 };
3822 
3823 static int
3824 ftrace_avail_open(struct inode *inode, struct file *file)
3825 {
3826 	struct ftrace_iterator *iter;
3827 	int ret;
3828 
3829 	ret = security_locked_down(LOCKDOWN_TRACEFS);
3830 	if (ret)
3831 		return ret;
3832 
3833 	if (unlikely(ftrace_disabled))
3834 		return -ENODEV;
3835 
3836 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3837 	if (!iter)
3838 		return -ENOMEM;
3839 
3840 	iter->pg = ftrace_pages_start;
3841 	iter->ops = &global_ops;
3842 
3843 	return 0;
3844 }
3845 
3846 static int
3847 ftrace_enabled_open(struct inode *inode, struct file *file)
3848 {
3849 	struct ftrace_iterator *iter;
3850 
3851 	/*
3852 	 * This shows us what functions are currently being
3853 	 * traced and by what. Not sure if we want lockdown
3854 	 * to hide such critical information for an admin.
3855 	 * Although, perhaps it can show information we don't
3856 	 * want people to see, but if something is tracing
3857 	 * something, we probably want to know about it.
3858 	 */
3859 
3860 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3861 	if (!iter)
3862 		return -ENOMEM;
3863 
3864 	iter->pg = ftrace_pages_start;
3865 	iter->flags = FTRACE_ITER_ENABLED;
3866 	iter->ops = &global_ops;
3867 
3868 	return 0;
3869 }
3870 
3871 /**
3872  * ftrace_regex_open - initialize function tracer filter files
3873  * @ops: The ftrace_ops that hold the hash filters
3874  * @flag: The type of filter to process
3875  * @inode: The inode, usually passed in to your open routine
3876  * @file: The file, usually passed in to your open routine
3877  *
3878  * ftrace_regex_open() initializes the filter files for the
3879  * @ops. Depending on @flag it may process the filter hash or
3880  * the notrace hash of @ops. With this called from the open
3881  * routine, you can use ftrace_filter_write() for the write
3882  * routine if @flag has FTRACE_ITER_FILTER set, or
3883  * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3884  * tracing_lseek() should be used as the lseek routine, and
3885  * release must call ftrace_regex_release().
3886  */
3887 int
3888 ftrace_regex_open(struct ftrace_ops *ops, int flag,
3889 		  struct inode *inode, struct file *file)
3890 {
3891 	struct ftrace_iterator *iter;
3892 	struct ftrace_hash *hash;
3893 	struct list_head *mod_head;
3894 	struct trace_array *tr = ops->private;
3895 	int ret = -ENOMEM;
3896 
3897 	ftrace_ops_init(ops);
3898 
3899 	if (unlikely(ftrace_disabled))
3900 		return -ENODEV;
3901 
3902 	if (tracing_check_open_get_tr(tr))
3903 		return -ENODEV;
3904 
3905 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3906 	if (!iter)
3907 		goto out;
3908 
3909 	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
3910 		goto out;
3911 
3912 	iter->ops = ops;
3913 	iter->flags = flag;
3914 	iter->tr = tr;
3915 
3916 	mutex_lock(&ops->func_hash->regex_lock);
3917 
3918 	if (flag & FTRACE_ITER_NOTRACE) {
3919 		hash = ops->func_hash->notrace_hash;
3920 		mod_head = tr ? &tr->mod_notrace : NULL;
3921 	} else {
3922 		hash = ops->func_hash->filter_hash;
3923 		mod_head = tr ? &tr->mod_trace : NULL;
3924 	}
3925 
3926 	iter->mod_list = mod_head;
3927 
3928 	if (file->f_mode & FMODE_WRITE) {
3929 		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
3930 
3931 		if (file->f_flags & O_TRUNC) {
3932 			iter->hash = alloc_ftrace_hash(size_bits);
3933 			clear_ftrace_mod_list(mod_head);
3934 	        } else {
3935 			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
3936 		}
3937 
3938 		if (!iter->hash) {
3939 			trace_parser_put(&iter->parser);
3940 			goto out_unlock;
3941 		}
3942 	} else
3943 		iter->hash = hash;
3944 
3945 	ret = 0;
3946 
3947 	if (file->f_mode & FMODE_READ) {
3948 		iter->pg = ftrace_pages_start;
3949 
3950 		ret = seq_open(file, &show_ftrace_seq_ops);
3951 		if (!ret) {
3952 			struct seq_file *m = file->private_data;
3953 			m->private = iter;
3954 		} else {
3955 			/* Failed */
3956 			free_ftrace_hash(iter->hash);
3957 			trace_parser_put(&iter->parser);
3958 		}
3959 	} else
3960 		file->private_data = iter;
3961 
3962  out_unlock:
3963 	mutex_unlock(&ops->func_hash->regex_lock);
3964 
3965  out:
3966 	if (ret) {
3967 		kfree(iter);
3968 		if (tr)
3969 			trace_array_put(tr);
3970 	}
3971 
3972 	return ret;
3973 }
3974 
3975 static int
3976 ftrace_filter_open(struct inode *inode, struct file *file)
3977 {
3978 	struct ftrace_ops *ops = inode->i_private;
3979 
3980 	/* Checks for tracefs lockdown */
3981 	return ftrace_regex_open(ops,
3982 			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
3983 			inode, file);
3984 }
3985 
3986 static int
3987 ftrace_notrace_open(struct inode *inode, struct file *file)
3988 {
3989 	struct ftrace_ops *ops = inode->i_private;
3990 
3991 	/* Checks for tracefs lockdown */
3992 	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3993 				 inode, file);
3994 }
3995 
3996 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
3997 struct ftrace_glob {
3998 	char *search;
3999 	unsigned len;
4000 	int type;
4001 };
4002 
4003 /*
4004  * If symbols in an architecture don't correspond exactly to the user-visible
4005  * name of what they represent, it is possible to define this function to
4006  * perform the necessary adjustments.
4007 */
4008 char * __weak arch_ftrace_match_adjust(char *str, const char *search)
4009 {
4010 	return str;
4011 }
4012 
4013 static int ftrace_match(char *str, struct ftrace_glob *g)
4014 {
4015 	int matched = 0;
4016 	int slen;
4017 
4018 	str = arch_ftrace_match_adjust(str, g->search);
4019 
4020 	switch (g->type) {
4021 	case MATCH_FULL:
4022 		if (strcmp(str, g->search) == 0)
4023 			matched = 1;
4024 		break;
4025 	case MATCH_FRONT_ONLY:
4026 		if (strncmp(str, g->search, g->len) == 0)
4027 			matched = 1;
4028 		break;
4029 	case MATCH_MIDDLE_ONLY:
4030 		if (strstr(str, g->search))
4031 			matched = 1;
4032 		break;
4033 	case MATCH_END_ONLY:
4034 		slen = strlen(str);
4035 		if (slen >= g->len &&
4036 		    memcmp(str + slen - g->len, g->search, g->len) == 0)
4037 			matched = 1;
4038 		break;
4039 	case MATCH_GLOB:
4040 		if (glob_match(g->search, str))
4041 			matched = 1;
4042 		break;
4043 	}
4044 
4045 	return matched;
4046 }
4047 
4048 static int
4049 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
4050 {
4051 	struct ftrace_func_entry *entry;
4052 	int ret = 0;
4053 
4054 	entry = ftrace_lookup_ip(hash, rec->ip);
4055 	if (clear_filter) {
4056 		/* Do nothing if it doesn't exist */
4057 		if (!entry)
4058 			return 0;
4059 
4060 		free_hash_entry(hash, entry);
4061 	} else {
4062 		/* Do nothing if it exists */
4063 		if (entry)
4064 			return 0;
4065 
4066 		ret = add_hash_entry(hash, rec->ip);
4067 	}
4068 	return ret;
4069 }
4070 
4071 static int
4072 add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
4073 		 int clear_filter)
4074 {
4075 	long index = simple_strtoul(func_g->search, NULL, 0);
4076 	struct ftrace_page *pg;
4077 	struct dyn_ftrace *rec;
4078 
4079 	/* The index starts at 1 */
4080 	if (--index < 0)
4081 		return 0;
4082 
4083 	do_for_each_ftrace_rec(pg, rec) {
4084 		if (pg->index <= index) {
4085 			index -= pg->index;
4086 			/* this is a double loop, break goes to the next page */
4087 			break;
4088 		}
4089 		rec = &pg->records[index];
4090 		enter_record(hash, rec, clear_filter);
4091 		return 1;
4092 	} while_for_each_ftrace_rec();
4093 	return 0;
4094 }
4095 
4096 #ifdef FTRACE_MCOUNT_MAX_OFFSET
4097 static int lookup_ip(unsigned long ip, char **modname, char *str)
4098 {
4099 	unsigned long offset;
4100 
4101 	kallsyms_lookup(ip, NULL, &offset, modname, str);
4102 	if (offset > FTRACE_MCOUNT_MAX_OFFSET)
4103 		return -1;
4104 	return 0;
4105 }
4106 #else
4107 static int lookup_ip(unsigned long ip, char **modname, char *str)
4108 {
4109 	kallsyms_lookup(ip, NULL, NULL, modname, str);
4110 	return 0;
4111 }
4112 #endif
4113 
4114 static int
4115 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
4116 		struct ftrace_glob *mod_g, int exclude_mod)
4117 {
4118 	char str[KSYM_SYMBOL_LEN];
4119 	char *modname;
4120 
4121 	if (lookup_ip(rec->ip, &modname, str)) {
4122 		/* This should only happen when a rec is disabled */
4123 		WARN_ON_ONCE(system_state == SYSTEM_RUNNING &&
4124 			     !(rec->flags & FTRACE_FL_DISABLED));
4125 		return 0;
4126 	}
4127 
4128 	if (mod_g) {
4129 		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
4130 
4131 		/* blank module name to match all modules */
4132 		if (!mod_g->len) {
4133 			/* blank module globbing: modname xor exclude_mod */
4134 			if (!exclude_mod != !modname)
4135 				goto func_match;
4136 			return 0;
4137 		}
4138 
4139 		/*
4140 		 * exclude_mod is set to trace everything but the given
4141 		 * module. If it is set and the module matches, then
4142 		 * return 0. If it is not set, and the module doesn't match
4143 		 * also return 0. Otherwise, check the function to see if
4144 		 * that matches.
4145 		 */
4146 		if (!mod_matches == !exclude_mod)
4147 			return 0;
4148 func_match:
4149 		/* blank search means to match all funcs in the mod */
4150 		if (!func_g->len)
4151 			return 1;
4152 	}
4153 
4154 	return ftrace_match(str, func_g);
4155 }
4156 
4157 static int
4158 match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
4159 {
4160 	struct ftrace_page *pg;
4161 	struct dyn_ftrace *rec;
4162 	struct ftrace_glob func_g = { .type = MATCH_FULL };
4163 	struct ftrace_glob mod_g = { .type = MATCH_FULL };
4164 	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
4165 	int exclude_mod = 0;
4166 	int found = 0;
4167 	int ret;
4168 	int clear_filter = 0;
4169 
4170 	if (func) {
4171 		func_g.type = filter_parse_regex(func, len, &func_g.search,
4172 						 &clear_filter);
4173 		func_g.len = strlen(func_g.search);
4174 	}
4175 
4176 	if (mod) {
4177 		mod_g.type = filter_parse_regex(mod, strlen(mod),
4178 				&mod_g.search, &exclude_mod);
4179 		mod_g.len = strlen(mod_g.search);
4180 	}
4181 
4182 	mutex_lock(&ftrace_lock);
4183 
4184 	if (unlikely(ftrace_disabled))
4185 		goto out_unlock;
4186 
4187 	if (func_g.type == MATCH_INDEX) {
4188 		found = add_rec_by_index(hash, &func_g, clear_filter);
4189 		goto out_unlock;
4190 	}
4191 
4192 	do_for_each_ftrace_rec(pg, rec) {
4193 
4194 		if (rec->flags & FTRACE_FL_DISABLED)
4195 			continue;
4196 
4197 		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
4198 			ret = enter_record(hash, rec, clear_filter);
4199 			if (ret < 0) {
4200 				found = ret;
4201 				goto out_unlock;
4202 			}
4203 			found = 1;
4204 		}
4205 	} while_for_each_ftrace_rec();
4206  out_unlock:
4207 	mutex_unlock(&ftrace_lock);
4208 
4209 	return found;
4210 }
4211 
4212 static int
4213 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
4214 {
4215 	return match_records(hash, buff, len, NULL);
4216 }
4217 
4218 static void ftrace_ops_update_code(struct ftrace_ops *ops,
4219 				   struct ftrace_ops_hash *old_hash)
4220 {
4221 	struct ftrace_ops *op;
4222 
4223 	if (!ftrace_enabled)
4224 		return;
4225 
4226 	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
4227 		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
4228 		return;
4229 	}
4230 
4231 	/*
4232 	 * If this is the shared global_ops filter, then we need to
4233 	 * check if there is another ops that shares it, is enabled.
4234 	 * If so, we still need to run the modify code.
4235 	 */
4236 	if (ops->func_hash != &global_ops.local_hash)
4237 		return;
4238 
4239 	do_for_each_ftrace_op(op, ftrace_ops_list) {
4240 		if (op->func_hash == &global_ops.local_hash &&
4241 		    op->flags & FTRACE_OPS_FL_ENABLED) {
4242 			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
4243 			/* Only need to do this once */
4244 			return;
4245 		}
4246 	} while_for_each_ftrace_op(op);
4247 }
4248 
4249 static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
4250 					   struct ftrace_hash **orig_hash,
4251 					   struct ftrace_hash *hash,
4252 					   int enable)
4253 {
4254 	struct ftrace_ops_hash old_hash_ops;
4255 	struct ftrace_hash *old_hash;
4256 	int ret;
4257 
4258 	old_hash = *orig_hash;
4259 	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
4260 	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
4261 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
4262 	if (!ret) {
4263 		ftrace_ops_update_code(ops, &old_hash_ops);
4264 		free_ftrace_hash_rcu(old_hash);
4265 	}
4266 	return ret;
4267 }
4268 
4269 static bool module_exists(const char *module)
4270 {
4271 	/* All modules have the symbol __this_module */
4272 	static const char this_mod[] = "__this_module";
4273 	char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
4274 	unsigned long val;
4275 	int n;
4276 
4277 	n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
4278 
4279 	if (n > sizeof(modname) - 1)
4280 		return false;
4281 
4282 	val = module_kallsyms_lookup_name(modname);
4283 	return val != 0;
4284 }
4285 
4286 static int cache_mod(struct trace_array *tr,
4287 		     const char *func, char *module, int enable)
4288 {
4289 	struct ftrace_mod_load *ftrace_mod, *n;
4290 	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
4291 	int ret;
4292 
4293 	mutex_lock(&ftrace_lock);
4294 
4295 	/* We do not cache inverse filters */
4296 	if (func[0] == '!') {
4297 		func++;
4298 		ret = -EINVAL;
4299 
4300 		/* Look to remove this hash */
4301 		list_for_each_entry_safe(ftrace_mod, n, head, list) {
4302 			if (strcmp(ftrace_mod->module, module) != 0)
4303 				continue;
4304 
4305 			/* no func matches all */
4306 			if (strcmp(func, "*") == 0 ||
4307 			    (ftrace_mod->func &&
4308 			     strcmp(ftrace_mod->func, func) == 0)) {
4309 				ret = 0;
4310 				free_ftrace_mod(ftrace_mod);
4311 				continue;
4312 			}
4313 		}
4314 		goto out;
4315 	}
4316 
4317 	ret = -EINVAL;
4318 	/* We only care about modules that have not been loaded yet */
4319 	if (module_exists(module))
4320 		goto out;
4321 
4322 	/* Save this string off, and execute it when the module is loaded */
4323 	ret = ftrace_add_mod(tr, func, module, enable);
4324  out:
4325 	mutex_unlock(&ftrace_lock);
4326 
4327 	return ret;
4328 }
4329 
4330 static int
4331 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4332 		 int reset, int enable);
4333 
4334 #ifdef CONFIG_MODULES
4335 static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
4336 			     char *mod, bool enable)
4337 {
4338 	struct ftrace_mod_load *ftrace_mod, *n;
4339 	struct ftrace_hash **orig_hash, *new_hash;
4340 	LIST_HEAD(process_mods);
4341 	char *func;
4342 
4343 	mutex_lock(&ops->func_hash->regex_lock);
4344 
4345 	if (enable)
4346 		orig_hash = &ops->func_hash->filter_hash;
4347 	else
4348 		orig_hash = &ops->func_hash->notrace_hash;
4349 
4350 	new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
4351 					      *orig_hash);
4352 	if (!new_hash)
4353 		goto out; /* warn? */
4354 
4355 	mutex_lock(&ftrace_lock);
4356 
4357 	list_for_each_entry_safe(ftrace_mod, n, head, list) {
4358 
4359 		if (strcmp(ftrace_mod->module, mod) != 0)
4360 			continue;
4361 
4362 		if (ftrace_mod->func)
4363 			func = kstrdup(ftrace_mod->func, GFP_KERNEL);
4364 		else
4365 			func = kstrdup("*", GFP_KERNEL);
4366 
4367 		if (!func) /* warn? */
4368 			continue;
4369 
4370 		list_move(&ftrace_mod->list, &process_mods);
4371 
4372 		/* Use the newly allocated func, as it may be "*" */
4373 		kfree(ftrace_mod->func);
4374 		ftrace_mod->func = func;
4375 	}
4376 
4377 	mutex_unlock(&ftrace_lock);
4378 
4379 	list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
4380 
4381 		func = ftrace_mod->func;
4382 
4383 		/* Grabs ftrace_lock, which is why we have this extra step */
4384 		match_records(new_hash, func, strlen(func), mod);
4385 		free_ftrace_mod(ftrace_mod);
4386 	}
4387 
4388 	if (enable && list_empty(head))
4389 		new_hash->flags &= ~FTRACE_HASH_FL_MOD;
4390 
4391 	mutex_lock(&ftrace_lock);
4392 
4393 	ftrace_hash_move_and_update_ops(ops, orig_hash,
4394 					      new_hash, enable);
4395 	mutex_unlock(&ftrace_lock);
4396 
4397  out:
4398 	mutex_unlock(&ops->func_hash->regex_lock);
4399 
4400 	free_ftrace_hash(new_hash);
4401 }
4402 
4403 static void process_cached_mods(const char *mod_name)
4404 {
4405 	struct trace_array *tr;
4406 	char *mod;
4407 
4408 	mod = kstrdup(mod_name, GFP_KERNEL);
4409 	if (!mod)
4410 		return;
4411 
4412 	mutex_lock(&trace_types_lock);
4413 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
4414 		if (!list_empty(&tr->mod_trace))
4415 			process_mod_list(&tr->mod_trace, tr->ops, mod, true);
4416 		if (!list_empty(&tr->mod_notrace))
4417 			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
4418 	}
4419 	mutex_unlock(&trace_types_lock);
4420 
4421 	kfree(mod);
4422 }
4423 #endif
4424 
4425 /*
4426  * We register the module command as a template to show others how
4427  * to register the a command as well.
4428  */
4429 
4430 static int
4431 ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
4432 		    char *func_orig, char *cmd, char *module, int enable)
4433 {
4434 	char *func;
4435 	int ret;
4436 
4437 	/* match_records() modifies func, and we need the original */
4438 	func = kstrdup(func_orig, GFP_KERNEL);
4439 	if (!func)
4440 		return -ENOMEM;
4441 
4442 	/*
4443 	 * cmd == 'mod' because we only registered this func
4444 	 * for the 'mod' ftrace_func_command.
4445 	 * But if you register one func with multiple commands,
4446 	 * you can tell which command was used by the cmd
4447 	 * parameter.
4448 	 */
4449 	ret = match_records(hash, func, strlen(func), module);
4450 	kfree(func);
4451 
4452 	if (!ret)
4453 		return cache_mod(tr, func_orig, module, enable);
4454 	if (ret < 0)
4455 		return ret;
4456 	return 0;
4457 }
4458 
4459 static struct ftrace_func_command ftrace_mod_cmd = {
4460 	.name			= "mod",
4461 	.func			= ftrace_mod_callback,
4462 };
4463 
4464 static int __init ftrace_mod_cmd_init(void)
4465 {
4466 	return register_ftrace_command(&ftrace_mod_cmd);
4467 }
4468 core_initcall(ftrace_mod_cmd_init);
4469 
4470 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
4471 				      struct ftrace_ops *op, struct ftrace_regs *fregs)
4472 {
4473 	struct ftrace_probe_ops *probe_ops;
4474 	struct ftrace_func_probe *probe;
4475 
4476 	probe = container_of(op, struct ftrace_func_probe, ops);
4477 	probe_ops = probe->probe_ops;
4478 
4479 	/*
4480 	 * Disable preemption for these calls to prevent a RCU grace
4481 	 * period. This syncs the hash iteration and freeing of items
4482 	 * on the hash. rcu_read_lock is too dangerous here.
4483 	 */
4484 	preempt_disable_notrace();
4485 	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
4486 	preempt_enable_notrace();
4487 }
4488 
4489 struct ftrace_func_map {
4490 	struct ftrace_func_entry	entry;
4491 	void				*data;
4492 };
4493 
4494 struct ftrace_func_mapper {
4495 	struct ftrace_hash		hash;
4496 };
4497 
4498 /**
4499  * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
4500  *
4501  * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
4502  */
4503 struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
4504 {
4505 	struct ftrace_hash *hash;
4506 
4507 	/*
4508 	 * The mapper is simply a ftrace_hash, but since the entries
4509 	 * in the hash are not ftrace_func_entry type, we define it
4510 	 * as a separate structure.
4511 	 */
4512 	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4513 	return (struct ftrace_func_mapper *)hash;
4514 }
4515 
4516 /**
4517  * ftrace_func_mapper_find_ip - Find some data mapped to an ip
4518  * @mapper: The mapper that has the ip maps
4519  * @ip: the instruction pointer to find the data for
4520  *
4521  * Returns the data mapped to @ip if found otherwise NULL. The return
4522  * is actually the address of the mapper data pointer. The address is
4523  * returned for use cases where the data is no bigger than a long, and
4524  * the user can use the data pointer as its data instead of having to
4525  * allocate more memory for the reference.
4526  */
4527 void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
4528 				  unsigned long ip)
4529 {
4530 	struct ftrace_func_entry *entry;
4531 	struct ftrace_func_map *map;
4532 
4533 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4534 	if (!entry)
4535 		return NULL;
4536 
4537 	map = (struct ftrace_func_map *)entry;
4538 	return &map->data;
4539 }
4540 
4541 /**
4542  * ftrace_func_mapper_add_ip - Map some data to an ip
4543  * @mapper: The mapper that has the ip maps
4544  * @ip: The instruction pointer address to map @data to
4545  * @data: The data to map to @ip
4546  *
4547  * Returns 0 on success otherwise an error.
4548  */
4549 int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
4550 			      unsigned long ip, void *data)
4551 {
4552 	struct ftrace_func_entry *entry;
4553 	struct ftrace_func_map *map;
4554 
4555 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4556 	if (entry)
4557 		return -EBUSY;
4558 
4559 	map = kmalloc(sizeof(*map), GFP_KERNEL);
4560 	if (!map)
4561 		return -ENOMEM;
4562 
4563 	map->entry.ip = ip;
4564 	map->data = data;
4565 
4566 	__add_hash_entry(&mapper->hash, &map->entry);
4567 
4568 	return 0;
4569 }
4570 
4571 /**
4572  * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
4573  * @mapper: The mapper that has the ip maps
4574  * @ip: The instruction pointer address to remove the data from
4575  *
4576  * Returns the data if it is found, otherwise NULL.
4577  * Note, if the data pointer is used as the data itself, (see
4578  * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
4579  * if the data pointer was set to zero.
4580  */
4581 void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
4582 				   unsigned long ip)
4583 {
4584 	struct ftrace_func_entry *entry;
4585 	struct ftrace_func_map *map;
4586 	void *data;
4587 
4588 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4589 	if (!entry)
4590 		return NULL;
4591 
4592 	map = (struct ftrace_func_map *)entry;
4593 	data = map->data;
4594 
4595 	remove_hash_entry(&mapper->hash, entry);
4596 	kfree(entry);
4597 
4598 	return data;
4599 }
4600 
4601 /**
4602  * free_ftrace_func_mapper - free a mapping of ips and data
4603  * @mapper: The mapper that has the ip maps
4604  * @free_func: A function to be called on each data item.
4605  *
4606  * This is used to free the function mapper. The @free_func is optional
4607  * and can be used if the data needs to be freed as well.
4608  */
4609 void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
4610 			     ftrace_mapper_func free_func)
4611 {
4612 	struct ftrace_func_entry *entry;
4613 	struct ftrace_func_map *map;
4614 	struct hlist_head *hhd;
4615 	int size, i;
4616 
4617 	if (!mapper)
4618 		return;
4619 
4620 	if (free_func && mapper->hash.count) {
4621 		size = 1 << mapper->hash.size_bits;
4622 		for (i = 0; i < size; i++) {
4623 			hhd = &mapper->hash.buckets[i];
4624 			hlist_for_each_entry(entry, hhd, hlist) {
4625 				map = (struct ftrace_func_map *)entry;
4626 				free_func(map);
4627 			}
4628 		}
4629 	}
4630 	free_ftrace_hash(&mapper->hash);
4631 }
4632 
4633 static void release_probe(struct ftrace_func_probe *probe)
4634 {
4635 	struct ftrace_probe_ops *probe_ops;
4636 
4637 	mutex_lock(&ftrace_lock);
4638 
4639 	WARN_ON(probe->ref <= 0);
4640 
4641 	/* Subtract the ref that was used to protect this instance */
4642 	probe->ref--;
4643 
4644 	if (!probe->ref) {
4645 		probe_ops = probe->probe_ops;
4646 		/*
4647 		 * Sending zero as ip tells probe_ops to free
4648 		 * the probe->data itself
4649 		 */
4650 		if (probe_ops->free)
4651 			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
4652 		list_del(&probe->list);
4653 		kfree(probe);
4654 	}
4655 	mutex_unlock(&ftrace_lock);
4656 }
4657 
4658 static void acquire_probe_locked(struct ftrace_func_probe *probe)
4659 {
4660 	/*
4661 	 * Add one ref to keep it from being freed when releasing the
4662 	 * ftrace_lock mutex.
4663 	 */
4664 	probe->ref++;
4665 }
4666 
4667 int
4668 register_ftrace_function_probe(char *glob, struct trace_array *tr,
4669 			       struct ftrace_probe_ops *probe_ops,
4670 			       void *data)
4671 {
4672 	struct ftrace_func_probe *probe = NULL, *iter;
4673 	struct ftrace_func_entry *entry;
4674 	struct ftrace_hash **orig_hash;
4675 	struct ftrace_hash *old_hash;
4676 	struct ftrace_hash *hash;
4677 	int count = 0;
4678 	int size;
4679 	int ret;
4680 	int i;
4681 
4682 	if (WARN_ON(!tr))
4683 		return -EINVAL;
4684 
4685 	/* We do not support '!' for function probes */
4686 	if (WARN_ON(glob[0] == '!'))
4687 		return -EINVAL;
4688 
4689 
4690 	mutex_lock(&ftrace_lock);
4691 	/* Check if the probe_ops is already registered */
4692 	list_for_each_entry(iter, &tr->func_probes, list) {
4693 		if (iter->probe_ops == probe_ops) {
4694 			probe = iter;
4695 			break;
4696 		}
4697 	}
4698 	if (!probe) {
4699 		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
4700 		if (!probe) {
4701 			mutex_unlock(&ftrace_lock);
4702 			return -ENOMEM;
4703 		}
4704 		probe->probe_ops = probe_ops;
4705 		probe->ops.func = function_trace_probe_call;
4706 		probe->tr = tr;
4707 		ftrace_ops_init(&probe->ops);
4708 		list_add(&probe->list, &tr->func_probes);
4709 	}
4710 
4711 	acquire_probe_locked(probe);
4712 
4713 	mutex_unlock(&ftrace_lock);
4714 
4715 	/*
4716 	 * Note, there's a small window here that the func_hash->filter_hash
4717 	 * may be NULL or empty. Need to be careful when reading the loop.
4718 	 */
4719 	mutex_lock(&probe->ops.func_hash->regex_lock);
4720 
4721 	orig_hash = &probe->ops.func_hash->filter_hash;
4722 	old_hash = *orig_hash;
4723 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4724 
4725 	if (!hash) {
4726 		ret = -ENOMEM;
4727 		goto out;
4728 	}
4729 
4730 	ret = ftrace_match_records(hash, glob, strlen(glob));
4731 
4732 	/* Nothing found? */
4733 	if (!ret)
4734 		ret = -EINVAL;
4735 
4736 	if (ret < 0)
4737 		goto out;
4738 
4739 	size = 1 << hash->size_bits;
4740 	for (i = 0; i < size; i++) {
4741 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4742 			if (ftrace_lookup_ip(old_hash, entry->ip))
4743 				continue;
4744 			/*
4745 			 * The caller might want to do something special
4746 			 * for each function we find. We call the callback
4747 			 * to give the caller an opportunity to do so.
4748 			 */
4749 			if (probe_ops->init) {
4750 				ret = probe_ops->init(probe_ops, tr,
4751 						      entry->ip, data,
4752 						      &probe->data);
4753 				if (ret < 0) {
4754 					if (probe_ops->free && count)
4755 						probe_ops->free(probe_ops, tr,
4756 								0, probe->data);
4757 					probe->data = NULL;
4758 					goto out;
4759 				}
4760 			}
4761 			count++;
4762 		}
4763 	}
4764 
4765 	mutex_lock(&ftrace_lock);
4766 
4767 	if (!count) {
4768 		/* Nothing was added? */
4769 		ret = -EINVAL;
4770 		goto out_unlock;
4771 	}
4772 
4773 	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4774 					      hash, 1);
4775 	if (ret < 0)
4776 		goto err_unlock;
4777 
4778 	/* One ref for each new function traced */
4779 	probe->ref += count;
4780 
4781 	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
4782 		ret = ftrace_startup(&probe->ops, 0);
4783 
4784  out_unlock:
4785 	mutex_unlock(&ftrace_lock);
4786 
4787 	if (!ret)
4788 		ret = count;
4789  out:
4790 	mutex_unlock(&probe->ops.func_hash->regex_lock);
4791 	free_ftrace_hash(hash);
4792 
4793 	release_probe(probe);
4794 
4795 	return ret;
4796 
4797  err_unlock:
4798 	if (!probe_ops->free || !count)
4799 		goto out_unlock;
4800 
4801 	/* Failed to do the move, need to call the free functions */
4802 	for (i = 0; i < size; i++) {
4803 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4804 			if (ftrace_lookup_ip(old_hash, entry->ip))
4805 				continue;
4806 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4807 		}
4808 	}
4809 	goto out_unlock;
4810 }
4811 
4812 int
4813 unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
4814 				      struct ftrace_probe_ops *probe_ops)
4815 {
4816 	struct ftrace_func_probe *probe = NULL, *iter;
4817 	struct ftrace_ops_hash old_hash_ops;
4818 	struct ftrace_func_entry *entry;
4819 	struct ftrace_glob func_g;
4820 	struct ftrace_hash **orig_hash;
4821 	struct ftrace_hash *old_hash;
4822 	struct ftrace_hash *hash = NULL;
4823 	struct hlist_node *tmp;
4824 	struct hlist_head hhd;
4825 	char str[KSYM_SYMBOL_LEN];
4826 	int count = 0;
4827 	int i, ret = -ENODEV;
4828 	int size;
4829 
4830 	if (!glob || !strlen(glob) || !strcmp(glob, "*"))
4831 		func_g.search = NULL;
4832 	else {
4833 		int not;
4834 
4835 		func_g.type = filter_parse_regex(glob, strlen(glob),
4836 						 &func_g.search, &not);
4837 		func_g.len = strlen(func_g.search);
4838 
4839 		/* we do not support '!' for function probes */
4840 		if (WARN_ON(not))
4841 			return -EINVAL;
4842 	}
4843 
4844 	mutex_lock(&ftrace_lock);
4845 	/* Check if the probe_ops is already registered */
4846 	list_for_each_entry(iter, &tr->func_probes, list) {
4847 		if (iter->probe_ops == probe_ops) {
4848 			probe = iter;
4849 			break;
4850 		}
4851 	}
4852 	if (!probe)
4853 		goto err_unlock_ftrace;
4854 
4855 	ret = -EINVAL;
4856 	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
4857 		goto err_unlock_ftrace;
4858 
4859 	acquire_probe_locked(probe);
4860 
4861 	mutex_unlock(&ftrace_lock);
4862 
4863 	mutex_lock(&probe->ops.func_hash->regex_lock);
4864 
4865 	orig_hash = &probe->ops.func_hash->filter_hash;
4866 	old_hash = *orig_hash;
4867 
4868 	if (ftrace_hash_empty(old_hash))
4869 		goto out_unlock;
4870 
4871 	old_hash_ops.filter_hash = old_hash;
4872 	/* Probes only have filters */
4873 	old_hash_ops.notrace_hash = NULL;
4874 
4875 	ret = -ENOMEM;
4876 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4877 	if (!hash)
4878 		goto out_unlock;
4879 
4880 	INIT_HLIST_HEAD(&hhd);
4881 
4882 	size = 1 << hash->size_bits;
4883 	for (i = 0; i < size; i++) {
4884 		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
4885 
4886 			if (func_g.search) {
4887 				kallsyms_lookup(entry->ip, NULL, NULL,
4888 						NULL, str);
4889 				if (!ftrace_match(str, &func_g))
4890 					continue;
4891 			}
4892 			count++;
4893 			remove_hash_entry(hash, entry);
4894 			hlist_add_head(&entry->hlist, &hhd);
4895 		}
4896 	}
4897 
4898 	/* Nothing found? */
4899 	if (!count) {
4900 		ret = -EINVAL;
4901 		goto out_unlock;
4902 	}
4903 
4904 	mutex_lock(&ftrace_lock);
4905 
4906 	WARN_ON(probe->ref < count);
4907 
4908 	probe->ref -= count;
4909 
4910 	if (ftrace_hash_empty(hash))
4911 		ftrace_shutdown(&probe->ops, 0);
4912 
4913 	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4914 					      hash, 1);
4915 
4916 	/* still need to update the function call sites */
4917 	if (ftrace_enabled && !ftrace_hash_empty(hash))
4918 		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
4919 				       &old_hash_ops);
4920 	synchronize_rcu();
4921 
4922 	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
4923 		hlist_del(&entry->hlist);
4924 		if (probe_ops->free)
4925 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4926 		kfree(entry);
4927 	}
4928 	mutex_unlock(&ftrace_lock);
4929 
4930  out_unlock:
4931 	mutex_unlock(&probe->ops.func_hash->regex_lock);
4932 	free_ftrace_hash(hash);
4933 
4934 	release_probe(probe);
4935 
4936 	return ret;
4937 
4938  err_unlock_ftrace:
4939 	mutex_unlock(&ftrace_lock);
4940 	return ret;
4941 }
4942 
4943 void clear_ftrace_function_probes(struct trace_array *tr)
4944 {
4945 	struct ftrace_func_probe *probe, *n;
4946 
4947 	list_for_each_entry_safe(probe, n, &tr->func_probes, list)
4948 		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
4949 }
4950 
4951 static LIST_HEAD(ftrace_commands);
4952 static DEFINE_MUTEX(ftrace_cmd_mutex);
4953 
4954 /*
4955  * Currently we only register ftrace commands from __init, so mark this
4956  * __init too.
4957  */
4958 __init int register_ftrace_command(struct ftrace_func_command *cmd)
4959 {
4960 	struct ftrace_func_command *p;
4961 	int ret = 0;
4962 
4963 	mutex_lock(&ftrace_cmd_mutex);
4964 	list_for_each_entry(p, &ftrace_commands, list) {
4965 		if (strcmp(cmd->name, p->name) == 0) {
4966 			ret = -EBUSY;
4967 			goto out_unlock;
4968 		}
4969 	}
4970 	list_add(&cmd->list, &ftrace_commands);
4971  out_unlock:
4972 	mutex_unlock(&ftrace_cmd_mutex);
4973 
4974 	return ret;
4975 }
4976 
4977 /*
4978  * Currently we only unregister ftrace commands from __init, so mark
4979  * this __init too.
4980  */
4981 __init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4982 {
4983 	struct ftrace_func_command *p, *n;
4984 	int ret = -ENODEV;
4985 
4986 	mutex_lock(&ftrace_cmd_mutex);
4987 	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
4988 		if (strcmp(cmd->name, p->name) == 0) {
4989 			ret = 0;
4990 			list_del_init(&p->list);
4991 			goto out_unlock;
4992 		}
4993 	}
4994  out_unlock:
4995 	mutex_unlock(&ftrace_cmd_mutex);
4996 
4997 	return ret;
4998 }
4999 
5000 static int ftrace_process_regex(struct ftrace_iterator *iter,
5001 				char *buff, int len, int enable)
5002 {
5003 	struct ftrace_hash *hash = iter->hash;
5004 	struct trace_array *tr = iter->ops->private;
5005 	char *func, *command, *next = buff;
5006 	struct ftrace_func_command *p;
5007 	int ret = -EINVAL;
5008 
5009 	func = strsep(&next, ":");
5010 
5011 	if (!next) {
5012 		ret = ftrace_match_records(hash, func, len);
5013 		if (!ret)
5014 			ret = -EINVAL;
5015 		if (ret < 0)
5016 			return ret;
5017 		return 0;
5018 	}
5019 
5020 	/* command found */
5021 
5022 	command = strsep(&next, ":");
5023 
5024 	mutex_lock(&ftrace_cmd_mutex);
5025 	list_for_each_entry(p, &ftrace_commands, list) {
5026 		if (strcmp(p->name, command) == 0) {
5027 			ret = p->func(tr, hash, func, command, next, enable);
5028 			goto out_unlock;
5029 		}
5030 	}
5031  out_unlock:
5032 	mutex_unlock(&ftrace_cmd_mutex);
5033 
5034 	return ret;
5035 }
5036 
5037 static ssize_t
5038 ftrace_regex_write(struct file *file, const char __user *ubuf,
5039 		   size_t cnt, loff_t *ppos, int enable)
5040 {
5041 	struct ftrace_iterator *iter;
5042 	struct trace_parser *parser;
5043 	ssize_t ret, read;
5044 
5045 	if (!cnt)
5046 		return 0;
5047 
5048 	if (file->f_mode & FMODE_READ) {
5049 		struct seq_file *m = file->private_data;
5050 		iter = m->private;
5051 	} else
5052 		iter = file->private_data;
5053 
5054 	if (unlikely(ftrace_disabled))
5055 		return -ENODEV;
5056 
5057 	/* iter->hash is a local copy, so we don't need regex_lock */
5058 
5059 	parser = &iter->parser;
5060 	read = trace_get_user(parser, ubuf, cnt, ppos);
5061 
5062 	if (read >= 0 && trace_parser_loaded(parser) &&
5063 	    !trace_parser_cont(parser)) {
5064 		ret = ftrace_process_regex(iter, parser->buffer,
5065 					   parser->idx, enable);
5066 		trace_parser_clear(parser);
5067 		if (ret < 0)
5068 			goto out;
5069 	}
5070 
5071 	ret = read;
5072  out:
5073 	return ret;
5074 }
5075 
5076 ssize_t
5077 ftrace_filter_write(struct file *file, const char __user *ubuf,
5078 		    size_t cnt, loff_t *ppos)
5079 {
5080 	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
5081 }
5082 
5083 ssize_t
5084 ftrace_notrace_write(struct file *file, const char __user *ubuf,
5085 		     size_t cnt, loff_t *ppos)
5086 {
5087 	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
5088 }
5089 
5090 static int
5091 __ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
5092 {
5093 	struct ftrace_func_entry *entry;
5094 
5095 	ip = ftrace_location(ip);
5096 	if (!ip)
5097 		return -EINVAL;
5098 
5099 	if (remove) {
5100 		entry = ftrace_lookup_ip(hash, ip);
5101 		if (!entry)
5102 			return -ENOENT;
5103 		free_hash_entry(hash, entry);
5104 		return 0;
5105 	}
5106 
5107 	return add_hash_entry(hash, ip);
5108 }
5109 
5110 static int
5111 ftrace_match_addr(struct ftrace_hash *hash, unsigned long *ips,
5112 		  unsigned int cnt, int remove)
5113 {
5114 	unsigned int i;
5115 	int err;
5116 
5117 	for (i = 0; i < cnt; i++) {
5118 		err = __ftrace_match_addr(hash, ips[i], remove);
5119 		if (err) {
5120 			/*
5121 			 * This expects the @hash is a temporary hash and if this
5122 			 * fails the caller must free the @hash.
5123 			 */
5124 			return err;
5125 		}
5126 	}
5127 	return 0;
5128 }
5129 
5130 static int
5131 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
5132 		unsigned long *ips, unsigned int cnt,
5133 		int remove, int reset, int enable)
5134 {
5135 	struct ftrace_hash **orig_hash;
5136 	struct ftrace_hash *hash;
5137 	int ret;
5138 
5139 	if (unlikely(ftrace_disabled))
5140 		return -ENODEV;
5141 
5142 	mutex_lock(&ops->func_hash->regex_lock);
5143 
5144 	if (enable)
5145 		orig_hash = &ops->func_hash->filter_hash;
5146 	else
5147 		orig_hash = &ops->func_hash->notrace_hash;
5148 
5149 	if (reset)
5150 		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
5151 	else
5152 		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
5153 
5154 	if (!hash) {
5155 		ret = -ENOMEM;
5156 		goto out_regex_unlock;
5157 	}
5158 
5159 	if (buf && !ftrace_match_records(hash, buf, len)) {
5160 		ret = -EINVAL;
5161 		goto out_regex_unlock;
5162 	}
5163 	if (ips) {
5164 		ret = ftrace_match_addr(hash, ips, cnt, remove);
5165 		if (ret < 0)
5166 			goto out_regex_unlock;
5167 	}
5168 
5169 	mutex_lock(&ftrace_lock);
5170 	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
5171 	mutex_unlock(&ftrace_lock);
5172 
5173  out_regex_unlock:
5174 	mutex_unlock(&ops->func_hash->regex_lock);
5175 
5176 	free_ftrace_hash(hash);
5177 	return ret;
5178 }
5179 
5180 static int
5181 ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt,
5182 		int remove, int reset, int enable)
5183 {
5184 	return ftrace_set_hash(ops, NULL, 0, ips, cnt, remove, reset, enable);
5185 }
5186 
5187 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
5188 
5189 struct ftrace_direct_func {
5190 	struct list_head	next;
5191 	unsigned long		addr;
5192 	int			count;
5193 };
5194 
5195 static LIST_HEAD(ftrace_direct_funcs);
5196 
5197 /**
5198  * ftrace_find_direct_func - test an address if it is a registered direct caller
5199  * @addr: The address of a registered direct caller
5200  *
5201  * This searches to see if a ftrace direct caller has been registered
5202  * at a specific address, and if so, it returns a descriptor for it.
5203  *
5204  * This can be used by architecture code to see if an address is
5205  * a direct caller (trampoline) attached to a fentry/mcount location.
5206  * This is useful for the function_graph tracer, as it may need to
5207  * do adjustments if it traced a location that also has a direct
5208  * trampoline attached to it.
5209  */
5210 struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
5211 {
5212 	struct ftrace_direct_func *entry;
5213 	bool found = false;
5214 
5215 	/* May be called by fgraph trampoline (protected by rcu tasks) */
5216 	list_for_each_entry_rcu(entry, &ftrace_direct_funcs, next) {
5217 		if (entry->addr == addr) {
5218 			found = true;
5219 			break;
5220 		}
5221 	}
5222 	if (found)
5223 		return entry;
5224 
5225 	return NULL;
5226 }
5227 
5228 static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr)
5229 {
5230 	struct ftrace_direct_func *direct;
5231 
5232 	direct = kmalloc(sizeof(*direct), GFP_KERNEL);
5233 	if (!direct)
5234 		return NULL;
5235 	direct->addr = addr;
5236 	direct->count = 0;
5237 	list_add_rcu(&direct->next, &ftrace_direct_funcs);
5238 	ftrace_direct_func_count++;
5239 	return direct;
5240 }
5241 
5242 static int register_ftrace_function_nolock(struct ftrace_ops *ops);
5243 
5244 /**
5245  * register_ftrace_direct - Call a custom trampoline directly
5246  * @ip: The address of the nop at the beginning of a function
5247  * @addr: The address of the trampoline to call at @ip
5248  *
5249  * This is used to connect a direct call from the nop location (@ip)
5250  * at the start of ftrace traced functions. The location that it calls
5251  * (@addr) must be able to handle a direct call, and save the parameters
5252  * of the function being traced, and restore them (or inject new ones
5253  * if needed), before returning.
5254  *
5255  * Returns:
5256  *  0 on success
5257  *  -EBUSY - Another direct function is already attached (there can be only one)
5258  *  -ENODEV - @ip does not point to a ftrace nop location (or not supported)
5259  *  -ENOMEM - There was an allocation failure.
5260  */
5261 int register_ftrace_direct(unsigned long ip, unsigned long addr)
5262 {
5263 	struct ftrace_direct_func *direct;
5264 	struct ftrace_func_entry *entry;
5265 	struct ftrace_hash *free_hash = NULL;
5266 	struct dyn_ftrace *rec;
5267 	int ret = -ENODEV;
5268 
5269 	mutex_lock(&direct_mutex);
5270 
5271 	ip = ftrace_location(ip);
5272 	if (!ip)
5273 		goto out_unlock;
5274 
5275 	/* See if there's a direct function at @ip already */
5276 	ret = -EBUSY;
5277 	if (ftrace_find_rec_direct(ip))
5278 		goto out_unlock;
5279 
5280 	ret = -ENODEV;
5281 	rec = lookup_rec(ip, ip);
5282 	if (!rec)
5283 		goto out_unlock;
5284 
5285 	/*
5286 	 * Check if the rec says it has a direct call but we didn't
5287 	 * find one earlier?
5288 	 */
5289 	if (WARN_ON(rec->flags & FTRACE_FL_DIRECT))
5290 		goto out_unlock;
5291 
5292 	/* Make sure the ip points to the exact record */
5293 	if (ip != rec->ip) {
5294 		ip = rec->ip;
5295 		/* Need to check this ip for a direct. */
5296 		if (ftrace_find_rec_direct(ip))
5297 			goto out_unlock;
5298 	}
5299 
5300 	ret = -ENOMEM;
5301 	direct = ftrace_find_direct_func(addr);
5302 	if (!direct) {
5303 		direct = ftrace_alloc_direct_func(addr);
5304 		if (!direct)
5305 			goto out_unlock;
5306 	}
5307 
5308 	entry = ftrace_add_rec_direct(ip, addr, &free_hash);
5309 	if (!entry)
5310 		goto out_unlock;
5311 
5312 	ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0);
5313 
5314 	if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) {
5315 		ret = register_ftrace_function_nolock(&direct_ops);
5316 		if (ret)
5317 			ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
5318 	}
5319 
5320 	if (ret) {
5321 		remove_hash_entry(direct_functions, entry);
5322 		kfree(entry);
5323 		if (!direct->count) {
5324 			list_del_rcu(&direct->next);
5325 			synchronize_rcu_tasks();
5326 			kfree(direct);
5327 			if (free_hash)
5328 				free_ftrace_hash(free_hash);
5329 			free_hash = NULL;
5330 			ftrace_direct_func_count--;
5331 		}
5332 	} else {
5333 		direct->count++;
5334 	}
5335  out_unlock:
5336 	mutex_unlock(&direct_mutex);
5337 
5338 	if (free_hash) {
5339 		synchronize_rcu_tasks();
5340 		free_ftrace_hash(free_hash);
5341 	}
5342 
5343 	return ret;
5344 }
5345 EXPORT_SYMBOL_GPL(register_ftrace_direct);
5346 
5347 static struct ftrace_func_entry *find_direct_entry(unsigned long *ip,
5348 						   struct dyn_ftrace **recp)
5349 {
5350 	struct ftrace_func_entry *entry;
5351 	struct dyn_ftrace *rec;
5352 
5353 	rec = lookup_rec(*ip, *ip);
5354 	if (!rec)
5355 		return NULL;
5356 
5357 	entry = __ftrace_lookup_ip(direct_functions, rec->ip);
5358 	if (!entry) {
5359 		WARN_ON(rec->flags & FTRACE_FL_DIRECT);
5360 		return NULL;
5361 	}
5362 
5363 	WARN_ON(!(rec->flags & FTRACE_FL_DIRECT));
5364 
5365 	/* Passed in ip just needs to be on the call site */
5366 	*ip = rec->ip;
5367 
5368 	if (recp)
5369 		*recp = rec;
5370 
5371 	return entry;
5372 }
5373 
5374 int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
5375 {
5376 	struct ftrace_direct_func *direct;
5377 	struct ftrace_func_entry *entry;
5378 	struct ftrace_hash *hash;
5379 	int ret = -ENODEV;
5380 
5381 	mutex_lock(&direct_mutex);
5382 
5383 	ip = ftrace_location(ip);
5384 	if (!ip)
5385 		goto out_unlock;
5386 
5387 	entry = find_direct_entry(&ip, NULL);
5388 	if (!entry)
5389 		goto out_unlock;
5390 
5391 	hash = direct_ops.func_hash->filter_hash;
5392 	if (hash->count == 1)
5393 		unregister_ftrace_function(&direct_ops);
5394 
5395 	ret = ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
5396 
5397 	WARN_ON(ret);
5398 
5399 	remove_hash_entry(direct_functions, entry);
5400 
5401 	direct = ftrace_find_direct_func(addr);
5402 	if (!WARN_ON(!direct)) {
5403 		/* This is the good path (see the ! before WARN) */
5404 		direct->count--;
5405 		WARN_ON(direct->count < 0);
5406 		if (!direct->count) {
5407 			list_del_rcu(&direct->next);
5408 			synchronize_rcu_tasks();
5409 			kfree(direct);
5410 			kfree(entry);
5411 			ftrace_direct_func_count--;
5412 		}
5413 	}
5414  out_unlock:
5415 	mutex_unlock(&direct_mutex);
5416 
5417 	return ret;
5418 }
5419 EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
5420 
5421 static struct ftrace_ops stub_ops = {
5422 	.func		= ftrace_stub,
5423 };
5424 
5425 /**
5426  * ftrace_modify_direct_caller - modify ftrace nop directly
5427  * @entry: The ftrace hash entry of the direct helper for @rec
5428  * @rec: The record representing the function site to patch
5429  * @old_addr: The location that the site at @rec->ip currently calls
5430  * @new_addr: The location that the site at @rec->ip should call
5431  *
5432  * An architecture may overwrite this function to optimize the
5433  * changing of the direct callback on an ftrace nop location.
5434  * This is called with the ftrace_lock mutex held, and no other
5435  * ftrace callbacks are on the associated record (@rec). Thus,
5436  * it is safe to modify the ftrace record, where it should be
5437  * currently calling @old_addr directly, to call @new_addr.
5438  *
5439  * This is called with direct_mutex locked.
5440  *
5441  * Safety checks should be made to make sure that the code at
5442  * @rec->ip is currently calling @old_addr. And this must
5443  * also update entry->direct to @new_addr.
5444  */
5445 int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
5446 				       struct dyn_ftrace *rec,
5447 				       unsigned long old_addr,
5448 				       unsigned long new_addr)
5449 {
5450 	unsigned long ip = rec->ip;
5451 	int ret;
5452 
5453 	lockdep_assert_held(&direct_mutex);
5454 
5455 	/*
5456 	 * The ftrace_lock was used to determine if the record
5457 	 * had more than one registered user to it. If it did,
5458 	 * we needed to prevent that from changing to do the quick
5459 	 * switch. But if it did not (only a direct caller was attached)
5460 	 * then this function is called. But this function can deal
5461 	 * with attached callers to the rec that we care about, and
5462 	 * since this function uses standard ftrace calls that take
5463 	 * the ftrace_lock mutex, we need to release it.
5464 	 */
5465 	mutex_unlock(&ftrace_lock);
5466 
5467 	/*
5468 	 * By setting a stub function at the same address, we force
5469 	 * the code to call the iterator and the direct_ops helper.
5470 	 * This means that @ip does not call the direct call, and
5471 	 * we can simply modify it.
5472 	 */
5473 	ret = ftrace_set_filter_ip(&stub_ops, ip, 0, 0);
5474 	if (ret)
5475 		goto out_lock;
5476 
5477 	ret = register_ftrace_function_nolock(&stub_ops);
5478 	if (ret) {
5479 		ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
5480 		goto out_lock;
5481 	}
5482 
5483 	entry->direct = new_addr;
5484 
5485 	/*
5486 	 * By removing the stub, we put back the direct call, calling
5487 	 * the @new_addr.
5488 	 */
5489 	unregister_ftrace_function(&stub_ops);
5490 	ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
5491 
5492  out_lock:
5493 	mutex_lock(&ftrace_lock);
5494 
5495 	return ret;
5496 }
5497 
5498 /**
5499  * modify_ftrace_direct - Modify an existing direct call to call something else
5500  * @ip: The instruction pointer to modify
5501  * @old_addr: The address that the current @ip calls directly
5502  * @new_addr: The address that the @ip should call
5503  *
5504  * This modifies a ftrace direct caller at an instruction pointer without
5505  * having to disable it first. The direct call will switch over to the
5506  * @new_addr without missing anything.
5507  *
5508  * Returns: zero on success. Non zero on error, which includes:
5509  *  -ENODEV : the @ip given has no direct caller attached
5510  *  -EINVAL : the @old_addr does not match the current direct caller
5511  */
5512 int modify_ftrace_direct(unsigned long ip,
5513 			 unsigned long old_addr, unsigned long new_addr)
5514 {
5515 	struct ftrace_direct_func *direct, *new_direct = NULL;
5516 	struct ftrace_func_entry *entry;
5517 	struct dyn_ftrace *rec;
5518 	int ret = -ENODEV;
5519 
5520 	mutex_lock(&direct_mutex);
5521 
5522 	mutex_lock(&ftrace_lock);
5523 
5524 	ip = ftrace_location(ip);
5525 	if (!ip)
5526 		goto out_unlock;
5527 
5528 	entry = find_direct_entry(&ip, &rec);
5529 	if (!entry)
5530 		goto out_unlock;
5531 
5532 	ret = -EINVAL;
5533 	if (entry->direct != old_addr)
5534 		goto out_unlock;
5535 
5536 	direct = ftrace_find_direct_func(old_addr);
5537 	if (WARN_ON(!direct))
5538 		goto out_unlock;
5539 	if (direct->count > 1) {
5540 		ret = -ENOMEM;
5541 		new_direct = ftrace_alloc_direct_func(new_addr);
5542 		if (!new_direct)
5543 			goto out_unlock;
5544 		direct->count--;
5545 		new_direct->count++;
5546 	} else {
5547 		direct->addr = new_addr;
5548 	}
5549 
5550 	/*
5551 	 * If there's no other ftrace callback on the rec->ip location,
5552 	 * then it can be changed directly by the architecture.
5553 	 * If there is another caller, then we just need to change the
5554 	 * direct caller helper to point to @new_addr.
5555 	 */
5556 	if (ftrace_rec_count(rec) == 1) {
5557 		ret = ftrace_modify_direct_caller(entry, rec, old_addr, new_addr);
5558 	} else {
5559 		entry->direct = new_addr;
5560 		ret = 0;
5561 	}
5562 
5563 	if (unlikely(ret && new_direct)) {
5564 		direct->count++;
5565 		list_del_rcu(&new_direct->next);
5566 		synchronize_rcu_tasks();
5567 		kfree(new_direct);
5568 		ftrace_direct_func_count--;
5569 	}
5570 
5571  out_unlock:
5572 	mutex_unlock(&ftrace_lock);
5573 	mutex_unlock(&direct_mutex);
5574 	return ret;
5575 }
5576 EXPORT_SYMBOL_GPL(modify_ftrace_direct);
5577 
5578 #define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS)
5579 
5580 static int check_direct_multi(struct ftrace_ops *ops)
5581 {
5582 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
5583 		return -EINVAL;
5584 	if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS)
5585 		return -EINVAL;
5586 	return 0;
5587 }
5588 
5589 static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr)
5590 {
5591 	struct ftrace_func_entry *entry, *del;
5592 	int size, i;
5593 
5594 	size = 1 << hash->size_bits;
5595 	for (i = 0; i < size; i++) {
5596 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5597 			del = __ftrace_lookup_ip(direct_functions, entry->ip);
5598 			if (del && del->direct == addr) {
5599 				remove_hash_entry(direct_functions, del);
5600 				kfree(del);
5601 			}
5602 		}
5603 	}
5604 }
5605 
5606 /**
5607  * register_ftrace_direct_multi - Call a custom trampoline directly
5608  * for multiple functions registered in @ops
5609  * @ops: The address of the struct ftrace_ops object
5610  * @addr: The address of the trampoline to call at @ops functions
5611  *
5612  * This is used to connect a direct calls to @addr from the nop locations
5613  * of the functions registered in @ops (with by ftrace_set_filter_ip
5614  * function).
5615  *
5616  * The location that it calls (@addr) must be able to handle a direct call,
5617  * and save the parameters of the function being traced, and restore them
5618  * (or inject new ones if needed), before returning.
5619  *
5620  * Returns:
5621  *  0 on success
5622  *  -EINVAL  - The @ops object was already registered with this call or
5623  *             when there are no functions in @ops object.
5624  *  -EBUSY   - Another direct function is already attached (there can be only one)
5625  *  -ENODEV  - @ip does not point to a ftrace nop location (or not supported)
5626  *  -ENOMEM  - There was an allocation failure.
5627  */
5628 int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
5629 {
5630 	struct ftrace_hash *hash, *free_hash = NULL;
5631 	struct ftrace_func_entry *entry, *new;
5632 	int err = -EBUSY, size, i;
5633 
5634 	if (ops->func || ops->trampoline)
5635 		return -EINVAL;
5636 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
5637 		return -EINVAL;
5638 	if (ops->flags & FTRACE_OPS_FL_ENABLED)
5639 		return -EINVAL;
5640 
5641 	hash = ops->func_hash->filter_hash;
5642 	if (ftrace_hash_empty(hash))
5643 		return -EINVAL;
5644 
5645 	mutex_lock(&direct_mutex);
5646 
5647 	/* Make sure requested entries are not already registered.. */
5648 	size = 1 << hash->size_bits;
5649 	for (i = 0; i < size; i++) {
5650 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5651 			if (ftrace_find_rec_direct(entry->ip))
5652 				goto out_unlock;
5653 		}
5654 	}
5655 
5656 	/* ... and insert them to direct_functions hash. */
5657 	err = -ENOMEM;
5658 	for (i = 0; i < size; i++) {
5659 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
5660 			new = ftrace_add_rec_direct(entry->ip, addr, &free_hash);
5661 			if (!new)
5662 				goto out_remove;
5663 			entry->direct = addr;
5664 		}
5665 	}
5666 
5667 	ops->func = call_direct_funcs;
5668 	ops->flags = MULTI_FLAGS;
5669 	ops->trampoline = FTRACE_REGS_ADDR;
5670 
5671 	err = register_ftrace_function_nolock(ops);
5672 
5673  out_remove:
5674 	if (err)
5675 		remove_direct_functions_hash(hash, addr);
5676 
5677  out_unlock:
5678 	mutex_unlock(&direct_mutex);
5679 
5680 	if (free_hash) {
5681 		synchronize_rcu_tasks();
5682 		free_ftrace_hash(free_hash);
5683 	}
5684 	return err;
5685 }
5686 EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
5687 
5688 /**
5689  * unregister_ftrace_direct_multi - Remove calls to custom trampoline
5690  * previously registered by register_ftrace_direct_multi for @ops object.
5691  * @ops: The address of the struct ftrace_ops object
5692  *
5693  * This is used to remove a direct calls to @addr from the nop locations
5694  * of the functions registered in @ops (with by ftrace_set_filter_ip
5695  * function).
5696  *
5697  * Returns:
5698  *  0 on success
5699  *  -EINVAL - The @ops object was not properly registered.
5700  */
5701 int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
5702 {
5703 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
5704 	int err;
5705 
5706 	if (check_direct_multi(ops))
5707 		return -EINVAL;
5708 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5709 		return -EINVAL;
5710 
5711 	mutex_lock(&direct_mutex);
5712 	err = unregister_ftrace_function(ops);
5713 	remove_direct_functions_hash(hash, addr);
5714 	mutex_unlock(&direct_mutex);
5715 
5716 	/* cleanup for possible another register call */
5717 	ops->func = NULL;
5718 	ops->trampoline = 0;
5719 	return err;
5720 }
5721 EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi);
5722 
5723 static int
5724 __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
5725 {
5726 	struct ftrace_hash *hash;
5727 	struct ftrace_func_entry *entry, *iter;
5728 	static struct ftrace_ops tmp_ops = {
5729 		.func		= ftrace_stub,
5730 		.flags		= FTRACE_OPS_FL_STUB,
5731 	};
5732 	int i, size;
5733 	int err;
5734 
5735 	lockdep_assert_held_once(&direct_mutex);
5736 
5737 	/* Enable the tmp_ops to have the same functions as the direct ops */
5738 	ftrace_ops_init(&tmp_ops);
5739 	tmp_ops.func_hash = ops->func_hash;
5740 
5741 	err = register_ftrace_function_nolock(&tmp_ops);
5742 	if (err)
5743 		return err;
5744 
5745 	/*
5746 	 * Now the ftrace_ops_list_func() is called to do the direct callers.
5747 	 * We can safely change the direct functions attached to each entry.
5748 	 */
5749 	mutex_lock(&ftrace_lock);
5750 
5751 	hash = ops->func_hash->filter_hash;
5752 	size = 1 << hash->size_bits;
5753 	for (i = 0; i < size; i++) {
5754 		hlist_for_each_entry(iter, &hash->buckets[i], hlist) {
5755 			entry = __ftrace_lookup_ip(direct_functions, iter->ip);
5756 			if (!entry)
5757 				continue;
5758 			entry->direct = addr;
5759 		}
5760 	}
5761 
5762 	mutex_unlock(&ftrace_lock);
5763 
5764 	/* Removing the tmp_ops will add the updated direct callers to the functions */
5765 	unregister_ftrace_function(&tmp_ops);
5766 
5767 	return err;
5768 }
5769 
5770 /**
5771  * modify_ftrace_direct_multi_nolock - Modify an existing direct 'multi' call
5772  * to call something else
5773  * @ops: The address of the struct ftrace_ops object
5774  * @addr: The address of the new trampoline to call at @ops functions
5775  *
5776  * This is used to unregister currently registered direct caller and
5777  * register new one @addr on functions registered in @ops object.
5778  *
5779  * Note there's window between ftrace_shutdown and ftrace_startup calls
5780  * where there will be no callbacks called.
5781  *
5782  * Caller should already have direct_mutex locked, so we don't lock
5783  * direct_mutex here.
5784  *
5785  * Returns: zero on success. Non zero on error, which includes:
5786  *  -EINVAL - The @ops object was not properly registered.
5787  */
5788 int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr)
5789 {
5790 	if (check_direct_multi(ops))
5791 		return -EINVAL;
5792 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5793 		return -EINVAL;
5794 
5795 	return __modify_ftrace_direct_multi(ops, addr);
5796 }
5797 EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock);
5798 
5799 /**
5800  * modify_ftrace_direct_multi - Modify an existing direct 'multi' call
5801  * to call something else
5802  * @ops: The address of the struct ftrace_ops object
5803  * @addr: The address of the new trampoline to call at @ops functions
5804  *
5805  * This is used to unregister currently registered direct caller and
5806  * register new one @addr on functions registered in @ops object.
5807  *
5808  * Note there's window between ftrace_shutdown and ftrace_startup calls
5809  * where there will be no callbacks called.
5810  *
5811  * Returns: zero on success. Non zero on error, which includes:
5812  *  -EINVAL - The @ops object was not properly registered.
5813  */
5814 int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
5815 {
5816 	int err;
5817 
5818 	if (check_direct_multi(ops))
5819 		return -EINVAL;
5820 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
5821 		return -EINVAL;
5822 
5823 	mutex_lock(&direct_mutex);
5824 	err = __modify_ftrace_direct_multi(ops, addr);
5825 	mutex_unlock(&direct_mutex);
5826 	return err;
5827 }
5828 EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi);
5829 #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
5830 
5831 /**
5832  * ftrace_set_filter_ip - set a function to filter on in ftrace by address
5833  * @ops - the ops to set the filter with
5834  * @ip - the address to add to or remove from the filter.
5835  * @remove - non zero to remove the ip from the filter
5836  * @reset - non zero to reset all filters before applying this filter.
5837  *
5838  * Filters denote which functions should be enabled when tracing is enabled
5839  * If @ip is NULL, it fails to update filter.
5840  */
5841 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
5842 			 int remove, int reset)
5843 {
5844 	ftrace_ops_init(ops);
5845 	return ftrace_set_addr(ops, &ip, 1, remove, reset, 1);
5846 }
5847 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
5848 
5849 /**
5850  * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses
5851  * @ops - the ops to set the filter with
5852  * @ips - the array of addresses to add to or remove from the filter.
5853  * @cnt - the number of addresses in @ips
5854  * @remove - non zero to remove ips from the filter
5855  * @reset - non zero to reset all filters before applying this filter.
5856  *
5857  * Filters denote which functions should be enabled when tracing is enabled
5858  * If @ips array or any ip specified within is NULL , it fails to update filter.
5859  */
5860 int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips,
5861 			  unsigned int cnt, int remove, int reset)
5862 {
5863 	ftrace_ops_init(ops);
5864 	return ftrace_set_addr(ops, ips, cnt, remove, reset, 1);
5865 }
5866 EXPORT_SYMBOL_GPL(ftrace_set_filter_ips);
5867 
5868 /**
5869  * ftrace_ops_set_global_filter - setup ops to use global filters
5870  * @ops - the ops which will use the global filters
5871  *
5872  * ftrace users who need global function trace filtering should call this.
5873  * It can set the global filter only if ops were not initialized before.
5874  */
5875 void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
5876 {
5877 	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
5878 		return;
5879 
5880 	ftrace_ops_init(ops);
5881 	ops->func_hash = &global_ops.local_hash;
5882 }
5883 EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
5884 
5885 static int
5886 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
5887 		 int reset, int enable)
5888 {
5889 	return ftrace_set_hash(ops, buf, len, NULL, 0, 0, reset, enable);
5890 }
5891 
5892 /**
5893  * ftrace_set_filter - set a function to filter on in ftrace
5894  * @ops - the ops to set the filter with
5895  * @buf - the string that holds the function filter text.
5896  * @len - the length of the string.
5897  * @reset - non zero to reset all filters before applying this filter.
5898  *
5899  * Filters denote which functions should be enabled when tracing is enabled.
5900  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
5901  */
5902 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
5903 		       int len, int reset)
5904 {
5905 	ftrace_ops_init(ops);
5906 	return ftrace_set_regex(ops, buf, len, reset, 1);
5907 }
5908 EXPORT_SYMBOL_GPL(ftrace_set_filter);
5909 
5910 /**
5911  * ftrace_set_notrace - set a function to not trace in ftrace
5912  * @ops - the ops to set the notrace filter with
5913  * @buf - the string that holds the function notrace text.
5914  * @len - the length of the string.
5915  * @reset - non zero to reset all filters before applying this filter.
5916  *
5917  * Notrace Filters denote which functions should not be enabled when tracing
5918  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
5919  * for tracing.
5920  */
5921 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
5922 			int len, int reset)
5923 {
5924 	ftrace_ops_init(ops);
5925 	return ftrace_set_regex(ops, buf, len, reset, 0);
5926 }
5927 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
5928 /**
5929  * ftrace_set_global_filter - set a function to filter on with global tracers
5930  * @buf - the string that holds the function filter text.
5931  * @len - the length of the string.
5932  * @reset - non zero to reset all filters before applying this filter.
5933  *
5934  * Filters denote which functions should be enabled when tracing is enabled.
5935  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
5936  */
5937 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
5938 {
5939 	ftrace_set_regex(&global_ops, buf, len, reset, 1);
5940 }
5941 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
5942 
5943 /**
5944  * ftrace_set_global_notrace - set a function to not trace with global tracers
5945  * @buf - the string that holds the function notrace text.
5946  * @len - the length of the string.
5947  * @reset - non zero to reset all filters before applying this filter.
5948  *
5949  * Notrace Filters denote which functions should not be enabled when tracing
5950  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
5951  * for tracing.
5952  */
5953 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
5954 {
5955 	ftrace_set_regex(&global_ops, buf, len, reset, 0);
5956 }
5957 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
5958 
5959 /*
5960  * command line interface to allow users to set filters on boot up.
5961  */
5962 #define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
5963 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
5964 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
5965 
5966 /* Used by function selftest to not test if filter is set */
5967 bool ftrace_filter_param __initdata;
5968 
5969 static int __init set_ftrace_notrace(char *str)
5970 {
5971 	ftrace_filter_param = true;
5972 	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
5973 	return 1;
5974 }
5975 __setup("ftrace_notrace=", set_ftrace_notrace);
5976 
5977 static int __init set_ftrace_filter(char *str)
5978 {
5979 	ftrace_filter_param = true;
5980 	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
5981 	return 1;
5982 }
5983 __setup("ftrace_filter=", set_ftrace_filter);
5984 
5985 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5986 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
5987 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
5988 static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
5989 
5990 static int __init set_graph_function(char *str)
5991 {
5992 	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
5993 	return 1;
5994 }
5995 __setup("ftrace_graph_filter=", set_graph_function);
5996 
5997 static int __init set_graph_notrace_function(char *str)
5998 {
5999 	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
6000 	return 1;
6001 }
6002 __setup("ftrace_graph_notrace=", set_graph_notrace_function);
6003 
6004 static int __init set_graph_max_depth_function(char *str)
6005 {
6006 	if (!str)
6007 		return 0;
6008 	fgraph_max_depth = simple_strtoul(str, NULL, 0);
6009 	return 1;
6010 }
6011 __setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
6012 
6013 static void __init set_ftrace_early_graph(char *buf, int enable)
6014 {
6015 	int ret;
6016 	char *func;
6017 	struct ftrace_hash *hash;
6018 
6019 	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
6020 	if (MEM_FAIL(!hash, "Failed to allocate hash\n"))
6021 		return;
6022 
6023 	while (buf) {
6024 		func = strsep(&buf, ",");
6025 		/* we allow only one expression at a time */
6026 		ret = ftrace_graph_set_hash(hash, func);
6027 		if (ret)
6028 			printk(KERN_DEBUG "ftrace: function %s not "
6029 					  "traceable\n", func);
6030 	}
6031 
6032 	if (enable)
6033 		ftrace_graph_hash = hash;
6034 	else
6035 		ftrace_graph_notrace_hash = hash;
6036 }
6037 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6038 
6039 void __init
6040 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
6041 {
6042 	char *func;
6043 
6044 	ftrace_ops_init(ops);
6045 
6046 	while (buf) {
6047 		func = strsep(&buf, ",");
6048 		ftrace_set_regex(ops, func, strlen(func), 0, enable);
6049 	}
6050 }
6051 
6052 static void __init set_ftrace_early_filters(void)
6053 {
6054 	if (ftrace_filter_buf[0])
6055 		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
6056 	if (ftrace_notrace_buf[0])
6057 		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
6058 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
6059 	if (ftrace_graph_buf[0])
6060 		set_ftrace_early_graph(ftrace_graph_buf, 1);
6061 	if (ftrace_graph_notrace_buf[0])
6062 		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
6063 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6064 }
6065 
6066 int ftrace_regex_release(struct inode *inode, struct file *file)
6067 {
6068 	struct seq_file *m = (struct seq_file *)file->private_data;
6069 	struct ftrace_iterator *iter;
6070 	struct ftrace_hash **orig_hash;
6071 	struct trace_parser *parser;
6072 	int filter_hash;
6073 
6074 	if (file->f_mode & FMODE_READ) {
6075 		iter = m->private;
6076 		seq_release(inode, file);
6077 	} else
6078 		iter = file->private_data;
6079 
6080 	parser = &iter->parser;
6081 	if (trace_parser_loaded(parser)) {
6082 		int enable = !(iter->flags & FTRACE_ITER_NOTRACE);
6083 
6084 		ftrace_process_regex(iter, parser->buffer,
6085 				     parser->idx, enable);
6086 	}
6087 
6088 	trace_parser_put(parser);
6089 
6090 	mutex_lock(&iter->ops->func_hash->regex_lock);
6091 
6092 	if (file->f_mode & FMODE_WRITE) {
6093 		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
6094 
6095 		if (filter_hash) {
6096 			orig_hash = &iter->ops->func_hash->filter_hash;
6097 			if (iter->tr) {
6098 				if (list_empty(&iter->tr->mod_trace))
6099 					iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
6100 				else
6101 					iter->hash->flags |= FTRACE_HASH_FL_MOD;
6102 			}
6103 		} else
6104 			orig_hash = &iter->ops->func_hash->notrace_hash;
6105 
6106 		mutex_lock(&ftrace_lock);
6107 		ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
6108 						      iter->hash, filter_hash);
6109 		mutex_unlock(&ftrace_lock);
6110 	} else {
6111 		/* For read only, the hash is the ops hash */
6112 		iter->hash = NULL;
6113 	}
6114 
6115 	mutex_unlock(&iter->ops->func_hash->regex_lock);
6116 	free_ftrace_hash(iter->hash);
6117 	if (iter->tr)
6118 		trace_array_put(iter->tr);
6119 	kfree(iter);
6120 
6121 	return 0;
6122 }
6123 
6124 static const struct file_operations ftrace_avail_fops = {
6125 	.open = ftrace_avail_open,
6126 	.read = seq_read,
6127 	.llseek = seq_lseek,
6128 	.release = seq_release_private,
6129 };
6130 
6131 static const struct file_operations ftrace_enabled_fops = {
6132 	.open = ftrace_enabled_open,
6133 	.read = seq_read,
6134 	.llseek = seq_lseek,
6135 	.release = seq_release_private,
6136 };
6137 
6138 static const struct file_operations ftrace_filter_fops = {
6139 	.open = ftrace_filter_open,
6140 	.read = seq_read,
6141 	.write = ftrace_filter_write,
6142 	.llseek = tracing_lseek,
6143 	.release = ftrace_regex_release,
6144 };
6145 
6146 static const struct file_operations ftrace_notrace_fops = {
6147 	.open = ftrace_notrace_open,
6148 	.read = seq_read,
6149 	.write = ftrace_notrace_write,
6150 	.llseek = tracing_lseek,
6151 	.release = ftrace_regex_release,
6152 };
6153 
6154 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
6155 
6156 static DEFINE_MUTEX(graph_lock);
6157 
6158 struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH;
6159 struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH;
6160 
6161 enum graph_filter_type {
6162 	GRAPH_FILTER_NOTRACE	= 0,
6163 	GRAPH_FILTER_FUNCTION,
6164 };
6165 
6166 #define FTRACE_GRAPH_EMPTY	((void *)1)
6167 
6168 struct ftrace_graph_data {
6169 	struct ftrace_hash		*hash;
6170 	struct ftrace_func_entry	*entry;
6171 	int				idx;   /* for hash table iteration */
6172 	enum graph_filter_type		type;
6173 	struct ftrace_hash		*new_hash;
6174 	const struct seq_operations	*seq_ops;
6175 	struct trace_parser		parser;
6176 };
6177 
6178 static void *
6179 __g_next(struct seq_file *m, loff_t *pos)
6180 {
6181 	struct ftrace_graph_data *fgd = m->private;
6182 	struct ftrace_func_entry *entry = fgd->entry;
6183 	struct hlist_head *head;
6184 	int i, idx = fgd->idx;
6185 
6186 	if (*pos >= fgd->hash->count)
6187 		return NULL;
6188 
6189 	if (entry) {
6190 		hlist_for_each_entry_continue(entry, hlist) {
6191 			fgd->entry = entry;
6192 			return entry;
6193 		}
6194 
6195 		idx++;
6196 	}
6197 
6198 	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
6199 		head = &fgd->hash->buckets[i];
6200 		hlist_for_each_entry(entry, head, hlist) {
6201 			fgd->entry = entry;
6202 			fgd->idx = i;
6203 			return entry;
6204 		}
6205 	}
6206 	return NULL;
6207 }
6208 
6209 static void *
6210 g_next(struct seq_file *m, void *v, loff_t *pos)
6211 {
6212 	(*pos)++;
6213 	return __g_next(m, pos);
6214 }
6215 
6216 static void *g_start(struct seq_file *m, loff_t *pos)
6217 {
6218 	struct ftrace_graph_data *fgd = m->private;
6219 
6220 	mutex_lock(&graph_lock);
6221 
6222 	if (fgd->type == GRAPH_FILTER_FUNCTION)
6223 		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
6224 					lockdep_is_held(&graph_lock));
6225 	else
6226 		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
6227 					lockdep_is_held(&graph_lock));
6228 
6229 	/* Nothing, tell g_show to print all functions are enabled */
6230 	if (ftrace_hash_empty(fgd->hash) && !*pos)
6231 		return FTRACE_GRAPH_EMPTY;
6232 
6233 	fgd->idx = 0;
6234 	fgd->entry = NULL;
6235 	return __g_next(m, pos);
6236 }
6237 
6238 static void g_stop(struct seq_file *m, void *p)
6239 {
6240 	mutex_unlock(&graph_lock);
6241 }
6242 
6243 static int g_show(struct seq_file *m, void *v)
6244 {
6245 	struct ftrace_func_entry *entry = v;
6246 
6247 	if (!entry)
6248 		return 0;
6249 
6250 	if (entry == FTRACE_GRAPH_EMPTY) {
6251 		struct ftrace_graph_data *fgd = m->private;
6252 
6253 		if (fgd->type == GRAPH_FILTER_FUNCTION)
6254 			seq_puts(m, "#### all functions enabled ####\n");
6255 		else
6256 			seq_puts(m, "#### no functions disabled ####\n");
6257 		return 0;
6258 	}
6259 
6260 	seq_printf(m, "%ps\n", (void *)entry->ip);
6261 
6262 	return 0;
6263 }
6264 
6265 static const struct seq_operations ftrace_graph_seq_ops = {
6266 	.start = g_start,
6267 	.next = g_next,
6268 	.stop = g_stop,
6269 	.show = g_show,
6270 };
6271 
6272 static int
6273 __ftrace_graph_open(struct inode *inode, struct file *file,
6274 		    struct ftrace_graph_data *fgd)
6275 {
6276 	int ret;
6277 	struct ftrace_hash *new_hash = NULL;
6278 
6279 	ret = security_locked_down(LOCKDOWN_TRACEFS);
6280 	if (ret)
6281 		return ret;
6282 
6283 	if (file->f_mode & FMODE_WRITE) {
6284 		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
6285 
6286 		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
6287 			return -ENOMEM;
6288 
6289 		if (file->f_flags & O_TRUNC)
6290 			new_hash = alloc_ftrace_hash(size_bits);
6291 		else
6292 			new_hash = alloc_and_copy_ftrace_hash(size_bits,
6293 							      fgd->hash);
6294 		if (!new_hash) {
6295 			ret = -ENOMEM;
6296 			goto out;
6297 		}
6298 	}
6299 
6300 	if (file->f_mode & FMODE_READ) {
6301 		ret = seq_open(file, &ftrace_graph_seq_ops);
6302 		if (!ret) {
6303 			struct seq_file *m = file->private_data;
6304 			m->private = fgd;
6305 		} else {
6306 			/* Failed */
6307 			free_ftrace_hash(new_hash);
6308 			new_hash = NULL;
6309 		}
6310 	} else
6311 		file->private_data = fgd;
6312 
6313 out:
6314 	if (ret < 0 && file->f_mode & FMODE_WRITE)
6315 		trace_parser_put(&fgd->parser);
6316 
6317 	fgd->new_hash = new_hash;
6318 
6319 	/*
6320 	 * All uses of fgd->hash must be taken with the graph_lock
6321 	 * held. The graph_lock is going to be released, so force
6322 	 * fgd->hash to be reinitialized when it is taken again.
6323 	 */
6324 	fgd->hash = NULL;
6325 
6326 	return ret;
6327 }
6328 
6329 static int
6330 ftrace_graph_open(struct inode *inode, struct file *file)
6331 {
6332 	struct ftrace_graph_data *fgd;
6333 	int ret;
6334 
6335 	if (unlikely(ftrace_disabled))
6336 		return -ENODEV;
6337 
6338 	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
6339 	if (fgd == NULL)
6340 		return -ENOMEM;
6341 
6342 	mutex_lock(&graph_lock);
6343 
6344 	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
6345 					lockdep_is_held(&graph_lock));
6346 	fgd->type = GRAPH_FILTER_FUNCTION;
6347 	fgd->seq_ops = &ftrace_graph_seq_ops;
6348 
6349 	ret = __ftrace_graph_open(inode, file, fgd);
6350 	if (ret < 0)
6351 		kfree(fgd);
6352 
6353 	mutex_unlock(&graph_lock);
6354 	return ret;
6355 }
6356 
6357 static int
6358 ftrace_graph_notrace_open(struct inode *inode, struct file *file)
6359 {
6360 	struct ftrace_graph_data *fgd;
6361 	int ret;
6362 
6363 	if (unlikely(ftrace_disabled))
6364 		return -ENODEV;
6365 
6366 	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
6367 	if (fgd == NULL)
6368 		return -ENOMEM;
6369 
6370 	mutex_lock(&graph_lock);
6371 
6372 	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
6373 					lockdep_is_held(&graph_lock));
6374 	fgd->type = GRAPH_FILTER_NOTRACE;
6375 	fgd->seq_ops = &ftrace_graph_seq_ops;
6376 
6377 	ret = __ftrace_graph_open(inode, file, fgd);
6378 	if (ret < 0)
6379 		kfree(fgd);
6380 
6381 	mutex_unlock(&graph_lock);
6382 	return ret;
6383 }
6384 
6385 static int
6386 ftrace_graph_release(struct inode *inode, struct file *file)
6387 {
6388 	struct ftrace_graph_data *fgd;
6389 	struct ftrace_hash *old_hash, *new_hash;
6390 	struct trace_parser *parser;
6391 	int ret = 0;
6392 
6393 	if (file->f_mode & FMODE_READ) {
6394 		struct seq_file *m = file->private_data;
6395 
6396 		fgd = m->private;
6397 		seq_release(inode, file);
6398 	} else {
6399 		fgd = file->private_data;
6400 	}
6401 
6402 
6403 	if (file->f_mode & FMODE_WRITE) {
6404 
6405 		parser = &fgd->parser;
6406 
6407 		if (trace_parser_loaded((parser))) {
6408 			ret = ftrace_graph_set_hash(fgd->new_hash,
6409 						    parser->buffer);
6410 		}
6411 
6412 		trace_parser_put(parser);
6413 
6414 		new_hash = __ftrace_hash_move(fgd->new_hash);
6415 		if (!new_hash) {
6416 			ret = -ENOMEM;
6417 			goto out;
6418 		}
6419 
6420 		mutex_lock(&graph_lock);
6421 
6422 		if (fgd->type == GRAPH_FILTER_FUNCTION) {
6423 			old_hash = rcu_dereference_protected(ftrace_graph_hash,
6424 					lockdep_is_held(&graph_lock));
6425 			rcu_assign_pointer(ftrace_graph_hash, new_hash);
6426 		} else {
6427 			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
6428 					lockdep_is_held(&graph_lock));
6429 			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
6430 		}
6431 
6432 		mutex_unlock(&graph_lock);
6433 
6434 		/*
6435 		 * We need to do a hard force of sched synchronization.
6436 		 * This is because we use preempt_disable() to do RCU, but
6437 		 * the function tracers can be called where RCU is not watching
6438 		 * (like before user_exit()). We can not rely on the RCU
6439 		 * infrastructure to do the synchronization, thus we must do it
6440 		 * ourselves.
6441 		 */
6442 		if (old_hash != EMPTY_HASH)
6443 			synchronize_rcu_tasks_rude();
6444 
6445 		free_ftrace_hash(old_hash);
6446 	}
6447 
6448  out:
6449 	free_ftrace_hash(fgd->new_hash);
6450 	kfree(fgd);
6451 
6452 	return ret;
6453 }
6454 
6455 static int
6456 ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
6457 {
6458 	struct ftrace_glob func_g;
6459 	struct dyn_ftrace *rec;
6460 	struct ftrace_page *pg;
6461 	struct ftrace_func_entry *entry;
6462 	int fail = 1;
6463 	int not;
6464 
6465 	/* decode regex */
6466 	func_g.type = filter_parse_regex(buffer, strlen(buffer),
6467 					 &func_g.search, &not);
6468 
6469 	func_g.len = strlen(func_g.search);
6470 
6471 	mutex_lock(&ftrace_lock);
6472 
6473 	if (unlikely(ftrace_disabled)) {
6474 		mutex_unlock(&ftrace_lock);
6475 		return -ENODEV;
6476 	}
6477 
6478 	do_for_each_ftrace_rec(pg, rec) {
6479 
6480 		if (rec->flags & FTRACE_FL_DISABLED)
6481 			continue;
6482 
6483 		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
6484 			entry = ftrace_lookup_ip(hash, rec->ip);
6485 
6486 			if (!not) {
6487 				fail = 0;
6488 
6489 				if (entry)
6490 					continue;
6491 				if (add_hash_entry(hash, rec->ip) < 0)
6492 					goto out;
6493 			} else {
6494 				if (entry) {
6495 					free_hash_entry(hash, entry);
6496 					fail = 0;
6497 				}
6498 			}
6499 		}
6500 	} while_for_each_ftrace_rec();
6501 out:
6502 	mutex_unlock(&ftrace_lock);
6503 
6504 	if (fail)
6505 		return -EINVAL;
6506 
6507 	return 0;
6508 }
6509 
6510 static ssize_t
6511 ftrace_graph_write(struct file *file, const char __user *ubuf,
6512 		   size_t cnt, loff_t *ppos)
6513 {
6514 	ssize_t read, ret = 0;
6515 	struct ftrace_graph_data *fgd = file->private_data;
6516 	struct trace_parser *parser;
6517 
6518 	if (!cnt)
6519 		return 0;
6520 
6521 	/* Read mode uses seq functions */
6522 	if (file->f_mode & FMODE_READ) {
6523 		struct seq_file *m = file->private_data;
6524 		fgd = m->private;
6525 	}
6526 
6527 	parser = &fgd->parser;
6528 
6529 	read = trace_get_user(parser, ubuf, cnt, ppos);
6530 
6531 	if (read >= 0 && trace_parser_loaded(parser) &&
6532 	    !trace_parser_cont(parser)) {
6533 
6534 		ret = ftrace_graph_set_hash(fgd->new_hash,
6535 					    parser->buffer);
6536 		trace_parser_clear(parser);
6537 	}
6538 
6539 	if (!ret)
6540 		ret = read;
6541 
6542 	return ret;
6543 }
6544 
6545 static const struct file_operations ftrace_graph_fops = {
6546 	.open		= ftrace_graph_open,
6547 	.read		= seq_read,
6548 	.write		= ftrace_graph_write,
6549 	.llseek		= tracing_lseek,
6550 	.release	= ftrace_graph_release,
6551 };
6552 
6553 static const struct file_operations ftrace_graph_notrace_fops = {
6554 	.open		= ftrace_graph_notrace_open,
6555 	.read		= seq_read,
6556 	.write		= ftrace_graph_write,
6557 	.llseek		= tracing_lseek,
6558 	.release	= ftrace_graph_release,
6559 };
6560 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6561 
6562 void ftrace_create_filter_files(struct ftrace_ops *ops,
6563 				struct dentry *parent)
6564 {
6565 
6566 	trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent,
6567 			  ops, &ftrace_filter_fops);
6568 
6569 	trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent,
6570 			  ops, &ftrace_notrace_fops);
6571 }
6572 
6573 /*
6574  * The name "destroy_filter_files" is really a misnomer. Although
6575  * in the future, it may actually delete the files, but this is
6576  * really intended to make sure the ops passed in are disabled
6577  * and that when this function returns, the caller is free to
6578  * free the ops.
6579  *
6580  * The "destroy" name is only to match the "create" name that this
6581  * should be paired with.
6582  */
6583 void ftrace_destroy_filter_files(struct ftrace_ops *ops)
6584 {
6585 	mutex_lock(&ftrace_lock);
6586 	if (ops->flags & FTRACE_OPS_FL_ENABLED)
6587 		ftrace_shutdown(ops, 0);
6588 	ops->flags |= FTRACE_OPS_FL_DELETED;
6589 	ftrace_free_filter(ops);
6590 	mutex_unlock(&ftrace_lock);
6591 }
6592 
6593 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
6594 {
6595 
6596 	trace_create_file("available_filter_functions", TRACE_MODE_READ,
6597 			d_tracer, NULL, &ftrace_avail_fops);
6598 
6599 	trace_create_file("enabled_functions", TRACE_MODE_READ,
6600 			d_tracer, NULL, &ftrace_enabled_fops);
6601 
6602 	ftrace_create_filter_files(&global_ops, d_tracer);
6603 
6604 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
6605 	trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer,
6606 				    NULL,
6607 				    &ftrace_graph_fops);
6608 	trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer,
6609 				    NULL,
6610 				    &ftrace_graph_notrace_fops);
6611 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
6612 
6613 	return 0;
6614 }
6615 
6616 static int ftrace_cmp_ips(const void *a, const void *b)
6617 {
6618 	const unsigned long *ipa = a;
6619 	const unsigned long *ipb = b;
6620 
6621 	if (*ipa > *ipb)
6622 		return 1;
6623 	if (*ipa < *ipb)
6624 		return -1;
6625 	return 0;
6626 }
6627 
6628 #ifdef CONFIG_FTRACE_SORT_STARTUP_TEST
6629 static void test_is_sorted(unsigned long *start, unsigned long count)
6630 {
6631 	int i;
6632 
6633 	for (i = 1; i < count; i++) {
6634 		if (WARN(start[i - 1] > start[i],
6635 			 "[%d] %pS at %lx is not sorted with %pS at %lx\n", i,
6636 			 (void *)start[i - 1], start[i - 1],
6637 			 (void *)start[i], start[i]))
6638 			break;
6639 	}
6640 	if (i == count)
6641 		pr_info("ftrace section at %px sorted properly\n", start);
6642 }
6643 #else
6644 static void test_is_sorted(unsigned long *start, unsigned long count)
6645 {
6646 }
6647 #endif
6648 
6649 static int ftrace_process_locs(struct module *mod,
6650 			       unsigned long *start,
6651 			       unsigned long *end)
6652 {
6653 	struct ftrace_page *start_pg;
6654 	struct ftrace_page *pg;
6655 	struct dyn_ftrace *rec;
6656 	unsigned long count;
6657 	unsigned long *p;
6658 	unsigned long addr;
6659 	unsigned long flags = 0; /* Shut up gcc */
6660 	int ret = -ENOMEM;
6661 
6662 	count = end - start;
6663 
6664 	if (!count)
6665 		return 0;
6666 
6667 	/*
6668 	 * Sorting mcount in vmlinux at build time depend on
6669 	 * CONFIG_BUILDTIME_MCOUNT_SORT, while mcount loc in
6670 	 * modules can not be sorted at build time.
6671 	 */
6672 	if (!IS_ENABLED(CONFIG_BUILDTIME_MCOUNT_SORT) || mod) {
6673 		sort(start, count, sizeof(*start),
6674 		     ftrace_cmp_ips, NULL);
6675 	} else {
6676 		test_is_sorted(start, count);
6677 	}
6678 
6679 	start_pg = ftrace_allocate_pages(count);
6680 	if (!start_pg)
6681 		return -ENOMEM;
6682 
6683 	mutex_lock(&ftrace_lock);
6684 
6685 	/*
6686 	 * Core and each module needs their own pages, as
6687 	 * modules will free them when they are removed.
6688 	 * Force a new page to be allocated for modules.
6689 	 */
6690 	if (!mod) {
6691 		WARN_ON(ftrace_pages || ftrace_pages_start);
6692 		/* First initialization */
6693 		ftrace_pages = ftrace_pages_start = start_pg;
6694 	} else {
6695 		if (!ftrace_pages)
6696 			goto out;
6697 
6698 		if (WARN_ON(ftrace_pages->next)) {
6699 			/* Hmm, we have free pages? */
6700 			while (ftrace_pages->next)
6701 				ftrace_pages = ftrace_pages->next;
6702 		}
6703 
6704 		ftrace_pages->next = start_pg;
6705 	}
6706 
6707 	p = start;
6708 	pg = start_pg;
6709 	while (p < end) {
6710 		unsigned long end_offset;
6711 		addr = ftrace_call_adjust(*p++);
6712 		/*
6713 		 * Some architecture linkers will pad between
6714 		 * the different mcount_loc sections of different
6715 		 * object files to satisfy alignments.
6716 		 * Skip any NULL pointers.
6717 		 */
6718 		if (!addr)
6719 			continue;
6720 
6721 		end_offset = (pg->index+1) * sizeof(pg->records[0]);
6722 		if (end_offset > PAGE_SIZE << pg->order) {
6723 			/* We should have allocated enough */
6724 			if (WARN_ON(!pg->next))
6725 				break;
6726 			pg = pg->next;
6727 		}
6728 
6729 		rec = &pg->records[pg->index++];
6730 		rec->ip = addr;
6731 	}
6732 
6733 	/* We should have used all pages */
6734 	WARN_ON(pg->next);
6735 
6736 	/* Assign the last page to ftrace_pages */
6737 	ftrace_pages = pg;
6738 
6739 	/*
6740 	 * We only need to disable interrupts on start up
6741 	 * because we are modifying code that an interrupt
6742 	 * may execute, and the modification is not atomic.
6743 	 * But for modules, nothing runs the code we modify
6744 	 * until we are finished with it, and there's no
6745 	 * reason to cause large interrupt latencies while we do it.
6746 	 */
6747 	if (!mod)
6748 		local_irq_save(flags);
6749 	ftrace_update_code(mod, start_pg);
6750 	if (!mod)
6751 		local_irq_restore(flags);
6752 	ret = 0;
6753  out:
6754 	mutex_unlock(&ftrace_lock);
6755 
6756 	return ret;
6757 }
6758 
6759 struct ftrace_mod_func {
6760 	struct list_head	list;
6761 	char			*name;
6762 	unsigned long		ip;
6763 	unsigned int		size;
6764 };
6765 
6766 struct ftrace_mod_map {
6767 	struct rcu_head		rcu;
6768 	struct list_head	list;
6769 	struct module		*mod;
6770 	unsigned long		start_addr;
6771 	unsigned long		end_addr;
6772 	struct list_head	funcs;
6773 	unsigned int		num_funcs;
6774 };
6775 
6776 static int ftrace_get_trampoline_kallsym(unsigned int symnum,
6777 					 unsigned long *value, char *type,
6778 					 char *name, char *module_name,
6779 					 int *exported)
6780 {
6781 	struct ftrace_ops *op;
6782 
6783 	list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) {
6784 		if (!op->trampoline || symnum--)
6785 			continue;
6786 		*value = op->trampoline;
6787 		*type = 't';
6788 		strlcpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN);
6789 		strlcpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN);
6790 		*exported = 0;
6791 		return 0;
6792 	}
6793 
6794 	return -ERANGE;
6795 }
6796 
6797 #if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES)
6798 /*
6799  * Check if the current ops references the given ip.
6800  *
6801  * If the ops traces all functions, then it was already accounted for.
6802  * If the ops does not trace the current record function, skip it.
6803  * If the ops ignores the function via notrace filter, skip it.
6804  */
6805 static bool
6806 ops_references_ip(struct ftrace_ops *ops, unsigned long ip)
6807 {
6808 	/* If ops isn't enabled, ignore it */
6809 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
6810 		return false;
6811 
6812 	/* If ops traces all then it includes this function */
6813 	if (ops_traces_mod(ops))
6814 		return true;
6815 
6816 	/* The function must be in the filter */
6817 	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
6818 	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip))
6819 		return false;
6820 
6821 	/* If in notrace hash, we ignore it too */
6822 	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip))
6823 		return false;
6824 
6825 	return true;
6826 }
6827 #endif
6828 
6829 #ifdef CONFIG_MODULES
6830 
6831 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
6832 
6833 static LIST_HEAD(ftrace_mod_maps);
6834 
6835 static int referenced_filters(struct dyn_ftrace *rec)
6836 {
6837 	struct ftrace_ops *ops;
6838 	int cnt = 0;
6839 
6840 	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
6841 		if (ops_references_ip(ops, rec->ip)) {
6842 			if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT))
6843 				continue;
6844 			if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY))
6845 				continue;
6846 			cnt++;
6847 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
6848 				rec->flags |= FTRACE_FL_REGS;
6849 			if (cnt == 1 && ops->trampoline)
6850 				rec->flags |= FTRACE_FL_TRAMP;
6851 			else
6852 				rec->flags &= ~FTRACE_FL_TRAMP;
6853 		}
6854 	}
6855 
6856 	return cnt;
6857 }
6858 
6859 static void
6860 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
6861 {
6862 	struct ftrace_func_entry *entry;
6863 	struct dyn_ftrace *rec;
6864 	int i;
6865 
6866 	if (ftrace_hash_empty(hash))
6867 		return;
6868 
6869 	for (i = 0; i < pg->index; i++) {
6870 		rec = &pg->records[i];
6871 		entry = __ftrace_lookup_ip(hash, rec->ip);
6872 		/*
6873 		 * Do not allow this rec to match again.
6874 		 * Yeah, it may waste some memory, but will be removed
6875 		 * if/when the hash is modified again.
6876 		 */
6877 		if (entry)
6878 			entry->ip = 0;
6879 	}
6880 }
6881 
6882 /* Clear any records from hashes */
6883 static void clear_mod_from_hashes(struct ftrace_page *pg)
6884 {
6885 	struct trace_array *tr;
6886 
6887 	mutex_lock(&trace_types_lock);
6888 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6889 		if (!tr->ops || !tr->ops->func_hash)
6890 			continue;
6891 		mutex_lock(&tr->ops->func_hash->regex_lock);
6892 		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
6893 		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
6894 		mutex_unlock(&tr->ops->func_hash->regex_lock);
6895 	}
6896 	mutex_unlock(&trace_types_lock);
6897 }
6898 
6899 static void ftrace_free_mod_map(struct rcu_head *rcu)
6900 {
6901 	struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
6902 	struct ftrace_mod_func *mod_func;
6903 	struct ftrace_mod_func *n;
6904 
6905 	/* All the contents of mod_map are now not visible to readers */
6906 	list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
6907 		kfree(mod_func->name);
6908 		list_del(&mod_func->list);
6909 		kfree(mod_func);
6910 	}
6911 
6912 	kfree(mod_map);
6913 }
6914 
6915 void ftrace_release_mod(struct module *mod)
6916 {
6917 	struct ftrace_mod_map *mod_map;
6918 	struct ftrace_mod_map *n;
6919 	struct dyn_ftrace *rec;
6920 	struct ftrace_page **last_pg;
6921 	struct ftrace_page *tmp_page = NULL;
6922 	struct ftrace_page *pg;
6923 
6924 	mutex_lock(&ftrace_lock);
6925 
6926 	if (ftrace_disabled)
6927 		goto out_unlock;
6928 
6929 	list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
6930 		if (mod_map->mod == mod) {
6931 			list_del_rcu(&mod_map->list);
6932 			call_rcu(&mod_map->rcu, ftrace_free_mod_map);
6933 			break;
6934 		}
6935 	}
6936 
6937 	/*
6938 	 * Each module has its own ftrace_pages, remove
6939 	 * them from the list.
6940 	 */
6941 	last_pg = &ftrace_pages_start;
6942 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
6943 		rec = &pg->records[0];
6944 		if (within_module_core(rec->ip, mod) ||
6945 		    within_module_init(rec->ip, mod)) {
6946 			/*
6947 			 * As core pages are first, the first
6948 			 * page should never be a module page.
6949 			 */
6950 			if (WARN_ON(pg == ftrace_pages_start))
6951 				goto out_unlock;
6952 
6953 			/* Check if we are deleting the last page */
6954 			if (pg == ftrace_pages)
6955 				ftrace_pages = next_to_ftrace_page(last_pg);
6956 
6957 			ftrace_update_tot_cnt -= pg->index;
6958 			*last_pg = pg->next;
6959 
6960 			pg->next = tmp_page;
6961 			tmp_page = pg;
6962 		} else
6963 			last_pg = &pg->next;
6964 	}
6965  out_unlock:
6966 	mutex_unlock(&ftrace_lock);
6967 
6968 	for (pg = tmp_page; pg; pg = tmp_page) {
6969 
6970 		/* Needs to be called outside of ftrace_lock */
6971 		clear_mod_from_hashes(pg);
6972 
6973 		if (pg->records) {
6974 			free_pages((unsigned long)pg->records, pg->order);
6975 			ftrace_number_of_pages -= 1 << pg->order;
6976 		}
6977 		tmp_page = pg->next;
6978 		kfree(pg);
6979 		ftrace_number_of_groups--;
6980 	}
6981 }
6982 
6983 void ftrace_module_enable(struct module *mod)
6984 {
6985 	struct dyn_ftrace *rec;
6986 	struct ftrace_page *pg;
6987 
6988 	mutex_lock(&ftrace_lock);
6989 
6990 	if (ftrace_disabled)
6991 		goto out_unlock;
6992 
6993 	/*
6994 	 * If the tracing is enabled, go ahead and enable the record.
6995 	 *
6996 	 * The reason not to enable the record immediately is the
6997 	 * inherent check of ftrace_make_nop/ftrace_make_call for
6998 	 * correct previous instructions.  Making first the NOP
6999 	 * conversion puts the module to the correct state, thus
7000 	 * passing the ftrace_make_call check.
7001 	 *
7002 	 * We also delay this to after the module code already set the
7003 	 * text to read-only, as we now need to set it back to read-write
7004 	 * so that we can modify the text.
7005 	 */
7006 	if (ftrace_start_up)
7007 		ftrace_arch_code_modify_prepare();
7008 
7009 	do_for_each_ftrace_rec(pg, rec) {
7010 		int cnt;
7011 		/*
7012 		 * do_for_each_ftrace_rec() is a double loop.
7013 		 * module text shares the pg. If a record is
7014 		 * not part of this module, then skip this pg,
7015 		 * which the "break" will do.
7016 		 */
7017 		if (!within_module_core(rec->ip, mod) &&
7018 		    !within_module_init(rec->ip, mod))
7019 			break;
7020 
7021 		/* Weak functions should still be ignored */
7022 		if (!test_for_valid_rec(rec)) {
7023 			/* Clear all other flags. Should not be enabled anyway */
7024 			rec->flags = FTRACE_FL_DISABLED;
7025 			continue;
7026 		}
7027 
7028 		cnt = 0;
7029 
7030 		/*
7031 		 * When adding a module, we need to check if tracers are
7032 		 * currently enabled and if they are, and can trace this record,
7033 		 * we need to enable the module functions as well as update the
7034 		 * reference counts for those function records.
7035 		 */
7036 		if (ftrace_start_up)
7037 			cnt += referenced_filters(rec);
7038 
7039 		rec->flags &= ~FTRACE_FL_DISABLED;
7040 		rec->flags += cnt;
7041 
7042 		if (ftrace_start_up && cnt) {
7043 			int failed = __ftrace_replace_code(rec, 1);
7044 			if (failed) {
7045 				ftrace_bug(failed, rec);
7046 				goto out_loop;
7047 			}
7048 		}
7049 
7050 	} while_for_each_ftrace_rec();
7051 
7052  out_loop:
7053 	if (ftrace_start_up)
7054 		ftrace_arch_code_modify_post_process();
7055 
7056  out_unlock:
7057 	mutex_unlock(&ftrace_lock);
7058 
7059 	process_cached_mods(mod->name);
7060 }
7061 
7062 void ftrace_module_init(struct module *mod)
7063 {
7064 	int ret;
7065 
7066 	if (ftrace_disabled || !mod->num_ftrace_callsites)
7067 		return;
7068 
7069 	ret = ftrace_process_locs(mod, mod->ftrace_callsites,
7070 				  mod->ftrace_callsites + mod->num_ftrace_callsites);
7071 	if (ret)
7072 		pr_warn("ftrace: failed to allocate entries for module '%s' functions\n",
7073 			mod->name);
7074 }
7075 
7076 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
7077 				struct dyn_ftrace *rec)
7078 {
7079 	struct ftrace_mod_func *mod_func;
7080 	unsigned long symsize;
7081 	unsigned long offset;
7082 	char str[KSYM_SYMBOL_LEN];
7083 	char *modname;
7084 	const char *ret;
7085 
7086 	ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
7087 	if (!ret)
7088 		return;
7089 
7090 	mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
7091 	if (!mod_func)
7092 		return;
7093 
7094 	mod_func->name = kstrdup(str, GFP_KERNEL);
7095 	if (!mod_func->name) {
7096 		kfree(mod_func);
7097 		return;
7098 	}
7099 
7100 	mod_func->ip = rec->ip - offset;
7101 	mod_func->size = symsize;
7102 
7103 	mod_map->num_funcs++;
7104 
7105 	list_add_rcu(&mod_func->list, &mod_map->funcs);
7106 }
7107 
7108 static struct ftrace_mod_map *
7109 allocate_ftrace_mod_map(struct module *mod,
7110 			unsigned long start, unsigned long end)
7111 {
7112 	struct ftrace_mod_map *mod_map;
7113 
7114 	mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
7115 	if (!mod_map)
7116 		return NULL;
7117 
7118 	mod_map->mod = mod;
7119 	mod_map->start_addr = start;
7120 	mod_map->end_addr = end;
7121 	mod_map->num_funcs = 0;
7122 
7123 	INIT_LIST_HEAD_RCU(&mod_map->funcs);
7124 
7125 	list_add_rcu(&mod_map->list, &ftrace_mod_maps);
7126 
7127 	return mod_map;
7128 }
7129 
7130 static const char *
7131 ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
7132 			   unsigned long addr, unsigned long *size,
7133 			   unsigned long *off, char *sym)
7134 {
7135 	struct ftrace_mod_func *found_func =  NULL;
7136 	struct ftrace_mod_func *mod_func;
7137 
7138 	list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
7139 		if (addr >= mod_func->ip &&
7140 		    addr < mod_func->ip + mod_func->size) {
7141 			found_func = mod_func;
7142 			break;
7143 		}
7144 	}
7145 
7146 	if (found_func) {
7147 		if (size)
7148 			*size = found_func->size;
7149 		if (off)
7150 			*off = addr - found_func->ip;
7151 		if (sym)
7152 			strlcpy(sym, found_func->name, KSYM_NAME_LEN);
7153 
7154 		return found_func->name;
7155 	}
7156 
7157 	return NULL;
7158 }
7159 
7160 const char *
7161 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
7162 		   unsigned long *off, char **modname, char *sym)
7163 {
7164 	struct ftrace_mod_map *mod_map;
7165 	const char *ret = NULL;
7166 
7167 	/* mod_map is freed via call_rcu() */
7168 	preempt_disable();
7169 	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
7170 		ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
7171 		if (ret) {
7172 			if (modname)
7173 				*modname = mod_map->mod->name;
7174 			break;
7175 		}
7176 	}
7177 	preempt_enable();
7178 
7179 	return ret;
7180 }
7181 
7182 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
7183 			   char *type, char *name,
7184 			   char *module_name, int *exported)
7185 {
7186 	struct ftrace_mod_map *mod_map;
7187 	struct ftrace_mod_func *mod_func;
7188 	int ret;
7189 
7190 	preempt_disable();
7191 	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
7192 
7193 		if (symnum >= mod_map->num_funcs) {
7194 			symnum -= mod_map->num_funcs;
7195 			continue;
7196 		}
7197 
7198 		list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
7199 			if (symnum > 1) {
7200 				symnum--;
7201 				continue;
7202 			}
7203 
7204 			*value = mod_func->ip;
7205 			*type = 'T';
7206 			strlcpy(name, mod_func->name, KSYM_NAME_LEN);
7207 			strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
7208 			*exported = 1;
7209 			preempt_enable();
7210 			return 0;
7211 		}
7212 		WARN_ON(1);
7213 		break;
7214 	}
7215 	ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
7216 					    module_name, exported);
7217 	preempt_enable();
7218 	return ret;
7219 }
7220 
7221 #else
7222 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
7223 				struct dyn_ftrace *rec) { }
7224 static inline struct ftrace_mod_map *
7225 allocate_ftrace_mod_map(struct module *mod,
7226 			unsigned long start, unsigned long end)
7227 {
7228 	return NULL;
7229 }
7230 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
7231 			   char *type, char *name, char *module_name,
7232 			   int *exported)
7233 {
7234 	int ret;
7235 
7236 	preempt_disable();
7237 	ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
7238 					    module_name, exported);
7239 	preempt_enable();
7240 	return ret;
7241 }
7242 #endif /* CONFIG_MODULES */
7243 
7244 struct ftrace_init_func {
7245 	struct list_head list;
7246 	unsigned long ip;
7247 };
7248 
7249 /* Clear any init ips from hashes */
7250 static void
7251 clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
7252 {
7253 	struct ftrace_func_entry *entry;
7254 
7255 	entry = ftrace_lookup_ip(hash, func->ip);
7256 	/*
7257 	 * Do not allow this rec to match again.
7258 	 * Yeah, it may waste some memory, but will be removed
7259 	 * if/when the hash is modified again.
7260 	 */
7261 	if (entry)
7262 		entry->ip = 0;
7263 }
7264 
7265 static void
7266 clear_func_from_hashes(struct ftrace_init_func *func)
7267 {
7268 	struct trace_array *tr;
7269 
7270 	mutex_lock(&trace_types_lock);
7271 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7272 		if (!tr->ops || !tr->ops->func_hash)
7273 			continue;
7274 		mutex_lock(&tr->ops->func_hash->regex_lock);
7275 		clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
7276 		clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
7277 		mutex_unlock(&tr->ops->func_hash->regex_lock);
7278 	}
7279 	mutex_unlock(&trace_types_lock);
7280 }
7281 
7282 static void add_to_clear_hash_list(struct list_head *clear_list,
7283 				   struct dyn_ftrace *rec)
7284 {
7285 	struct ftrace_init_func *func;
7286 
7287 	func = kmalloc(sizeof(*func), GFP_KERNEL);
7288 	if (!func) {
7289 		MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n");
7290 		return;
7291 	}
7292 
7293 	func->ip = rec->ip;
7294 	list_add(&func->list, clear_list);
7295 }
7296 
7297 void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
7298 {
7299 	unsigned long start = (unsigned long)(start_ptr);
7300 	unsigned long end = (unsigned long)(end_ptr);
7301 	struct ftrace_page **last_pg = &ftrace_pages_start;
7302 	struct ftrace_page *pg;
7303 	struct dyn_ftrace *rec;
7304 	struct dyn_ftrace key;
7305 	struct ftrace_mod_map *mod_map = NULL;
7306 	struct ftrace_init_func *func, *func_next;
7307 	struct list_head clear_hash;
7308 
7309 	INIT_LIST_HEAD(&clear_hash);
7310 
7311 	key.ip = start;
7312 	key.flags = end;	/* overload flags, as it is unsigned long */
7313 
7314 	mutex_lock(&ftrace_lock);
7315 
7316 	/*
7317 	 * If we are freeing module init memory, then check if
7318 	 * any tracer is active. If so, we need to save a mapping of
7319 	 * the module functions being freed with the address.
7320 	 */
7321 	if (mod && ftrace_ops_list != &ftrace_list_end)
7322 		mod_map = allocate_ftrace_mod_map(mod, start, end);
7323 
7324 	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
7325 		if (end < pg->records[0].ip ||
7326 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
7327 			continue;
7328  again:
7329 		rec = bsearch(&key, pg->records, pg->index,
7330 			      sizeof(struct dyn_ftrace),
7331 			      ftrace_cmp_recs);
7332 		if (!rec)
7333 			continue;
7334 
7335 		/* rec will be cleared from hashes after ftrace_lock unlock */
7336 		add_to_clear_hash_list(&clear_hash, rec);
7337 
7338 		if (mod_map)
7339 			save_ftrace_mod_rec(mod_map, rec);
7340 
7341 		pg->index--;
7342 		ftrace_update_tot_cnt--;
7343 		if (!pg->index) {
7344 			*last_pg = pg->next;
7345 			if (pg->records) {
7346 				free_pages((unsigned long)pg->records, pg->order);
7347 				ftrace_number_of_pages -= 1 << pg->order;
7348 			}
7349 			ftrace_number_of_groups--;
7350 			kfree(pg);
7351 			pg = container_of(last_pg, struct ftrace_page, next);
7352 			if (!(*last_pg))
7353 				ftrace_pages = pg;
7354 			continue;
7355 		}
7356 		memmove(rec, rec + 1,
7357 			(pg->index - (rec - pg->records)) * sizeof(*rec));
7358 		/* More than one function may be in this block */
7359 		goto again;
7360 	}
7361 	mutex_unlock(&ftrace_lock);
7362 
7363 	list_for_each_entry_safe(func, func_next, &clear_hash, list) {
7364 		clear_func_from_hashes(func);
7365 		kfree(func);
7366 	}
7367 }
7368 
7369 void __init ftrace_free_init_mem(void)
7370 {
7371 	void *start = (void *)(&__init_begin);
7372 	void *end = (void *)(&__init_end);
7373 
7374 	ftrace_boot_snapshot();
7375 
7376 	ftrace_free_mem(NULL, start, end);
7377 }
7378 
7379 int __init __weak ftrace_dyn_arch_init(void)
7380 {
7381 	return 0;
7382 }
7383 
7384 void __init ftrace_init(void)
7385 {
7386 	extern unsigned long __start_mcount_loc[];
7387 	extern unsigned long __stop_mcount_loc[];
7388 	unsigned long count, flags;
7389 	int ret;
7390 
7391 	local_irq_save(flags);
7392 	ret = ftrace_dyn_arch_init();
7393 	local_irq_restore(flags);
7394 	if (ret)
7395 		goto failed;
7396 
7397 	count = __stop_mcount_loc - __start_mcount_loc;
7398 	if (!count) {
7399 		pr_info("ftrace: No functions to be traced?\n");
7400 		goto failed;
7401 	}
7402 
7403 	pr_info("ftrace: allocating %ld entries in %ld pages\n",
7404 		count, count / ENTRIES_PER_PAGE + 1);
7405 
7406 	ret = ftrace_process_locs(NULL,
7407 				  __start_mcount_loc,
7408 				  __stop_mcount_loc);
7409 	if (ret) {
7410 		pr_warn("ftrace: failed to allocate entries for functions\n");
7411 		goto failed;
7412 	}
7413 
7414 	pr_info("ftrace: allocated %ld pages with %ld groups\n",
7415 		ftrace_number_of_pages, ftrace_number_of_groups);
7416 
7417 	last_ftrace_enabled = ftrace_enabled = 1;
7418 
7419 	set_ftrace_early_filters();
7420 
7421 	return;
7422  failed:
7423 	ftrace_disabled = 1;
7424 }
7425 
7426 /* Do nothing if arch does not support this */
7427 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
7428 {
7429 }
7430 
7431 static void ftrace_update_trampoline(struct ftrace_ops *ops)
7432 {
7433 	unsigned long trampoline = ops->trampoline;
7434 
7435 	arch_ftrace_update_trampoline(ops);
7436 	if (ops->trampoline && ops->trampoline != trampoline &&
7437 	    (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) {
7438 		/* Add to kallsyms before the perf events */
7439 		ftrace_add_trampoline_to_kallsyms(ops);
7440 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
7441 				   ops->trampoline, ops->trampoline_size, false,
7442 				   FTRACE_TRAMPOLINE_SYM);
7443 		/*
7444 		 * Record the perf text poke event after the ksymbol register
7445 		 * event.
7446 		 */
7447 		perf_event_text_poke((void *)ops->trampoline, NULL, 0,
7448 				     (void *)ops->trampoline,
7449 				     ops->trampoline_size);
7450 	}
7451 }
7452 
7453 void ftrace_init_trace_array(struct trace_array *tr)
7454 {
7455 	INIT_LIST_HEAD(&tr->func_probes);
7456 	INIT_LIST_HEAD(&tr->mod_trace);
7457 	INIT_LIST_HEAD(&tr->mod_notrace);
7458 }
7459 #else
7460 
7461 struct ftrace_ops global_ops = {
7462 	.func			= ftrace_stub,
7463 	.flags			= FTRACE_OPS_FL_INITIALIZED |
7464 				  FTRACE_OPS_FL_PID,
7465 };
7466 
7467 static int __init ftrace_nodyn_init(void)
7468 {
7469 	ftrace_enabled = 1;
7470 	return 0;
7471 }
7472 core_initcall(ftrace_nodyn_init);
7473 
7474 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
7475 static inline void ftrace_startup_all(int command) { }
7476 
7477 static void ftrace_update_trampoline(struct ftrace_ops *ops)
7478 {
7479 }
7480 
7481 #endif /* CONFIG_DYNAMIC_FTRACE */
7482 
7483 __init void ftrace_init_global_array_ops(struct trace_array *tr)
7484 {
7485 	tr->ops = &global_ops;
7486 	tr->ops->private = tr;
7487 	ftrace_init_trace_array(tr);
7488 }
7489 
7490 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
7491 {
7492 	/* If we filter on pids, update to use the pid function */
7493 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
7494 		if (WARN_ON(tr->ops->func != ftrace_stub))
7495 			printk("ftrace ops had %pS for function\n",
7496 			       tr->ops->func);
7497 	}
7498 	tr->ops->func = func;
7499 	tr->ops->private = tr;
7500 }
7501 
7502 void ftrace_reset_array_ops(struct trace_array *tr)
7503 {
7504 	tr->ops->func = ftrace_stub;
7505 }
7506 
7507 static nokprobe_inline void
7508 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
7509 		       struct ftrace_ops *ignored, struct ftrace_regs *fregs)
7510 {
7511 	struct pt_regs *regs = ftrace_get_regs(fregs);
7512 	struct ftrace_ops *op;
7513 	int bit;
7514 
7515 	/*
7516 	 * The ftrace_test_and_set_recursion() will disable preemption,
7517 	 * which is required since some of the ops may be dynamically
7518 	 * allocated, they must be freed after a synchronize_rcu().
7519 	 */
7520 	bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
7521 	if (bit < 0)
7522 		return;
7523 
7524 	do_for_each_ftrace_op(op, ftrace_ops_list) {
7525 		/* Stub functions don't need to be called nor tested */
7526 		if (op->flags & FTRACE_OPS_FL_STUB)
7527 			continue;
7528 		/*
7529 		 * Check the following for each ops before calling their func:
7530 		 *  if RCU flag is set, then rcu_is_watching() must be true
7531 		 *  if PER_CPU is set, then ftrace_function_local_disable()
7532 		 *                          must be false
7533 		 *  Otherwise test if the ip matches the ops filter
7534 		 *
7535 		 * If any of the above fails then the op->func() is not executed.
7536 		 */
7537 		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
7538 		    ftrace_ops_test(op, ip, regs)) {
7539 			if (FTRACE_WARN_ON(!op->func)) {
7540 				pr_warn("op=%p %pS\n", op, op);
7541 				goto out;
7542 			}
7543 			op->func(ip, parent_ip, op, fregs);
7544 		}
7545 	} while_for_each_ftrace_op(op);
7546 out:
7547 	trace_clear_recursion(bit);
7548 }
7549 
7550 /*
7551  * Some archs only support passing ip and parent_ip. Even though
7552  * the list function ignores the op parameter, we do not want any
7553  * C side effects, where a function is called without the caller
7554  * sending a third parameter.
7555  * Archs are to support both the regs and ftrace_ops at the same time.
7556  * If they support ftrace_ops, it is assumed they support regs.
7557  * If call backs want to use regs, they must either check for regs
7558  * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
7559  * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
7560  * An architecture can pass partial regs with ftrace_ops and still
7561  * set the ARCH_SUPPORTS_FTRACE_OPS.
7562  *
7563  * In vmlinux.lds.h, ftrace_ops_list_func() is defined to be
7564  * arch_ftrace_ops_list_func.
7565  */
7566 #if ARCH_SUPPORTS_FTRACE_OPS
7567 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
7568 			       struct ftrace_ops *op, struct ftrace_regs *fregs)
7569 {
7570 	__ftrace_ops_list_func(ip, parent_ip, NULL, fregs);
7571 }
7572 #else
7573 void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
7574 {
7575 	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
7576 }
7577 #endif
7578 NOKPROBE_SYMBOL(arch_ftrace_ops_list_func);
7579 
7580 /*
7581  * If there's only one function registered but it does not support
7582  * recursion, needs RCU protection and/or requires per cpu handling, then
7583  * this function will be called by the mcount trampoline.
7584  */
7585 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
7586 				   struct ftrace_ops *op, struct ftrace_regs *fregs)
7587 {
7588 	int bit;
7589 
7590 	bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
7591 	if (bit < 0)
7592 		return;
7593 
7594 	if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
7595 		op->func(ip, parent_ip, op, fregs);
7596 
7597 	trace_clear_recursion(bit);
7598 }
7599 NOKPROBE_SYMBOL(ftrace_ops_assist_func);
7600 
7601 /**
7602  * ftrace_ops_get_func - get the function a trampoline should call
7603  * @ops: the ops to get the function for
7604  *
7605  * Normally the mcount trampoline will call the ops->func, but there
7606  * are times that it should not. For example, if the ops does not
7607  * have its own recursion protection, then it should call the
7608  * ftrace_ops_assist_func() instead.
7609  *
7610  * Returns the function that the trampoline should call for @ops.
7611  */
7612 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
7613 {
7614 	/*
7615 	 * If the function does not handle recursion or needs to be RCU safe,
7616 	 * then we need to call the assist handler.
7617 	 */
7618 	if (ops->flags & (FTRACE_OPS_FL_RECURSION |
7619 			  FTRACE_OPS_FL_RCU))
7620 		return ftrace_ops_assist_func;
7621 
7622 	return ops->func;
7623 }
7624 
7625 static void
7626 ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
7627 				     struct task_struct *prev,
7628 				     struct task_struct *next,
7629 				     unsigned int prev_state)
7630 {
7631 	struct trace_array *tr = data;
7632 	struct trace_pid_list *pid_list;
7633 	struct trace_pid_list *no_pid_list;
7634 
7635 	pid_list = rcu_dereference_sched(tr->function_pids);
7636 	no_pid_list = rcu_dereference_sched(tr->function_no_pids);
7637 
7638 	if (trace_ignore_this_task(pid_list, no_pid_list, next))
7639 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7640 			       FTRACE_PID_IGNORE);
7641 	else
7642 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7643 			       next->pid);
7644 }
7645 
7646 static void
7647 ftrace_pid_follow_sched_process_fork(void *data,
7648 				     struct task_struct *self,
7649 				     struct task_struct *task)
7650 {
7651 	struct trace_pid_list *pid_list;
7652 	struct trace_array *tr = data;
7653 
7654 	pid_list = rcu_dereference_sched(tr->function_pids);
7655 	trace_filter_add_remove_task(pid_list, self, task);
7656 
7657 	pid_list = rcu_dereference_sched(tr->function_no_pids);
7658 	trace_filter_add_remove_task(pid_list, self, task);
7659 }
7660 
7661 static void
7662 ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
7663 {
7664 	struct trace_pid_list *pid_list;
7665 	struct trace_array *tr = data;
7666 
7667 	pid_list = rcu_dereference_sched(tr->function_pids);
7668 	trace_filter_add_remove_task(pid_list, NULL, task);
7669 
7670 	pid_list = rcu_dereference_sched(tr->function_no_pids);
7671 	trace_filter_add_remove_task(pid_list, NULL, task);
7672 }
7673 
7674 void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
7675 {
7676 	if (enable) {
7677 		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
7678 						  tr);
7679 		register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
7680 						  tr);
7681 	} else {
7682 		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
7683 						    tr);
7684 		unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
7685 						    tr);
7686 	}
7687 }
7688 
7689 static void clear_ftrace_pids(struct trace_array *tr, int type)
7690 {
7691 	struct trace_pid_list *pid_list;
7692 	struct trace_pid_list *no_pid_list;
7693 	int cpu;
7694 
7695 	pid_list = rcu_dereference_protected(tr->function_pids,
7696 					     lockdep_is_held(&ftrace_lock));
7697 	no_pid_list = rcu_dereference_protected(tr->function_no_pids,
7698 						lockdep_is_held(&ftrace_lock));
7699 
7700 	/* Make sure there's something to do */
7701 	if (!pid_type_enabled(type, pid_list, no_pid_list))
7702 		return;
7703 
7704 	/* See if the pids still need to be checked after this */
7705 	if (!still_need_pid_events(type, pid_list, no_pid_list)) {
7706 		unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
7707 		for_each_possible_cpu(cpu)
7708 			per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE;
7709 	}
7710 
7711 	if (type & TRACE_PIDS)
7712 		rcu_assign_pointer(tr->function_pids, NULL);
7713 
7714 	if (type & TRACE_NO_PIDS)
7715 		rcu_assign_pointer(tr->function_no_pids, NULL);
7716 
7717 	/* Wait till all users are no longer using pid filtering */
7718 	synchronize_rcu();
7719 
7720 	if ((type & TRACE_PIDS) && pid_list)
7721 		trace_pid_list_free(pid_list);
7722 
7723 	if ((type & TRACE_NO_PIDS) && no_pid_list)
7724 		trace_pid_list_free(no_pid_list);
7725 }
7726 
7727 void ftrace_clear_pids(struct trace_array *tr)
7728 {
7729 	mutex_lock(&ftrace_lock);
7730 
7731 	clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
7732 
7733 	mutex_unlock(&ftrace_lock);
7734 }
7735 
7736 static void ftrace_pid_reset(struct trace_array *tr, int type)
7737 {
7738 	mutex_lock(&ftrace_lock);
7739 	clear_ftrace_pids(tr, type);
7740 
7741 	ftrace_update_pid_func();
7742 	ftrace_startup_all(0);
7743 
7744 	mutex_unlock(&ftrace_lock);
7745 }
7746 
7747 /* Greater than any max PID */
7748 #define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
7749 
7750 static void *fpid_start(struct seq_file *m, loff_t *pos)
7751 	__acquires(RCU)
7752 {
7753 	struct trace_pid_list *pid_list;
7754 	struct trace_array *tr = m->private;
7755 
7756 	mutex_lock(&ftrace_lock);
7757 	rcu_read_lock_sched();
7758 
7759 	pid_list = rcu_dereference_sched(tr->function_pids);
7760 
7761 	if (!pid_list)
7762 		return !(*pos) ? FTRACE_NO_PIDS : NULL;
7763 
7764 	return trace_pid_start(pid_list, pos);
7765 }
7766 
7767 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
7768 {
7769 	struct trace_array *tr = m->private;
7770 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
7771 
7772 	if (v == FTRACE_NO_PIDS) {
7773 		(*pos)++;
7774 		return NULL;
7775 	}
7776 	return trace_pid_next(pid_list, v, pos);
7777 }
7778 
7779 static void fpid_stop(struct seq_file *m, void *p)
7780 	__releases(RCU)
7781 {
7782 	rcu_read_unlock_sched();
7783 	mutex_unlock(&ftrace_lock);
7784 }
7785 
7786 static int fpid_show(struct seq_file *m, void *v)
7787 {
7788 	if (v == FTRACE_NO_PIDS) {
7789 		seq_puts(m, "no pid\n");
7790 		return 0;
7791 	}
7792 
7793 	return trace_pid_show(m, v);
7794 }
7795 
7796 static const struct seq_operations ftrace_pid_sops = {
7797 	.start = fpid_start,
7798 	.next = fpid_next,
7799 	.stop = fpid_stop,
7800 	.show = fpid_show,
7801 };
7802 
7803 static void *fnpid_start(struct seq_file *m, loff_t *pos)
7804 	__acquires(RCU)
7805 {
7806 	struct trace_pid_list *pid_list;
7807 	struct trace_array *tr = m->private;
7808 
7809 	mutex_lock(&ftrace_lock);
7810 	rcu_read_lock_sched();
7811 
7812 	pid_list = rcu_dereference_sched(tr->function_no_pids);
7813 
7814 	if (!pid_list)
7815 		return !(*pos) ? FTRACE_NO_PIDS : NULL;
7816 
7817 	return trace_pid_start(pid_list, pos);
7818 }
7819 
7820 static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos)
7821 {
7822 	struct trace_array *tr = m->private;
7823 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids);
7824 
7825 	if (v == FTRACE_NO_PIDS) {
7826 		(*pos)++;
7827 		return NULL;
7828 	}
7829 	return trace_pid_next(pid_list, v, pos);
7830 }
7831 
7832 static const struct seq_operations ftrace_no_pid_sops = {
7833 	.start = fnpid_start,
7834 	.next = fnpid_next,
7835 	.stop = fpid_stop,
7836 	.show = fpid_show,
7837 };
7838 
7839 static int pid_open(struct inode *inode, struct file *file, int type)
7840 {
7841 	const struct seq_operations *seq_ops;
7842 	struct trace_array *tr = inode->i_private;
7843 	struct seq_file *m;
7844 	int ret = 0;
7845 
7846 	ret = tracing_check_open_get_tr(tr);
7847 	if (ret)
7848 		return ret;
7849 
7850 	if ((file->f_mode & FMODE_WRITE) &&
7851 	    (file->f_flags & O_TRUNC))
7852 		ftrace_pid_reset(tr, type);
7853 
7854 	switch (type) {
7855 	case TRACE_PIDS:
7856 		seq_ops = &ftrace_pid_sops;
7857 		break;
7858 	case TRACE_NO_PIDS:
7859 		seq_ops = &ftrace_no_pid_sops;
7860 		break;
7861 	default:
7862 		trace_array_put(tr);
7863 		WARN_ON_ONCE(1);
7864 		return -EINVAL;
7865 	}
7866 
7867 	ret = seq_open(file, seq_ops);
7868 	if (ret < 0) {
7869 		trace_array_put(tr);
7870 	} else {
7871 		m = file->private_data;
7872 		/* copy tr over to seq ops */
7873 		m->private = tr;
7874 	}
7875 
7876 	return ret;
7877 }
7878 
7879 static int
7880 ftrace_pid_open(struct inode *inode, struct file *file)
7881 {
7882 	return pid_open(inode, file, TRACE_PIDS);
7883 }
7884 
7885 static int
7886 ftrace_no_pid_open(struct inode *inode, struct file *file)
7887 {
7888 	return pid_open(inode, file, TRACE_NO_PIDS);
7889 }
7890 
7891 static void ignore_task_cpu(void *data)
7892 {
7893 	struct trace_array *tr = data;
7894 	struct trace_pid_list *pid_list;
7895 	struct trace_pid_list *no_pid_list;
7896 
7897 	/*
7898 	 * This function is called by on_each_cpu() while the
7899 	 * event_mutex is held.
7900 	 */
7901 	pid_list = rcu_dereference_protected(tr->function_pids,
7902 					     mutex_is_locked(&ftrace_lock));
7903 	no_pid_list = rcu_dereference_protected(tr->function_no_pids,
7904 						mutex_is_locked(&ftrace_lock));
7905 
7906 	if (trace_ignore_this_task(pid_list, no_pid_list, current))
7907 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7908 			       FTRACE_PID_IGNORE);
7909 	else
7910 		this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
7911 			       current->pid);
7912 }
7913 
7914 static ssize_t
7915 pid_write(struct file *filp, const char __user *ubuf,
7916 	  size_t cnt, loff_t *ppos, int type)
7917 {
7918 	struct seq_file *m = filp->private_data;
7919 	struct trace_array *tr = m->private;
7920 	struct trace_pid_list *filtered_pids;
7921 	struct trace_pid_list *other_pids;
7922 	struct trace_pid_list *pid_list;
7923 	ssize_t ret;
7924 
7925 	if (!cnt)
7926 		return 0;
7927 
7928 	mutex_lock(&ftrace_lock);
7929 
7930 	switch (type) {
7931 	case TRACE_PIDS:
7932 		filtered_pids = rcu_dereference_protected(tr->function_pids,
7933 					     lockdep_is_held(&ftrace_lock));
7934 		other_pids = rcu_dereference_protected(tr->function_no_pids,
7935 					     lockdep_is_held(&ftrace_lock));
7936 		break;
7937 	case TRACE_NO_PIDS:
7938 		filtered_pids = rcu_dereference_protected(tr->function_no_pids,
7939 					     lockdep_is_held(&ftrace_lock));
7940 		other_pids = rcu_dereference_protected(tr->function_pids,
7941 					     lockdep_is_held(&ftrace_lock));
7942 		break;
7943 	default:
7944 		ret = -EINVAL;
7945 		WARN_ON_ONCE(1);
7946 		goto out;
7947 	}
7948 
7949 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
7950 	if (ret < 0)
7951 		goto out;
7952 
7953 	switch (type) {
7954 	case TRACE_PIDS:
7955 		rcu_assign_pointer(tr->function_pids, pid_list);
7956 		break;
7957 	case TRACE_NO_PIDS:
7958 		rcu_assign_pointer(tr->function_no_pids, pid_list);
7959 		break;
7960 	}
7961 
7962 
7963 	if (filtered_pids) {
7964 		synchronize_rcu();
7965 		trace_pid_list_free(filtered_pids);
7966 	} else if (pid_list && !other_pids) {
7967 		/* Register a probe to set whether to ignore the tracing of a task */
7968 		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
7969 	}
7970 
7971 	/*
7972 	 * Ignoring of pids is done at task switch. But we have to
7973 	 * check for those tasks that are currently running.
7974 	 * Always do this in case a pid was appended or removed.
7975 	 */
7976 	on_each_cpu(ignore_task_cpu, tr, 1);
7977 
7978 	ftrace_update_pid_func();
7979 	ftrace_startup_all(0);
7980  out:
7981 	mutex_unlock(&ftrace_lock);
7982 
7983 	if (ret > 0)
7984 		*ppos += ret;
7985 
7986 	return ret;
7987 }
7988 
7989 static ssize_t
7990 ftrace_pid_write(struct file *filp, const char __user *ubuf,
7991 		 size_t cnt, loff_t *ppos)
7992 {
7993 	return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
7994 }
7995 
7996 static ssize_t
7997 ftrace_no_pid_write(struct file *filp, const char __user *ubuf,
7998 		    size_t cnt, loff_t *ppos)
7999 {
8000 	return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
8001 }
8002 
8003 static int
8004 ftrace_pid_release(struct inode *inode, struct file *file)
8005 {
8006 	struct trace_array *tr = inode->i_private;
8007 
8008 	trace_array_put(tr);
8009 
8010 	return seq_release(inode, file);
8011 }
8012 
8013 static const struct file_operations ftrace_pid_fops = {
8014 	.open		= ftrace_pid_open,
8015 	.write		= ftrace_pid_write,
8016 	.read		= seq_read,
8017 	.llseek		= tracing_lseek,
8018 	.release	= ftrace_pid_release,
8019 };
8020 
8021 static const struct file_operations ftrace_no_pid_fops = {
8022 	.open		= ftrace_no_pid_open,
8023 	.write		= ftrace_no_pid_write,
8024 	.read		= seq_read,
8025 	.llseek		= tracing_lseek,
8026 	.release	= ftrace_pid_release,
8027 };
8028 
8029 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
8030 {
8031 	trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer,
8032 			    tr, &ftrace_pid_fops);
8033 	trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE,
8034 			  d_tracer, tr, &ftrace_no_pid_fops);
8035 }
8036 
8037 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
8038 					 struct dentry *d_tracer)
8039 {
8040 	/* Only the top level directory has the dyn_tracefs and profile */
8041 	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
8042 
8043 	ftrace_init_dyn_tracefs(d_tracer);
8044 	ftrace_profile_tracefs(d_tracer);
8045 }
8046 
8047 /**
8048  * ftrace_kill - kill ftrace
8049  *
8050  * This function should be used by panic code. It stops ftrace
8051  * but in a not so nice way. If you need to simply kill ftrace
8052  * from a non-atomic section, use ftrace_kill.
8053  */
8054 void ftrace_kill(void)
8055 {
8056 	ftrace_disabled = 1;
8057 	ftrace_enabled = 0;
8058 	ftrace_trace_function = ftrace_stub;
8059 }
8060 
8061 /**
8062  * ftrace_is_dead - Test if ftrace is dead or not.
8063  *
8064  * Returns 1 if ftrace is "dead", zero otherwise.
8065  */
8066 int ftrace_is_dead(void)
8067 {
8068 	return ftrace_disabled;
8069 }
8070 
8071 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
8072 /*
8073  * When registering ftrace_ops with IPMODIFY, it is necessary to make sure
8074  * it doesn't conflict with any direct ftrace_ops. If there is existing
8075  * direct ftrace_ops on a kernel function being patched, call
8076  * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing.
8077  *
8078  * @ops:     ftrace_ops being registered.
8079  *
8080  * Returns:
8081  *         0 on success;
8082  *         Negative on failure.
8083  */
8084 static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops)
8085 {
8086 	struct ftrace_func_entry *entry;
8087 	struct ftrace_hash *hash;
8088 	struct ftrace_ops *op;
8089 	int size, i, ret;
8090 
8091 	lockdep_assert_held_once(&direct_mutex);
8092 
8093 	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
8094 		return 0;
8095 
8096 	hash = ops->func_hash->filter_hash;
8097 	size = 1 << hash->size_bits;
8098 	for (i = 0; i < size; i++) {
8099 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
8100 			unsigned long ip = entry->ip;
8101 			bool found_op = false;
8102 
8103 			mutex_lock(&ftrace_lock);
8104 			do_for_each_ftrace_op(op, ftrace_ops_list) {
8105 				if (!(op->flags & FTRACE_OPS_FL_DIRECT))
8106 					continue;
8107 				if (ops_references_ip(op, ip)) {
8108 					found_op = true;
8109 					break;
8110 				}
8111 			} while_for_each_ftrace_op(op);
8112 			mutex_unlock(&ftrace_lock);
8113 
8114 			if (found_op) {
8115 				if (!op->ops_func)
8116 					return -EBUSY;
8117 
8118 				ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER);
8119 				if (ret)
8120 					return ret;
8121 			}
8122 		}
8123 	}
8124 
8125 	return 0;
8126 }
8127 
8128 /*
8129  * Similar to prepare_direct_functions_for_ipmodify, clean up after ops
8130  * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT
8131  * ops.
8132  */
8133 static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops)
8134 {
8135 	struct ftrace_func_entry *entry;
8136 	struct ftrace_hash *hash;
8137 	struct ftrace_ops *op;
8138 	int size, i;
8139 
8140 	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
8141 		return;
8142 
8143 	mutex_lock(&direct_mutex);
8144 
8145 	hash = ops->func_hash->filter_hash;
8146 	size = 1 << hash->size_bits;
8147 	for (i = 0; i < size; i++) {
8148 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
8149 			unsigned long ip = entry->ip;
8150 			bool found_op = false;
8151 
8152 			mutex_lock(&ftrace_lock);
8153 			do_for_each_ftrace_op(op, ftrace_ops_list) {
8154 				if (!(op->flags & FTRACE_OPS_FL_DIRECT))
8155 					continue;
8156 				if (ops_references_ip(op, ip)) {
8157 					found_op = true;
8158 					break;
8159 				}
8160 			} while_for_each_ftrace_op(op);
8161 			mutex_unlock(&ftrace_lock);
8162 
8163 			/* The cleanup is optional, ignore any errors */
8164 			if (found_op && op->ops_func)
8165 				op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER);
8166 		}
8167 	}
8168 	mutex_unlock(&direct_mutex);
8169 }
8170 
8171 #define lock_direct_mutex()	mutex_lock(&direct_mutex)
8172 #define unlock_direct_mutex()	mutex_unlock(&direct_mutex)
8173 
8174 #else  /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
8175 
8176 static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops)
8177 {
8178 	return 0;
8179 }
8180 
8181 static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops)
8182 {
8183 }
8184 
8185 #define lock_direct_mutex()	do { } while (0)
8186 #define unlock_direct_mutex()	do { } while (0)
8187 
8188 #endif  /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
8189 
8190 /*
8191  * Similar to register_ftrace_function, except we don't lock direct_mutex.
8192  */
8193 static int register_ftrace_function_nolock(struct ftrace_ops *ops)
8194 {
8195 	int ret;
8196 
8197 	ftrace_ops_init(ops);
8198 
8199 	mutex_lock(&ftrace_lock);
8200 
8201 	ret = ftrace_startup(ops, 0);
8202 
8203 	mutex_unlock(&ftrace_lock);
8204 
8205 	return ret;
8206 }
8207 
8208 /**
8209  * register_ftrace_function - register a function for profiling
8210  * @ops:	ops structure that holds the function for profiling.
8211  *
8212  * Register a function to be called by all functions in the
8213  * kernel.
8214  *
8215  * Note: @ops->func and all the functions it calls must be labeled
8216  *       with "notrace", otherwise it will go into a
8217  *       recursive loop.
8218  */
8219 int register_ftrace_function(struct ftrace_ops *ops)
8220 {
8221 	int ret;
8222 
8223 	lock_direct_mutex();
8224 	ret = prepare_direct_functions_for_ipmodify(ops);
8225 	if (ret < 0)
8226 		goto out_unlock;
8227 
8228 	ret = register_ftrace_function_nolock(ops);
8229 
8230 out_unlock:
8231 	unlock_direct_mutex();
8232 	return ret;
8233 }
8234 EXPORT_SYMBOL_GPL(register_ftrace_function);
8235 
8236 /**
8237  * unregister_ftrace_function - unregister a function for profiling.
8238  * @ops:	ops structure that holds the function to unregister
8239  *
8240  * Unregister a function that was added to be called by ftrace profiling.
8241  */
8242 int unregister_ftrace_function(struct ftrace_ops *ops)
8243 {
8244 	int ret;
8245 
8246 	mutex_lock(&ftrace_lock);
8247 	ret = ftrace_shutdown(ops, 0);
8248 	mutex_unlock(&ftrace_lock);
8249 
8250 	cleanup_direct_functions_after_ipmodify(ops);
8251 	return ret;
8252 }
8253 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
8254 
8255 static int symbols_cmp(const void *a, const void *b)
8256 {
8257 	const char **str_a = (const char **) a;
8258 	const char **str_b = (const char **) b;
8259 
8260 	return strcmp(*str_a, *str_b);
8261 }
8262 
8263 struct kallsyms_data {
8264 	unsigned long *addrs;
8265 	const char **syms;
8266 	size_t cnt;
8267 	size_t found;
8268 };
8269 
8270 static int kallsyms_callback(void *data, const char *name,
8271 			     struct module *mod, unsigned long addr)
8272 {
8273 	struct kallsyms_data *args = data;
8274 	const char **sym;
8275 	int idx;
8276 
8277 	sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp);
8278 	if (!sym)
8279 		return 0;
8280 
8281 	idx = sym - args->syms;
8282 	if (args->addrs[idx])
8283 		return 0;
8284 
8285 	if (!ftrace_location(addr))
8286 		return 0;
8287 
8288 	args->addrs[idx] = addr;
8289 	args->found++;
8290 	return args->found == args->cnt ? 1 : 0;
8291 }
8292 
8293 /**
8294  * ftrace_lookup_symbols - Lookup addresses for array of symbols
8295  *
8296  * @sorted_syms: array of symbols pointers symbols to resolve,
8297  * must be alphabetically sorted
8298  * @cnt: number of symbols/addresses in @syms/@addrs arrays
8299  * @addrs: array for storing resulting addresses
8300  *
8301  * This function looks up addresses for array of symbols provided in
8302  * @syms array (must be alphabetically sorted) and stores them in
8303  * @addrs array, which needs to be big enough to store at least @cnt
8304  * addresses.
8305  *
8306  * This function returns 0 if all provided symbols are found,
8307  * -ESRCH otherwise.
8308  */
8309 int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs)
8310 {
8311 	struct kallsyms_data args;
8312 	int err;
8313 
8314 	memset(addrs, 0, sizeof(*addrs) * cnt);
8315 	args.addrs = addrs;
8316 	args.syms = sorted_syms;
8317 	args.cnt = cnt;
8318 	args.found = 0;
8319 	err = kallsyms_on_each_symbol(kallsyms_callback, &args);
8320 	if (err < 0)
8321 		return err;
8322 	return args.found == args.cnt ? 0 : -ESRCH;
8323 }
8324 
8325 #ifdef CONFIG_SYSCTL
8326 
8327 #ifdef CONFIG_DYNAMIC_FTRACE
8328 static void ftrace_startup_sysctl(void)
8329 {
8330 	int command;
8331 
8332 	if (unlikely(ftrace_disabled))
8333 		return;
8334 
8335 	/* Force update next time */
8336 	saved_ftrace_func = NULL;
8337 	/* ftrace_start_up is true if we want ftrace running */
8338 	if (ftrace_start_up) {
8339 		command = FTRACE_UPDATE_CALLS;
8340 		if (ftrace_graph_active)
8341 			command |= FTRACE_START_FUNC_RET;
8342 		ftrace_startup_enable(command);
8343 	}
8344 }
8345 
8346 static void ftrace_shutdown_sysctl(void)
8347 {
8348 	int command;
8349 
8350 	if (unlikely(ftrace_disabled))
8351 		return;
8352 
8353 	/* ftrace_start_up is true if ftrace is running */
8354 	if (ftrace_start_up) {
8355 		command = FTRACE_DISABLE_CALLS;
8356 		if (ftrace_graph_active)
8357 			command |= FTRACE_STOP_FUNC_RET;
8358 		ftrace_run_update_code(command);
8359 	}
8360 }
8361 #else
8362 # define ftrace_startup_sysctl()       do { } while (0)
8363 # define ftrace_shutdown_sysctl()      do { } while (0)
8364 #endif /* CONFIG_DYNAMIC_FTRACE */
8365 
8366 static bool is_permanent_ops_registered(void)
8367 {
8368 	struct ftrace_ops *op;
8369 
8370 	do_for_each_ftrace_op(op, ftrace_ops_list) {
8371 		if (op->flags & FTRACE_OPS_FL_PERMANENT)
8372 			return true;
8373 	} while_for_each_ftrace_op(op);
8374 
8375 	return false;
8376 }
8377 
8378 static int
8379 ftrace_enable_sysctl(struct ctl_table *table, int write,
8380 		     void *buffer, size_t *lenp, loff_t *ppos)
8381 {
8382 	int ret = -ENODEV;
8383 
8384 	mutex_lock(&ftrace_lock);
8385 
8386 	if (unlikely(ftrace_disabled))
8387 		goto out;
8388 
8389 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
8390 
8391 	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
8392 		goto out;
8393 
8394 	if (ftrace_enabled) {
8395 
8396 		/* we are starting ftrace again */
8397 		if (rcu_dereference_protected(ftrace_ops_list,
8398 			lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
8399 			update_ftrace_function();
8400 
8401 		ftrace_startup_sysctl();
8402 
8403 	} else {
8404 		if (is_permanent_ops_registered()) {
8405 			ftrace_enabled = true;
8406 			ret = -EBUSY;
8407 			goto out;
8408 		}
8409 
8410 		/* stopping ftrace calls (just send to ftrace_stub) */
8411 		ftrace_trace_function = ftrace_stub;
8412 
8413 		ftrace_shutdown_sysctl();
8414 	}
8415 
8416 	last_ftrace_enabled = !!ftrace_enabled;
8417  out:
8418 	mutex_unlock(&ftrace_lock);
8419 	return ret;
8420 }
8421 
8422 static struct ctl_table ftrace_sysctls[] = {
8423 	{
8424 		.procname       = "ftrace_enabled",
8425 		.data           = &ftrace_enabled,
8426 		.maxlen         = sizeof(int),
8427 		.mode           = 0644,
8428 		.proc_handler   = ftrace_enable_sysctl,
8429 	},
8430 	{}
8431 };
8432 
8433 static int __init ftrace_sysctl_init(void)
8434 {
8435 	register_sysctl_init("kernel", ftrace_sysctls);
8436 	return 0;
8437 }
8438 late_initcall(ftrace_sysctl_init);
8439 #endif
8440