xref: /linux/kernel/trace/ftrace.c (revision ec63e2a4897075e427c121d863bd89c44578094f)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Infrastructure for profiling code inserted by 'gcc -pg'.
4  *
5  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7  *
8  * Originally ported from the -rt patch by:
9  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10  *
11  * Based on code in the latency_tracer, that is:
12  *
13  *  Copyright (C) 2004-2006 Ingo Molnar
14  *  Copyright (C) 2004 Nadia Yvette Chambers
15  */
16 
17 #include <linux/stop_machine.h>
18 #include <linux/clocksource.h>
19 #include <linux/sched/task.h>
20 #include <linux/kallsyms.h>
21 #include <linux/seq_file.h>
22 #include <linux/tracefs.h>
23 #include <linux/hardirq.h>
24 #include <linux/kthread.h>
25 #include <linux/uaccess.h>
26 #include <linux/bsearch.h>
27 #include <linux/module.h>
28 #include <linux/ftrace.h>
29 #include <linux/sysctl.h>
30 #include <linux/slab.h>
31 #include <linux/ctype.h>
32 #include <linux/sort.h>
33 #include <linux/list.h>
34 #include <linux/hash.h>
35 #include <linux/rcupdate.h>
36 #include <linux/kprobes.h>
37 
38 #include <trace/events/sched.h>
39 
40 #include <asm/sections.h>
41 #include <asm/setup.h>
42 
43 #include "ftrace_internal.h"
44 #include "trace_output.h"
45 #include "trace_stat.h"
46 
47 #define FTRACE_WARN_ON(cond)			\
48 	({					\
49 		int ___r = cond;		\
50 		if (WARN_ON(___r))		\
51 			ftrace_kill();		\
52 		___r;				\
53 	})
54 
55 #define FTRACE_WARN_ON_ONCE(cond)		\
56 	({					\
57 		int ___r = cond;		\
58 		if (WARN_ON_ONCE(___r))		\
59 			ftrace_kill();		\
60 		___r;				\
61 	})
62 
63 /* hash bits for specific function selection */
64 #define FTRACE_HASH_BITS 7
65 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
66 #define FTRACE_HASH_DEFAULT_BITS 10
67 #define FTRACE_HASH_MAX_BITS 12
68 
69 #ifdef CONFIG_DYNAMIC_FTRACE
70 #define INIT_OPS_HASH(opsname)	\
71 	.func_hash		= &opsname.local_hash,			\
72 	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
73 #define ASSIGN_OPS_HASH(opsname, val) \
74 	.func_hash		= val, \
75 	.local_hash.regex_lock	= __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
76 #else
77 #define INIT_OPS_HASH(opsname)
78 #define ASSIGN_OPS_HASH(opsname, val)
79 #endif
80 
81 enum {
82 	FTRACE_MODIFY_ENABLE_FL		= (1 << 0),
83 	FTRACE_MODIFY_MAY_SLEEP_FL	= (1 << 1),
84 };
85 
86 struct ftrace_ops ftrace_list_end __read_mostly = {
87 	.func		= ftrace_stub,
88 	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
89 	INIT_OPS_HASH(ftrace_list_end)
90 };
91 
92 /* ftrace_enabled is a method to turn ftrace on or off */
93 int ftrace_enabled __read_mostly;
94 static int last_ftrace_enabled;
95 
96 /* Current function tracing op */
97 struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
98 /* What to set function_trace_op to */
99 static struct ftrace_ops *set_function_trace_op;
100 
101 static bool ftrace_pids_enabled(struct ftrace_ops *ops)
102 {
103 	struct trace_array *tr;
104 
105 	if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
106 		return false;
107 
108 	tr = ops->private;
109 
110 	return tr->function_pids != NULL;
111 }
112 
113 static void ftrace_update_trampoline(struct ftrace_ops *ops);
114 
115 /*
116  * ftrace_disabled is set when an anomaly is discovered.
117  * ftrace_disabled is much stronger than ftrace_enabled.
118  */
119 static int ftrace_disabled __read_mostly;
120 
121 DEFINE_MUTEX(ftrace_lock);
122 
123 struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
124 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
125 struct ftrace_ops global_ops;
126 
127 #if ARCH_SUPPORTS_FTRACE_OPS
128 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
129 				 struct ftrace_ops *op, struct pt_regs *regs);
130 #else
131 /* See comment below, where ftrace_ops_list_func is defined */
132 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
133 #define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
134 #endif
135 
136 static inline void ftrace_ops_init(struct ftrace_ops *ops)
137 {
138 #ifdef CONFIG_DYNAMIC_FTRACE
139 	if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
140 		mutex_init(&ops->local_hash.regex_lock);
141 		ops->func_hash = &ops->local_hash;
142 		ops->flags |= FTRACE_OPS_FL_INITIALIZED;
143 	}
144 #endif
145 }
146 
147 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
148 			    struct ftrace_ops *op, struct pt_regs *regs)
149 {
150 	struct trace_array *tr = op->private;
151 
152 	if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
153 		return;
154 
155 	op->saved_func(ip, parent_ip, op, regs);
156 }
157 
158 static void ftrace_sync(struct work_struct *work)
159 {
160 	/*
161 	 * This function is just a stub to implement a hard force
162 	 * of synchronize_rcu(). This requires synchronizing
163 	 * tasks even in userspace and idle.
164 	 *
165 	 * Yes, function tracing is rude.
166 	 */
167 }
168 
169 static void ftrace_sync_ipi(void *data)
170 {
171 	/* Probably not needed, but do it anyway */
172 	smp_rmb();
173 }
174 
175 static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
176 {
177 	/*
178 	 * If this is a dynamic, RCU, or per CPU ops, or we force list func,
179 	 * then it needs to call the list anyway.
180 	 */
181 	if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
182 	    FTRACE_FORCE_LIST_FUNC)
183 		return ftrace_ops_list_func;
184 
185 	return ftrace_ops_get_func(ops);
186 }
187 
188 static void update_ftrace_function(void)
189 {
190 	ftrace_func_t func;
191 
192 	/*
193 	 * Prepare the ftrace_ops that the arch callback will use.
194 	 * If there's only one ftrace_ops registered, the ftrace_ops_list
195 	 * will point to the ops we want.
196 	 */
197 	set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
198 						lockdep_is_held(&ftrace_lock));
199 
200 	/* If there's no ftrace_ops registered, just call the stub function */
201 	if (set_function_trace_op == &ftrace_list_end) {
202 		func = ftrace_stub;
203 
204 	/*
205 	 * If we are at the end of the list and this ops is
206 	 * recursion safe and not dynamic and the arch supports passing ops,
207 	 * then have the mcount trampoline call the function directly.
208 	 */
209 	} else if (rcu_dereference_protected(ftrace_ops_list->next,
210 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
211 		func = ftrace_ops_get_list_func(ftrace_ops_list);
212 
213 	} else {
214 		/* Just use the default ftrace_ops */
215 		set_function_trace_op = &ftrace_list_end;
216 		func = ftrace_ops_list_func;
217 	}
218 
219 	update_function_graph_func();
220 
221 	/* If there's no change, then do nothing more here */
222 	if (ftrace_trace_function == func)
223 		return;
224 
225 	/*
226 	 * If we are using the list function, it doesn't care
227 	 * about the function_trace_ops.
228 	 */
229 	if (func == ftrace_ops_list_func) {
230 		ftrace_trace_function = func;
231 		/*
232 		 * Don't even bother setting function_trace_ops,
233 		 * it would be racy to do so anyway.
234 		 */
235 		return;
236 	}
237 
238 #ifndef CONFIG_DYNAMIC_FTRACE
239 	/*
240 	 * For static tracing, we need to be a bit more careful.
241 	 * The function change takes affect immediately. Thus,
242 	 * we need to coorditate the setting of the function_trace_ops
243 	 * with the setting of the ftrace_trace_function.
244 	 *
245 	 * Set the function to the list ops, which will call the
246 	 * function we want, albeit indirectly, but it handles the
247 	 * ftrace_ops and doesn't depend on function_trace_op.
248 	 */
249 	ftrace_trace_function = ftrace_ops_list_func;
250 	/*
251 	 * Make sure all CPUs see this. Yes this is slow, but static
252 	 * tracing is slow and nasty to have enabled.
253 	 */
254 	schedule_on_each_cpu(ftrace_sync);
255 	/* Now all cpus are using the list ops. */
256 	function_trace_op = set_function_trace_op;
257 	/* Make sure the function_trace_op is visible on all CPUs */
258 	smp_wmb();
259 	/* Nasty way to force a rmb on all cpus */
260 	smp_call_function(ftrace_sync_ipi, NULL, 1);
261 	/* OK, we are all set to update the ftrace_trace_function now! */
262 #endif /* !CONFIG_DYNAMIC_FTRACE */
263 
264 	ftrace_trace_function = func;
265 }
266 
267 static void add_ftrace_ops(struct ftrace_ops __rcu **list,
268 			   struct ftrace_ops *ops)
269 {
270 	rcu_assign_pointer(ops->next, *list);
271 
272 	/*
273 	 * We are entering ops into the list but another
274 	 * CPU might be walking that list. We need to make sure
275 	 * the ops->next pointer is valid before another CPU sees
276 	 * the ops pointer included into the list.
277 	 */
278 	rcu_assign_pointer(*list, ops);
279 }
280 
281 static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
282 			     struct ftrace_ops *ops)
283 {
284 	struct ftrace_ops **p;
285 
286 	/*
287 	 * If we are removing the last function, then simply point
288 	 * to the ftrace_stub.
289 	 */
290 	if (rcu_dereference_protected(*list,
291 			lockdep_is_held(&ftrace_lock)) == ops &&
292 	    rcu_dereference_protected(ops->next,
293 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
294 		*list = &ftrace_list_end;
295 		return 0;
296 	}
297 
298 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
299 		if (*p == ops)
300 			break;
301 
302 	if (*p != ops)
303 		return -1;
304 
305 	*p = (*p)->next;
306 	return 0;
307 }
308 
309 static void ftrace_update_trampoline(struct ftrace_ops *ops);
310 
311 int __register_ftrace_function(struct ftrace_ops *ops)
312 {
313 	if (ops->flags & FTRACE_OPS_FL_DELETED)
314 		return -EINVAL;
315 
316 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
317 		return -EBUSY;
318 
319 #ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
320 	/*
321 	 * If the ftrace_ops specifies SAVE_REGS, then it only can be used
322 	 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
323 	 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
324 	 */
325 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
326 	    !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
327 		return -EINVAL;
328 
329 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
330 		ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
331 #endif
332 
333 	if (!core_kernel_data((unsigned long)ops))
334 		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
335 
336 	add_ftrace_ops(&ftrace_ops_list, ops);
337 
338 	/* Always save the function, and reset at unregistering */
339 	ops->saved_func = ops->func;
340 
341 	if (ftrace_pids_enabled(ops))
342 		ops->func = ftrace_pid_func;
343 
344 	ftrace_update_trampoline(ops);
345 
346 	if (ftrace_enabled)
347 		update_ftrace_function();
348 
349 	return 0;
350 }
351 
352 int __unregister_ftrace_function(struct ftrace_ops *ops)
353 {
354 	int ret;
355 
356 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
357 		return -EBUSY;
358 
359 	ret = remove_ftrace_ops(&ftrace_ops_list, ops);
360 
361 	if (ret < 0)
362 		return ret;
363 
364 	if (ftrace_enabled)
365 		update_ftrace_function();
366 
367 	ops->func = ops->saved_func;
368 
369 	return 0;
370 }
371 
372 static void ftrace_update_pid_func(void)
373 {
374 	struct ftrace_ops *op;
375 
376 	/* Only do something if we are tracing something */
377 	if (ftrace_trace_function == ftrace_stub)
378 		return;
379 
380 	do_for_each_ftrace_op(op, ftrace_ops_list) {
381 		if (op->flags & FTRACE_OPS_FL_PID) {
382 			op->func = ftrace_pids_enabled(op) ?
383 				ftrace_pid_func : op->saved_func;
384 			ftrace_update_trampoline(op);
385 		}
386 	} while_for_each_ftrace_op(op);
387 
388 	update_ftrace_function();
389 }
390 
391 #ifdef CONFIG_FUNCTION_PROFILER
392 struct ftrace_profile {
393 	struct hlist_node		node;
394 	unsigned long			ip;
395 	unsigned long			counter;
396 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
397 	unsigned long long		time;
398 	unsigned long long		time_squared;
399 #endif
400 };
401 
402 struct ftrace_profile_page {
403 	struct ftrace_profile_page	*next;
404 	unsigned long			index;
405 	struct ftrace_profile		records[];
406 };
407 
408 struct ftrace_profile_stat {
409 	atomic_t			disabled;
410 	struct hlist_head		*hash;
411 	struct ftrace_profile_page	*pages;
412 	struct ftrace_profile_page	*start;
413 	struct tracer_stat		stat;
414 };
415 
416 #define PROFILE_RECORDS_SIZE						\
417 	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
418 
419 #define PROFILES_PER_PAGE					\
420 	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
421 
422 static int ftrace_profile_enabled __read_mostly;
423 
424 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
425 static DEFINE_MUTEX(ftrace_profile_lock);
426 
427 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
428 
429 #define FTRACE_PROFILE_HASH_BITS 10
430 #define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
431 
432 static void *
433 function_stat_next(void *v, int idx)
434 {
435 	struct ftrace_profile *rec = v;
436 	struct ftrace_profile_page *pg;
437 
438 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
439 
440  again:
441 	if (idx != 0)
442 		rec++;
443 
444 	if ((void *)rec >= (void *)&pg->records[pg->index]) {
445 		pg = pg->next;
446 		if (!pg)
447 			return NULL;
448 		rec = &pg->records[0];
449 		if (!rec->counter)
450 			goto again;
451 	}
452 
453 	return rec;
454 }
455 
456 static void *function_stat_start(struct tracer_stat *trace)
457 {
458 	struct ftrace_profile_stat *stat =
459 		container_of(trace, struct ftrace_profile_stat, stat);
460 
461 	if (!stat || !stat->start)
462 		return NULL;
463 
464 	return function_stat_next(&stat->start->records[0], 0);
465 }
466 
467 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
468 /* function graph compares on total time */
469 static int function_stat_cmp(void *p1, void *p2)
470 {
471 	struct ftrace_profile *a = p1;
472 	struct ftrace_profile *b = p2;
473 
474 	if (a->time < b->time)
475 		return -1;
476 	if (a->time > b->time)
477 		return 1;
478 	else
479 		return 0;
480 }
481 #else
482 /* not function graph compares against hits */
483 static int function_stat_cmp(void *p1, void *p2)
484 {
485 	struct ftrace_profile *a = p1;
486 	struct ftrace_profile *b = p2;
487 
488 	if (a->counter < b->counter)
489 		return -1;
490 	if (a->counter > b->counter)
491 		return 1;
492 	else
493 		return 0;
494 }
495 #endif
496 
497 static int function_stat_headers(struct seq_file *m)
498 {
499 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
500 	seq_puts(m, "  Function                               "
501 		 "Hit    Time            Avg             s^2\n"
502 		    "  --------                               "
503 		 "---    ----            ---             ---\n");
504 #else
505 	seq_puts(m, "  Function                               Hit\n"
506 		    "  --------                               ---\n");
507 #endif
508 	return 0;
509 }
510 
511 static int function_stat_show(struct seq_file *m, void *v)
512 {
513 	struct ftrace_profile *rec = v;
514 	char str[KSYM_SYMBOL_LEN];
515 	int ret = 0;
516 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
517 	static struct trace_seq s;
518 	unsigned long long avg;
519 	unsigned long long stddev;
520 #endif
521 	mutex_lock(&ftrace_profile_lock);
522 
523 	/* we raced with function_profile_reset() */
524 	if (unlikely(rec->counter == 0)) {
525 		ret = -EBUSY;
526 		goto out;
527 	}
528 
529 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
530 	avg = rec->time;
531 	do_div(avg, rec->counter);
532 	if (tracing_thresh && (avg < tracing_thresh))
533 		goto out;
534 #endif
535 
536 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
537 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
538 
539 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
540 	seq_puts(m, "    ");
541 
542 	/* Sample standard deviation (s^2) */
543 	if (rec->counter <= 1)
544 		stddev = 0;
545 	else {
546 		/*
547 		 * Apply Welford's method:
548 		 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
549 		 */
550 		stddev = rec->counter * rec->time_squared -
551 			 rec->time * rec->time;
552 
553 		/*
554 		 * Divide only 1000 for ns^2 -> us^2 conversion.
555 		 * trace_print_graph_duration will divide 1000 again.
556 		 */
557 		do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
558 	}
559 
560 	trace_seq_init(&s);
561 	trace_print_graph_duration(rec->time, &s);
562 	trace_seq_puts(&s, "    ");
563 	trace_print_graph_duration(avg, &s);
564 	trace_seq_puts(&s, "    ");
565 	trace_print_graph_duration(stddev, &s);
566 	trace_print_seq(m, &s);
567 #endif
568 	seq_putc(m, '\n');
569 out:
570 	mutex_unlock(&ftrace_profile_lock);
571 
572 	return ret;
573 }
574 
575 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
576 {
577 	struct ftrace_profile_page *pg;
578 
579 	pg = stat->pages = stat->start;
580 
581 	while (pg) {
582 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
583 		pg->index = 0;
584 		pg = pg->next;
585 	}
586 
587 	memset(stat->hash, 0,
588 	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
589 }
590 
591 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
592 {
593 	struct ftrace_profile_page *pg;
594 	int functions;
595 	int pages;
596 	int i;
597 
598 	/* If we already allocated, do nothing */
599 	if (stat->pages)
600 		return 0;
601 
602 	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
603 	if (!stat->pages)
604 		return -ENOMEM;
605 
606 #ifdef CONFIG_DYNAMIC_FTRACE
607 	functions = ftrace_update_tot_cnt;
608 #else
609 	/*
610 	 * We do not know the number of functions that exist because
611 	 * dynamic tracing is what counts them. With past experience
612 	 * we have around 20K functions. That should be more than enough.
613 	 * It is highly unlikely we will execute every function in
614 	 * the kernel.
615 	 */
616 	functions = 20000;
617 #endif
618 
619 	pg = stat->start = stat->pages;
620 
621 	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
622 
623 	for (i = 1; i < pages; i++) {
624 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
625 		if (!pg->next)
626 			goto out_free;
627 		pg = pg->next;
628 	}
629 
630 	return 0;
631 
632  out_free:
633 	pg = stat->start;
634 	while (pg) {
635 		unsigned long tmp = (unsigned long)pg;
636 
637 		pg = pg->next;
638 		free_page(tmp);
639 	}
640 
641 	stat->pages = NULL;
642 	stat->start = NULL;
643 
644 	return -ENOMEM;
645 }
646 
647 static int ftrace_profile_init_cpu(int cpu)
648 {
649 	struct ftrace_profile_stat *stat;
650 	int size;
651 
652 	stat = &per_cpu(ftrace_profile_stats, cpu);
653 
654 	if (stat->hash) {
655 		/* If the profile is already created, simply reset it */
656 		ftrace_profile_reset(stat);
657 		return 0;
658 	}
659 
660 	/*
661 	 * We are profiling all functions, but usually only a few thousand
662 	 * functions are hit. We'll make a hash of 1024 items.
663 	 */
664 	size = FTRACE_PROFILE_HASH_SIZE;
665 
666 	stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
667 
668 	if (!stat->hash)
669 		return -ENOMEM;
670 
671 	/* Preallocate the function profiling pages */
672 	if (ftrace_profile_pages_init(stat) < 0) {
673 		kfree(stat->hash);
674 		stat->hash = NULL;
675 		return -ENOMEM;
676 	}
677 
678 	return 0;
679 }
680 
681 static int ftrace_profile_init(void)
682 {
683 	int cpu;
684 	int ret = 0;
685 
686 	for_each_possible_cpu(cpu) {
687 		ret = ftrace_profile_init_cpu(cpu);
688 		if (ret)
689 			break;
690 	}
691 
692 	return ret;
693 }
694 
695 /* interrupts must be disabled */
696 static struct ftrace_profile *
697 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
698 {
699 	struct ftrace_profile *rec;
700 	struct hlist_head *hhd;
701 	unsigned long key;
702 
703 	key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
704 	hhd = &stat->hash[key];
705 
706 	if (hlist_empty(hhd))
707 		return NULL;
708 
709 	hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
710 		if (rec->ip == ip)
711 			return rec;
712 	}
713 
714 	return NULL;
715 }
716 
717 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
718 			       struct ftrace_profile *rec)
719 {
720 	unsigned long key;
721 
722 	key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
723 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
724 }
725 
726 /*
727  * The memory is already allocated, this simply finds a new record to use.
728  */
729 static struct ftrace_profile *
730 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
731 {
732 	struct ftrace_profile *rec = NULL;
733 
734 	/* prevent recursion (from NMIs) */
735 	if (atomic_inc_return(&stat->disabled) != 1)
736 		goto out;
737 
738 	/*
739 	 * Try to find the function again since an NMI
740 	 * could have added it
741 	 */
742 	rec = ftrace_find_profiled_func(stat, ip);
743 	if (rec)
744 		goto out;
745 
746 	if (stat->pages->index == PROFILES_PER_PAGE) {
747 		if (!stat->pages->next)
748 			goto out;
749 		stat->pages = stat->pages->next;
750 	}
751 
752 	rec = &stat->pages->records[stat->pages->index++];
753 	rec->ip = ip;
754 	ftrace_add_profile(stat, rec);
755 
756  out:
757 	atomic_dec(&stat->disabled);
758 
759 	return rec;
760 }
761 
762 static void
763 function_profile_call(unsigned long ip, unsigned long parent_ip,
764 		      struct ftrace_ops *ops, struct pt_regs *regs)
765 {
766 	struct ftrace_profile_stat *stat;
767 	struct ftrace_profile *rec;
768 	unsigned long flags;
769 
770 	if (!ftrace_profile_enabled)
771 		return;
772 
773 	local_irq_save(flags);
774 
775 	stat = this_cpu_ptr(&ftrace_profile_stats);
776 	if (!stat->hash || !ftrace_profile_enabled)
777 		goto out;
778 
779 	rec = ftrace_find_profiled_func(stat, ip);
780 	if (!rec) {
781 		rec = ftrace_profile_alloc(stat, ip);
782 		if (!rec)
783 			goto out;
784 	}
785 
786 	rec->counter++;
787  out:
788 	local_irq_restore(flags);
789 }
790 
791 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
792 static bool fgraph_graph_time = true;
793 
794 void ftrace_graph_graph_time_control(bool enable)
795 {
796 	fgraph_graph_time = enable;
797 }
798 
799 static int profile_graph_entry(struct ftrace_graph_ent *trace)
800 {
801 	struct ftrace_ret_stack *ret_stack;
802 
803 	function_profile_call(trace->func, 0, NULL, NULL);
804 
805 	/* If function graph is shutting down, ret_stack can be NULL */
806 	if (!current->ret_stack)
807 		return 0;
808 
809 	ret_stack = ftrace_graph_get_ret_stack(current, 0);
810 	if (ret_stack)
811 		ret_stack->subtime = 0;
812 
813 	return 1;
814 }
815 
816 static void profile_graph_return(struct ftrace_graph_ret *trace)
817 {
818 	struct ftrace_ret_stack *ret_stack;
819 	struct ftrace_profile_stat *stat;
820 	unsigned long long calltime;
821 	struct ftrace_profile *rec;
822 	unsigned long flags;
823 
824 	local_irq_save(flags);
825 	stat = this_cpu_ptr(&ftrace_profile_stats);
826 	if (!stat->hash || !ftrace_profile_enabled)
827 		goto out;
828 
829 	/* If the calltime was zero'd ignore it */
830 	if (!trace->calltime)
831 		goto out;
832 
833 	calltime = trace->rettime - trace->calltime;
834 
835 	if (!fgraph_graph_time) {
836 
837 		/* Append this call time to the parent time to subtract */
838 		ret_stack = ftrace_graph_get_ret_stack(current, 1);
839 		if (ret_stack)
840 			ret_stack->subtime += calltime;
841 
842 		ret_stack = ftrace_graph_get_ret_stack(current, 0);
843 		if (ret_stack && ret_stack->subtime < calltime)
844 			calltime -= ret_stack->subtime;
845 		else
846 			calltime = 0;
847 	}
848 
849 	rec = ftrace_find_profiled_func(stat, trace->func);
850 	if (rec) {
851 		rec->time += calltime;
852 		rec->time_squared += calltime * calltime;
853 	}
854 
855  out:
856 	local_irq_restore(flags);
857 }
858 
859 static struct fgraph_ops fprofiler_ops = {
860 	.entryfunc = &profile_graph_entry,
861 	.retfunc = &profile_graph_return,
862 };
863 
864 static int register_ftrace_profiler(void)
865 {
866 	return register_ftrace_graph(&fprofiler_ops);
867 }
868 
869 static void unregister_ftrace_profiler(void)
870 {
871 	unregister_ftrace_graph(&fprofiler_ops);
872 }
873 #else
874 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
875 	.func		= function_profile_call,
876 	.flags		= FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
877 	INIT_OPS_HASH(ftrace_profile_ops)
878 };
879 
880 static int register_ftrace_profiler(void)
881 {
882 	return register_ftrace_function(&ftrace_profile_ops);
883 }
884 
885 static void unregister_ftrace_profiler(void)
886 {
887 	unregister_ftrace_function(&ftrace_profile_ops);
888 }
889 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
890 
891 static ssize_t
892 ftrace_profile_write(struct file *filp, const char __user *ubuf,
893 		     size_t cnt, loff_t *ppos)
894 {
895 	unsigned long val;
896 	int ret;
897 
898 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
899 	if (ret)
900 		return ret;
901 
902 	val = !!val;
903 
904 	mutex_lock(&ftrace_profile_lock);
905 	if (ftrace_profile_enabled ^ val) {
906 		if (val) {
907 			ret = ftrace_profile_init();
908 			if (ret < 0) {
909 				cnt = ret;
910 				goto out;
911 			}
912 
913 			ret = register_ftrace_profiler();
914 			if (ret < 0) {
915 				cnt = ret;
916 				goto out;
917 			}
918 			ftrace_profile_enabled = 1;
919 		} else {
920 			ftrace_profile_enabled = 0;
921 			/*
922 			 * unregister_ftrace_profiler calls stop_machine
923 			 * so this acts like an synchronize_rcu.
924 			 */
925 			unregister_ftrace_profiler();
926 		}
927 	}
928  out:
929 	mutex_unlock(&ftrace_profile_lock);
930 
931 	*ppos += cnt;
932 
933 	return cnt;
934 }
935 
936 static ssize_t
937 ftrace_profile_read(struct file *filp, char __user *ubuf,
938 		     size_t cnt, loff_t *ppos)
939 {
940 	char buf[64];		/* big enough to hold a number */
941 	int r;
942 
943 	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
944 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
945 }
946 
947 static const struct file_operations ftrace_profile_fops = {
948 	.open		= tracing_open_generic,
949 	.read		= ftrace_profile_read,
950 	.write		= ftrace_profile_write,
951 	.llseek		= default_llseek,
952 };
953 
954 /* used to initialize the real stat files */
955 static struct tracer_stat function_stats __initdata = {
956 	.name		= "functions",
957 	.stat_start	= function_stat_start,
958 	.stat_next	= function_stat_next,
959 	.stat_cmp	= function_stat_cmp,
960 	.stat_headers	= function_stat_headers,
961 	.stat_show	= function_stat_show
962 };
963 
964 static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
965 {
966 	struct ftrace_profile_stat *stat;
967 	struct dentry *entry;
968 	char *name;
969 	int ret;
970 	int cpu;
971 
972 	for_each_possible_cpu(cpu) {
973 		stat = &per_cpu(ftrace_profile_stats, cpu);
974 
975 		name = kasprintf(GFP_KERNEL, "function%d", cpu);
976 		if (!name) {
977 			/*
978 			 * The files created are permanent, if something happens
979 			 * we still do not free memory.
980 			 */
981 			WARN(1,
982 			     "Could not allocate stat file for cpu %d\n",
983 			     cpu);
984 			return;
985 		}
986 		stat->stat = function_stats;
987 		stat->stat.name = name;
988 		ret = register_stat_tracer(&stat->stat);
989 		if (ret) {
990 			WARN(1,
991 			     "Could not register function stat for cpu %d\n",
992 			     cpu);
993 			kfree(name);
994 			return;
995 		}
996 	}
997 
998 	entry = tracefs_create_file("function_profile_enabled", 0644,
999 				    d_tracer, NULL, &ftrace_profile_fops);
1000 	if (!entry)
1001 		pr_warn("Could not create tracefs 'function_profile_enabled' entry\n");
1002 }
1003 
1004 #else /* CONFIG_FUNCTION_PROFILER */
1005 static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
1006 {
1007 }
1008 #endif /* CONFIG_FUNCTION_PROFILER */
1009 
1010 #ifdef CONFIG_DYNAMIC_FTRACE
1011 
1012 static struct ftrace_ops *removed_ops;
1013 
1014 /*
1015  * Set when doing a global update, like enabling all recs or disabling them.
1016  * It is not set when just updating a single ftrace_ops.
1017  */
1018 static bool update_all_ops;
1019 
1020 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
1021 # error Dynamic ftrace depends on MCOUNT_RECORD
1022 #endif
1023 
1024 struct ftrace_func_entry {
1025 	struct hlist_node hlist;
1026 	unsigned long ip;
1027 };
1028 
1029 struct ftrace_func_probe {
1030 	struct ftrace_probe_ops	*probe_ops;
1031 	struct ftrace_ops	ops;
1032 	struct trace_array	*tr;
1033 	struct list_head	list;
1034 	void			*data;
1035 	int			ref;
1036 };
1037 
1038 /*
1039  * We make these constant because no one should touch them,
1040  * but they are used as the default "empty hash", to avoid allocating
1041  * it all the time. These are in a read only section such that if
1042  * anyone does try to modify it, it will cause an exception.
1043  */
1044 static const struct hlist_head empty_buckets[1];
1045 static const struct ftrace_hash empty_hash = {
1046 	.buckets = (struct hlist_head *)empty_buckets,
1047 };
1048 #define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1049 
1050 struct ftrace_ops global_ops = {
1051 	.func				= ftrace_stub,
1052 	.local_hash.notrace_hash	= EMPTY_HASH,
1053 	.local_hash.filter_hash		= EMPTY_HASH,
1054 	INIT_OPS_HASH(global_ops)
1055 	.flags				= FTRACE_OPS_FL_RECURSION_SAFE |
1056 					  FTRACE_OPS_FL_INITIALIZED |
1057 					  FTRACE_OPS_FL_PID,
1058 };
1059 
1060 /*
1061  * Used by the stack undwinder to know about dynamic ftrace trampolines.
1062  */
1063 struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
1064 {
1065 	struct ftrace_ops *op = NULL;
1066 
1067 	/*
1068 	 * Some of the ops may be dynamically allocated,
1069 	 * they are freed after a synchronize_rcu().
1070 	 */
1071 	preempt_disable_notrace();
1072 
1073 	do_for_each_ftrace_op(op, ftrace_ops_list) {
1074 		/*
1075 		 * This is to check for dynamically allocated trampolines.
1076 		 * Trampolines that are in kernel text will have
1077 		 * core_kernel_text() return true.
1078 		 */
1079 		if (op->trampoline && op->trampoline_size)
1080 			if (addr >= op->trampoline &&
1081 			    addr < op->trampoline + op->trampoline_size) {
1082 				preempt_enable_notrace();
1083 				return op;
1084 			}
1085 	} while_for_each_ftrace_op(op);
1086 	preempt_enable_notrace();
1087 
1088 	return NULL;
1089 }
1090 
1091 /*
1092  * This is used by __kernel_text_address() to return true if the
1093  * address is on a dynamically allocated trampoline that would
1094  * not return true for either core_kernel_text() or
1095  * is_module_text_address().
1096  */
1097 bool is_ftrace_trampoline(unsigned long addr)
1098 {
1099 	return ftrace_ops_trampoline(addr) != NULL;
1100 }
1101 
1102 struct ftrace_page {
1103 	struct ftrace_page	*next;
1104 	struct dyn_ftrace	*records;
1105 	int			index;
1106 	int			size;
1107 };
1108 
1109 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
1110 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1111 
1112 /* estimate from running different kernels */
1113 #define NR_TO_INIT		10000
1114 
1115 static struct ftrace_page	*ftrace_pages_start;
1116 static struct ftrace_page	*ftrace_pages;
1117 
1118 static __always_inline unsigned long
1119 ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
1120 {
1121 	if (hash->size_bits > 0)
1122 		return hash_long(ip, hash->size_bits);
1123 
1124 	return 0;
1125 }
1126 
1127 /* Only use this function if ftrace_hash_empty() has already been tested */
1128 static __always_inline struct ftrace_func_entry *
1129 __ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1130 {
1131 	unsigned long key;
1132 	struct ftrace_func_entry *entry;
1133 	struct hlist_head *hhd;
1134 
1135 	key = ftrace_hash_key(hash, ip);
1136 	hhd = &hash->buckets[key];
1137 
1138 	hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
1139 		if (entry->ip == ip)
1140 			return entry;
1141 	}
1142 	return NULL;
1143 }
1144 
1145 /**
1146  * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1147  * @hash: The hash to look at
1148  * @ip: The instruction pointer to test
1149  *
1150  * Search a given @hash to see if a given instruction pointer (@ip)
1151  * exists in it.
1152  *
1153  * Returns the entry that holds the @ip if found. NULL otherwise.
1154  */
1155 struct ftrace_func_entry *
1156 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1157 {
1158 	if (ftrace_hash_empty(hash))
1159 		return NULL;
1160 
1161 	return __ftrace_lookup_ip(hash, ip);
1162 }
1163 
1164 static void __add_hash_entry(struct ftrace_hash *hash,
1165 			     struct ftrace_func_entry *entry)
1166 {
1167 	struct hlist_head *hhd;
1168 	unsigned long key;
1169 
1170 	key = ftrace_hash_key(hash, entry->ip);
1171 	hhd = &hash->buckets[key];
1172 	hlist_add_head(&entry->hlist, hhd);
1173 	hash->count++;
1174 }
1175 
1176 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1177 {
1178 	struct ftrace_func_entry *entry;
1179 
1180 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1181 	if (!entry)
1182 		return -ENOMEM;
1183 
1184 	entry->ip = ip;
1185 	__add_hash_entry(hash, entry);
1186 
1187 	return 0;
1188 }
1189 
1190 static void
1191 free_hash_entry(struct ftrace_hash *hash,
1192 		  struct ftrace_func_entry *entry)
1193 {
1194 	hlist_del(&entry->hlist);
1195 	kfree(entry);
1196 	hash->count--;
1197 }
1198 
1199 static void
1200 remove_hash_entry(struct ftrace_hash *hash,
1201 		  struct ftrace_func_entry *entry)
1202 {
1203 	hlist_del_rcu(&entry->hlist);
1204 	hash->count--;
1205 }
1206 
1207 static void ftrace_hash_clear(struct ftrace_hash *hash)
1208 {
1209 	struct hlist_head *hhd;
1210 	struct hlist_node *tn;
1211 	struct ftrace_func_entry *entry;
1212 	int size = 1 << hash->size_bits;
1213 	int i;
1214 
1215 	if (!hash->count)
1216 		return;
1217 
1218 	for (i = 0; i < size; i++) {
1219 		hhd = &hash->buckets[i];
1220 		hlist_for_each_entry_safe(entry, tn, hhd, hlist)
1221 			free_hash_entry(hash, entry);
1222 	}
1223 	FTRACE_WARN_ON(hash->count);
1224 }
1225 
1226 static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
1227 {
1228 	list_del(&ftrace_mod->list);
1229 	kfree(ftrace_mod->module);
1230 	kfree(ftrace_mod->func);
1231 	kfree(ftrace_mod);
1232 }
1233 
1234 static void clear_ftrace_mod_list(struct list_head *head)
1235 {
1236 	struct ftrace_mod_load *p, *n;
1237 
1238 	/* stack tracer isn't supported yet */
1239 	if (!head)
1240 		return;
1241 
1242 	mutex_lock(&ftrace_lock);
1243 	list_for_each_entry_safe(p, n, head, list)
1244 		free_ftrace_mod(p);
1245 	mutex_unlock(&ftrace_lock);
1246 }
1247 
1248 static void free_ftrace_hash(struct ftrace_hash *hash)
1249 {
1250 	if (!hash || hash == EMPTY_HASH)
1251 		return;
1252 	ftrace_hash_clear(hash);
1253 	kfree(hash->buckets);
1254 	kfree(hash);
1255 }
1256 
1257 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1258 {
1259 	struct ftrace_hash *hash;
1260 
1261 	hash = container_of(rcu, struct ftrace_hash, rcu);
1262 	free_ftrace_hash(hash);
1263 }
1264 
1265 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1266 {
1267 	if (!hash || hash == EMPTY_HASH)
1268 		return;
1269 	call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
1270 }
1271 
1272 void ftrace_free_filter(struct ftrace_ops *ops)
1273 {
1274 	ftrace_ops_init(ops);
1275 	free_ftrace_hash(ops->func_hash->filter_hash);
1276 	free_ftrace_hash(ops->func_hash->notrace_hash);
1277 }
1278 
1279 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1280 {
1281 	struct ftrace_hash *hash;
1282 	int size;
1283 
1284 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1285 	if (!hash)
1286 		return NULL;
1287 
1288 	size = 1 << size_bits;
1289 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1290 
1291 	if (!hash->buckets) {
1292 		kfree(hash);
1293 		return NULL;
1294 	}
1295 
1296 	hash->size_bits = size_bits;
1297 
1298 	return hash;
1299 }
1300 
1301 
1302 static int ftrace_add_mod(struct trace_array *tr,
1303 			  const char *func, const char *module,
1304 			  int enable)
1305 {
1306 	struct ftrace_mod_load *ftrace_mod;
1307 	struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
1308 
1309 	ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
1310 	if (!ftrace_mod)
1311 		return -ENOMEM;
1312 
1313 	ftrace_mod->func = kstrdup(func, GFP_KERNEL);
1314 	ftrace_mod->module = kstrdup(module, GFP_KERNEL);
1315 	ftrace_mod->enable = enable;
1316 
1317 	if (!ftrace_mod->func || !ftrace_mod->module)
1318 		goto out_free;
1319 
1320 	list_add(&ftrace_mod->list, mod_head);
1321 
1322 	return 0;
1323 
1324  out_free:
1325 	free_ftrace_mod(ftrace_mod);
1326 
1327 	return -ENOMEM;
1328 }
1329 
1330 static struct ftrace_hash *
1331 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1332 {
1333 	struct ftrace_func_entry *entry;
1334 	struct ftrace_hash *new_hash;
1335 	int size;
1336 	int ret;
1337 	int i;
1338 
1339 	new_hash = alloc_ftrace_hash(size_bits);
1340 	if (!new_hash)
1341 		return NULL;
1342 
1343 	if (hash)
1344 		new_hash->flags = hash->flags;
1345 
1346 	/* Empty hash? */
1347 	if (ftrace_hash_empty(hash))
1348 		return new_hash;
1349 
1350 	size = 1 << hash->size_bits;
1351 	for (i = 0; i < size; i++) {
1352 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
1353 			ret = add_hash_entry(new_hash, entry->ip);
1354 			if (ret < 0)
1355 				goto free_hash;
1356 		}
1357 	}
1358 
1359 	FTRACE_WARN_ON(new_hash->count != hash->count);
1360 
1361 	return new_hash;
1362 
1363  free_hash:
1364 	free_ftrace_hash(new_hash);
1365 	return NULL;
1366 }
1367 
1368 static void
1369 ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
1370 static void
1371 ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
1372 
1373 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1374 				       struct ftrace_hash *new_hash);
1375 
1376 static struct ftrace_hash *
1377 __ftrace_hash_move(struct ftrace_hash *src)
1378 {
1379 	struct ftrace_func_entry *entry;
1380 	struct hlist_node *tn;
1381 	struct hlist_head *hhd;
1382 	struct ftrace_hash *new_hash;
1383 	int size = src->count;
1384 	int bits = 0;
1385 	int i;
1386 
1387 	/*
1388 	 * If the new source is empty, just return the empty_hash.
1389 	 */
1390 	if (ftrace_hash_empty(src))
1391 		return EMPTY_HASH;
1392 
1393 	/*
1394 	 * Make the hash size about 1/2 the # found
1395 	 */
1396 	for (size /= 2; size; size >>= 1)
1397 		bits++;
1398 
1399 	/* Don't allocate too much */
1400 	if (bits > FTRACE_HASH_MAX_BITS)
1401 		bits = FTRACE_HASH_MAX_BITS;
1402 
1403 	new_hash = alloc_ftrace_hash(bits);
1404 	if (!new_hash)
1405 		return NULL;
1406 
1407 	new_hash->flags = src->flags;
1408 
1409 	size = 1 << src->size_bits;
1410 	for (i = 0; i < size; i++) {
1411 		hhd = &src->buckets[i];
1412 		hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
1413 			remove_hash_entry(src, entry);
1414 			__add_hash_entry(new_hash, entry);
1415 		}
1416 	}
1417 
1418 	return new_hash;
1419 }
1420 
1421 static int
1422 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1423 		 struct ftrace_hash **dst, struct ftrace_hash *src)
1424 {
1425 	struct ftrace_hash *new_hash;
1426 	int ret;
1427 
1428 	/* Reject setting notrace hash on IPMODIFY ftrace_ops */
1429 	if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
1430 		return -EINVAL;
1431 
1432 	new_hash = __ftrace_hash_move(src);
1433 	if (!new_hash)
1434 		return -ENOMEM;
1435 
1436 	/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
1437 	if (enable) {
1438 		/* IPMODIFY should be updated only when filter_hash updating */
1439 		ret = ftrace_hash_ipmodify_update(ops, new_hash);
1440 		if (ret < 0) {
1441 			free_ftrace_hash(new_hash);
1442 			return ret;
1443 		}
1444 	}
1445 
1446 	/*
1447 	 * Remove the current set, update the hash and add
1448 	 * them back.
1449 	 */
1450 	ftrace_hash_rec_disable_modify(ops, enable);
1451 
1452 	rcu_assign_pointer(*dst, new_hash);
1453 
1454 	ftrace_hash_rec_enable_modify(ops, enable);
1455 
1456 	return 0;
1457 }
1458 
1459 static bool hash_contains_ip(unsigned long ip,
1460 			     struct ftrace_ops_hash *hash)
1461 {
1462 	/*
1463 	 * The function record is a match if it exists in the filter
1464 	 * hash and not in the notrace hash. Note, an emty hash is
1465 	 * considered a match for the filter hash, but an empty
1466 	 * notrace hash is considered not in the notrace hash.
1467 	 */
1468 	return (ftrace_hash_empty(hash->filter_hash) ||
1469 		__ftrace_lookup_ip(hash->filter_hash, ip)) &&
1470 		(ftrace_hash_empty(hash->notrace_hash) ||
1471 		 !__ftrace_lookup_ip(hash->notrace_hash, ip));
1472 }
1473 
1474 /*
1475  * Test the hashes for this ops to see if we want to call
1476  * the ops->func or not.
1477  *
1478  * It's a match if the ip is in the ops->filter_hash or
1479  * the filter_hash does not exist or is empty,
1480  *  AND
1481  * the ip is not in the ops->notrace_hash.
1482  *
1483  * This needs to be called with preemption disabled as
1484  * the hashes are freed with call_rcu().
1485  */
1486 int
1487 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
1488 {
1489 	struct ftrace_ops_hash hash;
1490 	int ret;
1491 
1492 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
1493 	/*
1494 	 * There's a small race when adding ops that the ftrace handler
1495 	 * that wants regs, may be called without them. We can not
1496 	 * allow that handler to be called if regs is NULL.
1497 	 */
1498 	if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
1499 		return 0;
1500 #endif
1501 
1502 	rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
1503 	rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
1504 
1505 	if (hash_contains_ip(ip, &hash))
1506 		ret = 1;
1507 	else
1508 		ret = 0;
1509 
1510 	return ret;
1511 }
1512 
1513 /*
1514  * This is a double for. Do not use 'break' to break out of the loop,
1515  * you must use a goto.
1516  */
1517 #define do_for_each_ftrace_rec(pg, rec)					\
1518 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1519 		int _____i;						\
1520 		for (_____i = 0; _____i < pg->index; _____i++) {	\
1521 			rec = &pg->records[_____i];
1522 
1523 #define while_for_each_ftrace_rec()		\
1524 		}				\
1525 	}
1526 
1527 
1528 static int ftrace_cmp_recs(const void *a, const void *b)
1529 {
1530 	const struct dyn_ftrace *key = a;
1531 	const struct dyn_ftrace *rec = b;
1532 
1533 	if (key->flags < rec->ip)
1534 		return -1;
1535 	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1536 		return 1;
1537 	return 0;
1538 }
1539 
1540 /**
1541  * ftrace_location_range - return the first address of a traced location
1542  *	if it touches the given ip range
1543  * @start: start of range to search.
1544  * @end: end of range to search (inclusive). @end points to the last byte
1545  *	to check.
1546  *
1547  * Returns rec->ip if the related ftrace location is a least partly within
1548  * the given address range. That is, the first address of the instruction
1549  * that is either a NOP or call to the function tracer. It checks the ftrace
1550  * internal tables to determine if the address belongs or not.
1551  */
1552 unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1553 {
1554 	struct ftrace_page *pg;
1555 	struct dyn_ftrace *rec;
1556 	struct dyn_ftrace key;
1557 
1558 	key.ip = start;
1559 	key.flags = end;	/* overload flags, as it is unsigned long */
1560 
1561 	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1562 		if (end < pg->records[0].ip ||
1563 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1564 			continue;
1565 		rec = bsearch(&key, pg->records, pg->index,
1566 			      sizeof(struct dyn_ftrace),
1567 			      ftrace_cmp_recs);
1568 		if (rec)
1569 			return rec->ip;
1570 	}
1571 
1572 	return 0;
1573 }
1574 
1575 /**
1576  * ftrace_location - return true if the ip giving is a traced location
1577  * @ip: the instruction pointer to check
1578  *
1579  * Returns rec->ip if @ip given is a pointer to a ftrace location.
1580  * That is, the instruction that is either a NOP or call to
1581  * the function tracer. It checks the ftrace internal tables to
1582  * determine if the address belongs or not.
1583  */
1584 unsigned long ftrace_location(unsigned long ip)
1585 {
1586 	return ftrace_location_range(ip, ip);
1587 }
1588 
1589 /**
1590  * ftrace_text_reserved - return true if range contains an ftrace location
1591  * @start: start of range to search
1592  * @end: end of range to search (inclusive). @end points to the last byte to check.
1593  *
1594  * Returns 1 if @start and @end contains a ftrace location.
1595  * That is, the instruction that is either a NOP or call to
1596  * the function tracer. It checks the ftrace internal tables to
1597  * determine if the address belongs or not.
1598  */
1599 int ftrace_text_reserved(const void *start, const void *end)
1600 {
1601 	unsigned long ret;
1602 
1603 	ret = ftrace_location_range((unsigned long)start,
1604 				    (unsigned long)end);
1605 
1606 	return (int)!!ret;
1607 }
1608 
1609 /* Test if ops registered to this rec needs regs */
1610 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1611 {
1612 	struct ftrace_ops *ops;
1613 	bool keep_regs = false;
1614 
1615 	for (ops = ftrace_ops_list;
1616 	     ops != &ftrace_list_end; ops = ops->next) {
1617 		/* pass rec in as regs to have non-NULL val */
1618 		if (ftrace_ops_test(ops, rec->ip, rec)) {
1619 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1620 				keep_regs = true;
1621 				break;
1622 			}
1623 		}
1624 	}
1625 
1626 	return  keep_regs;
1627 }
1628 
1629 static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
1630 				     int filter_hash,
1631 				     bool inc)
1632 {
1633 	struct ftrace_hash *hash;
1634 	struct ftrace_hash *other_hash;
1635 	struct ftrace_page *pg;
1636 	struct dyn_ftrace *rec;
1637 	bool update = false;
1638 	int count = 0;
1639 	int all = false;
1640 
1641 	/* Only update if the ops has been registered */
1642 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1643 		return false;
1644 
1645 	/*
1646 	 * In the filter_hash case:
1647 	 *   If the count is zero, we update all records.
1648 	 *   Otherwise we just update the items in the hash.
1649 	 *
1650 	 * In the notrace_hash case:
1651 	 *   We enable the update in the hash.
1652 	 *   As disabling notrace means enabling the tracing,
1653 	 *   and enabling notrace means disabling, the inc variable
1654 	 *   gets inversed.
1655 	 */
1656 	if (filter_hash) {
1657 		hash = ops->func_hash->filter_hash;
1658 		other_hash = ops->func_hash->notrace_hash;
1659 		if (ftrace_hash_empty(hash))
1660 			all = true;
1661 	} else {
1662 		inc = !inc;
1663 		hash = ops->func_hash->notrace_hash;
1664 		other_hash = ops->func_hash->filter_hash;
1665 		/*
1666 		 * If the notrace hash has no items,
1667 		 * then there's nothing to do.
1668 		 */
1669 		if (ftrace_hash_empty(hash))
1670 			return false;
1671 	}
1672 
1673 	do_for_each_ftrace_rec(pg, rec) {
1674 		int in_other_hash = 0;
1675 		int in_hash = 0;
1676 		int match = 0;
1677 
1678 		if (rec->flags & FTRACE_FL_DISABLED)
1679 			continue;
1680 
1681 		if (all) {
1682 			/*
1683 			 * Only the filter_hash affects all records.
1684 			 * Update if the record is not in the notrace hash.
1685 			 */
1686 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1687 				match = 1;
1688 		} else {
1689 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1690 			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1691 
1692 			/*
1693 			 * If filter_hash is set, we want to match all functions
1694 			 * that are in the hash but not in the other hash.
1695 			 *
1696 			 * If filter_hash is not set, then we are decrementing.
1697 			 * That means we match anything that is in the hash
1698 			 * and also in the other_hash. That is, we need to turn
1699 			 * off functions in the other hash because they are disabled
1700 			 * by this hash.
1701 			 */
1702 			if (filter_hash && in_hash && !in_other_hash)
1703 				match = 1;
1704 			else if (!filter_hash && in_hash &&
1705 				 (in_other_hash || ftrace_hash_empty(other_hash)))
1706 				match = 1;
1707 		}
1708 		if (!match)
1709 			continue;
1710 
1711 		if (inc) {
1712 			rec->flags++;
1713 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1714 				return false;
1715 
1716 			/*
1717 			 * If there's only a single callback registered to a
1718 			 * function, and the ops has a trampoline registered
1719 			 * for it, then we can call it directly.
1720 			 */
1721 			if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1722 				rec->flags |= FTRACE_FL_TRAMP;
1723 			else
1724 				/*
1725 				 * If we are adding another function callback
1726 				 * to this function, and the previous had a
1727 				 * custom trampoline in use, then we need to go
1728 				 * back to the default trampoline.
1729 				 */
1730 				rec->flags &= ~FTRACE_FL_TRAMP;
1731 
1732 			/*
1733 			 * If any ops wants regs saved for this function
1734 			 * then all ops will get saved regs.
1735 			 */
1736 			if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
1737 				rec->flags |= FTRACE_FL_REGS;
1738 		} else {
1739 			if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1740 				return false;
1741 			rec->flags--;
1742 
1743 			/*
1744 			 * If the rec had REGS enabled and the ops that is
1745 			 * being removed had REGS set, then see if there is
1746 			 * still any ops for this record that wants regs.
1747 			 * If not, we can stop recording them.
1748 			 */
1749 			if (ftrace_rec_count(rec) > 0 &&
1750 			    rec->flags & FTRACE_FL_REGS &&
1751 			    ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
1752 				if (!test_rec_ops_needs_regs(rec))
1753 					rec->flags &= ~FTRACE_FL_REGS;
1754 			}
1755 
1756 			/*
1757 			 * If the rec had TRAMP enabled, then it needs to
1758 			 * be cleared. As TRAMP can only be enabled iff
1759 			 * there is only a single ops attached to it.
1760 			 * In otherwords, always disable it on decrementing.
1761 			 * In the future, we may set it if rec count is
1762 			 * decremented to one, and the ops that is left
1763 			 * has a trampoline.
1764 			 */
1765 			rec->flags &= ~FTRACE_FL_TRAMP;
1766 
1767 			/*
1768 			 * flags will be cleared in ftrace_check_record()
1769 			 * if rec count is zero.
1770 			 */
1771 		}
1772 		count++;
1773 
1774 		/* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */
1775 		update |= ftrace_test_record(rec, 1) != FTRACE_UPDATE_IGNORE;
1776 
1777 		/* Shortcut, if we handled all records, we are done. */
1778 		if (!all && count == hash->count)
1779 			return update;
1780 	} while_for_each_ftrace_rec();
1781 
1782 	return update;
1783 }
1784 
1785 static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
1786 				    int filter_hash)
1787 {
1788 	return __ftrace_hash_rec_update(ops, filter_hash, 0);
1789 }
1790 
1791 static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
1792 				   int filter_hash)
1793 {
1794 	return __ftrace_hash_rec_update(ops, filter_hash, 1);
1795 }
1796 
1797 static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
1798 					  int filter_hash, int inc)
1799 {
1800 	struct ftrace_ops *op;
1801 
1802 	__ftrace_hash_rec_update(ops, filter_hash, inc);
1803 
1804 	if (ops->func_hash != &global_ops.local_hash)
1805 		return;
1806 
1807 	/*
1808 	 * If the ops shares the global_ops hash, then we need to update
1809 	 * all ops that are enabled and use this hash.
1810 	 */
1811 	do_for_each_ftrace_op(op, ftrace_ops_list) {
1812 		/* Already done */
1813 		if (op == ops)
1814 			continue;
1815 		if (op->func_hash == &global_ops.local_hash)
1816 			__ftrace_hash_rec_update(op, filter_hash, inc);
1817 	} while_for_each_ftrace_op(op);
1818 }
1819 
1820 static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
1821 					   int filter_hash)
1822 {
1823 	ftrace_hash_rec_update_modify(ops, filter_hash, 0);
1824 }
1825 
1826 static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
1827 					  int filter_hash)
1828 {
1829 	ftrace_hash_rec_update_modify(ops, filter_hash, 1);
1830 }
1831 
1832 /*
1833  * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
1834  * or no-needed to update, -EBUSY if it detects a conflict of the flag
1835  * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1836  * Note that old_hash and new_hash has below meanings
1837  *  - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1838  *  - If the hash is EMPTY_HASH, it hits nothing
1839  *  - Anything else hits the recs which match the hash entries.
1840  */
1841 static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
1842 					 struct ftrace_hash *old_hash,
1843 					 struct ftrace_hash *new_hash)
1844 {
1845 	struct ftrace_page *pg;
1846 	struct dyn_ftrace *rec, *end = NULL;
1847 	int in_old, in_new;
1848 
1849 	/* Only update if the ops has been registered */
1850 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1851 		return 0;
1852 
1853 	if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
1854 		return 0;
1855 
1856 	/*
1857 	 * Since the IPMODIFY is a very address sensitive action, we do not
1858 	 * allow ftrace_ops to set all functions to new hash.
1859 	 */
1860 	if (!new_hash || !old_hash)
1861 		return -EINVAL;
1862 
1863 	/* Update rec->flags */
1864 	do_for_each_ftrace_rec(pg, rec) {
1865 
1866 		if (rec->flags & FTRACE_FL_DISABLED)
1867 			continue;
1868 
1869 		/* We need to update only differences of filter_hash */
1870 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1871 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1872 		if (in_old == in_new)
1873 			continue;
1874 
1875 		if (in_new) {
1876 			/* New entries must ensure no others are using it */
1877 			if (rec->flags & FTRACE_FL_IPMODIFY)
1878 				goto rollback;
1879 			rec->flags |= FTRACE_FL_IPMODIFY;
1880 		} else /* Removed entry */
1881 			rec->flags &= ~FTRACE_FL_IPMODIFY;
1882 	} while_for_each_ftrace_rec();
1883 
1884 	return 0;
1885 
1886 rollback:
1887 	end = rec;
1888 
1889 	/* Roll back what we did above */
1890 	do_for_each_ftrace_rec(pg, rec) {
1891 
1892 		if (rec->flags & FTRACE_FL_DISABLED)
1893 			continue;
1894 
1895 		if (rec == end)
1896 			goto err_out;
1897 
1898 		in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
1899 		in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
1900 		if (in_old == in_new)
1901 			continue;
1902 
1903 		if (in_new)
1904 			rec->flags &= ~FTRACE_FL_IPMODIFY;
1905 		else
1906 			rec->flags |= FTRACE_FL_IPMODIFY;
1907 	} while_for_each_ftrace_rec();
1908 
1909 err_out:
1910 	return -EBUSY;
1911 }
1912 
1913 static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
1914 {
1915 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1916 
1917 	if (ftrace_hash_empty(hash))
1918 		hash = NULL;
1919 
1920 	return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
1921 }
1922 
1923 /* Disabling always succeeds */
1924 static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
1925 {
1926 	struct ftrace_hash *hash = ops->func_hash->filter_hash;
1927 
1928 	if (ftrace_hash_empty(hash))
1929 		hash = NULL;
1930 
1931 	__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
1932 }
1933 
1934 static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
1935 				       struct ftrace_hash *new_hash)
1936 {
1937 	struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
1938 
1939 	if (ftrace_hash_empty(old_hash))
1940 		old_hash = NULL;
1941 
1942 	if (ftrace_hash_empty(new_hash))
1943 		new_hash = NULL;
1944 
1945 	return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
1946 }
1947 
1948 static void print_ip_ins(const char *fmt, const unsigned char *p)
1949 {
1950 	int i;
1951 
1952 	printk(KERN_CONT "%s", fmt);
1953 
1954 	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1955 		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1956 }
1957 
1958 static struct ftrace_ops *
1959 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1960 static struct ftrace_ops *
1961 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1962 
1963 enum ftrace_bug_type ftrace_bug_type;
1964 const void *ftrace_expected;
1965 
1966 static void print_bug_type(void)
1967 {
1968 	switch (ftrace_bug_type) {
1969 	case FTRACE_BUG_UNKNOWN:
1970 		break;
1971 	case FTRACE_BUG_INIT:
1972 		pr_info("Initializing ftrace call sites\n");
1973 		break;
1974 	case FTRACE_BUG_NOP:
1975 		pr_info("Setting ftrace call site to NOP\n");
1976 		break;
1977 	case FTRACE_BUG_CALL:
1978 		pr_info("Setting ftrace call site to call ftrace function\n");
1979 		break;
1980 	case FTRACE_BUG_UPDATE:
1981 		pr_info("Updating ftrace call site to call a different ftrace function\n");
1982 		break;
1983 	}
1984 }
1985 
1986 /**
1987  * ftrace_bug - report and shutdown function tracer
1988  * @failed: The failed type (EFAULT, EINVAL, EPERM)
1989  * @rec: The record that failed
1990  *
1991  * The arch code that enables or disables the function tracing
1992  * can call ftrace_bug() when it has detected a problem in
1993  * modifying the code. @failed should be one of either:
1994  * EFAULT - if the problem happens on reading the @ip address
1995  * EINVAL - if what is read at @ip is not what was expected
1996  * EPERM - if the problem happens on writing to the @ip address
1997  */
1998 void ftrace_bug(int failed, struct dyn_ftrace *rec)
1999 {
2000 	unsigned long ip = rec ? rec->ip : 0;
2001 
2002 	switch (failed) {
2003 	case -EFAULT:
2004 		FTRACE_WARN_ON_ONCE(1);
2005 		pr_info("ftrace faulted on modifying ");
2006 		print_ip_sym(ip);
2007 		break;
2008 	case -EINVAL:
2009 		FTRACE_WARN_ON_ONCE(1);
2010 		pr_info("ftrace failed to modify ");
2011 		print_ip_sym(ip);
2012 		print_ip_ins(" actual:   ", (unsigned char *)ip);
2013 		pr_cont("\n");
2014 		if (ftrace_expected) {
2015 			print_ip_ins(" expected: ", ftrace_expected);
2016 			pr_cont("\n");
2017 		}
2018 		break;
2019 	case -EPERM:
2020 		FTRACE_WARN_ON_ONCE(1);
2021 		pr_info("ftrace faulted on writing ");
2022 		print_ip_sym(ip);
2023 		break;
2024 	default:
2025 		FTRACE_WARN_ON_ONCE(1);
2026 		pr_info("ftrace faulted on unknown error ");
2027 		print_ip_sym(ip);
2028 	}
2029 	print_bug_type();
2030 	if (rec) {
2031 		struct ftrace_ops *ops = NULL;
2032 
2033 		pr_info("ftrace record flags: %lx\n", rec->flags);
2034 		pr_cont(" (%ld)%s", ftrace_rec_count(rec),
2035 			rec->flags & FTRACE_FL_REGS ? " R" : "  ");
2036 		if (rec->flags & FTRACE_FL_TRAMP_EN) {
2037 			ops = ftrace_find_tramp_ops_any(rec);
2038 			if (ops) {
2039 				do {
2040 					pr_cont("\ttramp: %pS (%pS)",
2041 						(void *)ops->trampoline,
2042 						(void *)ops->func);
2043 					ops = ftrace_find_tramp_ops_next(rec, ops);
2044 				} while (ops);
2045 			} else
2046 				pr_cont("\ttramp: ERROR!");
2047 
2048 		}
2049 		ip = ftrace_get_addr_curr(rec);
2050 		pr_cont("\n expected tramp: %lx\n", ip);
2051 	}
2052 }
2053 
2054 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
2055 {
2056 	unsigned long flag = 0UL;
2057 
2058 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2059 
2060 	if (rec->flags & FTRACE_FL_DISABLED)
2061 		return FTRACE_UPDATE_IGNORE;
2062 
2063 	/*
2064 	 * If we are updating calls:
2065 	 *
2066 	 *   If the record has a ref count, then we need to enable it
2067 	 *   because someone is using it.
2068 	 *
2069 	 *   Otherwise we make sure its disabled.
2070 	 *
2071 	 * If we are disabling calls, then disable all records that
2072 	 * are enabled.
2073 	 */
2074 	if (enable && ftrace_rec_count(rec))
2075 		flag = FTRACE_FL_ENABLED;
2076 
2077 	/*
2078 	 * If enabling and the REGS flag does not match the REGS_EN, or
2079 	 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
2080 	 * this record. Set flags to fail the compare against ENABLED.
2081 	 */
2082 	if (flag) {
2083 		if (!(rec->flags & FTRACE_FL_REGS) !=
2084 		    !(rec->flags & FTRACE_FL_REGS_EN))
2085 			flag |= FTRACE_FL_REGS;
2086 
2087 		if (!(rec->flags & FTRACE_FL_TRAMP) !=
2088 		    !(rec->flags & FTRACE_FL_TRAMP_EN))
2089 			flag |= FTRACE_FL_TRAMP;
2090 	}
2091 
2092 	/* If the state of this record hasn't changed, then do nothing */
2093 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2094 		return FTRACE_UPDATE_IGNORE;
2095 
2096 	if (flag) {
2097 		/* Save off if rec is being enabled (for return value) */
2098 		flag ^= rec->flags & FTRACE_FL_ENABLED;
2099 
2100 		if (update) {
2101 			rec->flags |= FTRACE_FL_ENABLED;
2102 			if (flag & FTRACE_FL_REGS) {
2103 				if (rec->flags & FTRACE_FL_REGS)
2104 					rec->flags |= FTRACE_FL_REGS_EN;
2105 				else
2106 					rec->flags &= ~FTRACE_FL_REGS_EN;
2107 			}
2108 			if (flag & FTRACE_FL_TRAMP) {
2109 				if (rec->flags & FTRACE_FL_TRAMP)
2110 					rec->flags |= FTRACE_FL_TRAMP_EN;
2111 				else
2112 					rec->flags &= ~FTRACE_FL_TRAMP_EN;
2113 			}
2114 		}
2115 
2116 		/*
2117 		 * If this record is being updated from a nop, then
2118 		 *   return UPDATE_MAKE_CALL.
2119 		 * Otherwise,
2120 		 *   return UPDATE_MODIFY_CALL to tell the caller to convert
2121 		 *   from the save regs, to a non-save regs function or
2122 		 *   vice versa, or from a trampoline call.
2123 		 */
2124 		if (flag & FTRACE_FL_ENABLED) {
2125 			ftrace_bug_type = FTRACE_BUG_CALL;
2126 			return FTRACE_UPDATE_MAKE_CALL;
2127 		}
2128 
2129 		ftrace_bug_type = FTRACE_BUG_UPDATE;
2130 		return FTRACE_UPDATE_MODIFY_CALL;
2131 	}
2132 
2133 	if (update) {
2134 		/* If there's no more users, clear all flags */
2135 		if (!ftrace_rec_count(rec))
2136 			rec->flags = 0;
2137 		else
2138 			/*
2139 			 * Just disable the record, but keep the ops TRAMP
2140 			 * and REGS states. The _EN flags must be disabled though.
2141 			 */
2142 			rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2143 					FTRACE_FL_REGS_EN);
2144 	}
2145 
2146 	ftrace_bug_type = FTRACE_BUG_NOP;
2147 	return FTRACE_UPDATE_MAKE_NOP;
2148 }
2149 
2150 /**
2151  * ftrace_update_record, set a record that now is tracing or not
2152  * @rec: the record to update
2153  * @enable: set to 1 if the record is tracing, zero to force disable
2154  *
2155  * The records that represent all functions that can be traced need
2156  * to be updated when tracing has been enabled.
2157  */
2158 int ftrace_update_record(struct dyn_ftrace *rec, int enable)
2159 {
2160 	return ftrace_check_record(rec, enable, 1);
2161 }
2162 
2163 /**
2164  * ftrace_test_record, check if the record has been enabled or not
2165  * @rec: the record to test
2166  * @enable: set to 1 to check if enabled, 0 if it is disabled
2167  *
2168  * The arch code may need to test if a record is already set to
2169  * tracing to determine how to modify the function code that it
2170  * represents.
2171  */
2172 int ftrace_test_record(struct dyn_ftrace *rec, int enable)
2173 {
2174 	return ftrace_check_record(rec, enable, 0);
2175 }
2176 
2177 static struct ftrace_ops *
2178 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2179 {
2180 	struct ftrace_ops *op;
2181 	unsigned long ip = rec->ip;
2182 
2183 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2184 
2185 		if (!op->trampoline)
2186 			continue;
2187 
2188 		if (hash_contains_ip(ip, op->func_hash))
2189 			return op;
2190 	} while_for_each_ftrace_op(op);
2191 
2192 	return NULL;
2193 }
2194 
2195 static struct ftrace_ops *
2196 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2197 			   struct ftrace_ops *op)
2198 {
2199 	unsigned long ip = rec->ip;
2200 
2201 	while_for_each_ftrace_op(op) {
2202 
2203 		if (!op->trampoline)
2204 			continue;
2205 
2206 		if (hash_contains_ip(ip, op->func_hash))
2207 			return op;
2208 	}
2209 
2210 	return NULL;
2211 }
2212 
2213 static struct ftrace_ops *
2214 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2215 {
2216 	struct ftrace_ops *op;
2217 	unsigned long ip = rec->ip;
2218 
2219 	/*
2220 	 * Need to check removed ops first.
2221 	 * If they are being removed, and this rec has a tramp,
2222 	 * and this rec is in the ops list, then it would be the
2223 	 * one with the tramp.
2224 	 */
2225 	if (removed_ops) {
2226 		if (hash_contains_ip(ip, &removed_ops->old_hash))
2227 			return removed_ops;
2228 	}
2229 
2230 	/*
2231 	 * Need to find the current trampoline for a rec.
2232 	 * Now, a trampoline is only attached to a rec if there
2233 	 * was a single 'ops' attached to it. But this can be called
2234 	 * when we are adding another op to the rec or removing the
2235 	 * current one. Thus, if the op is being added, we can
2236 	 * ignore it because it hasn't attached itself to the rec
2237 	 * yet.
2238 	 *
2239 	 * If an ops is being modified (hooking to different functions)
2240 	 * then we don't care about the new functions that are being
2241 	 * added, just the old ones (that are probably being removed).
2242 	 *
2243 	 * If we are adding an ops to a function that already is using
2244 	 * a trampoline, it needs to be removed (trampolines are only
2245 	 * for single ops connected), then an ops that is not being
2246 	 * modified also needs to be checked.
2247 	 */
2248 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2249 
2250 		if (!op->trampoline)
2251 			continue;
2252 
2253 		/*
2254 		 * If the ops is being added, it hasn't gotten to
2255 		 * the point to be removed from this tree yet.
2256 		 */
2257 		if (op->flags & FTRACE_OPS_FL_ADDING)
2258 			continue;
2259 
2260 
2261 		/*
2262 		 * If the ops is being modified and is in the old
2263 		 * hash, then it is probably being removed from this
2264 		 * function.
2265 		 */
2266 		if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
2267 		    hash_contains_ip(ip, &op->old_hash))
2268 			return op;
2269 		/*
2270 		 * If the ops is not being added or modified, and it's
2271 		 * in its normal filter hash, then this must be the one
2272 		 * we want!
2273 		 */
2274 		if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
2275 		    hash_contains_ip(ip, op->func_hash))
2276 			return op;
2277 
2278 	} while_for_each_ftrace_op(op);
2279 
2280 	return NULL;
2281 }
2282 
2283 static struct ftrace_ops *
2284 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2285 {
2286 	struct ftrace_ops *op;
2287 	unsigned long ip = rec->ip;
2288 
2289 	do_for_each_ftrace_op(op, ftrace_ops_list) {
2290 		/* pass rec in as regs to have non-NULL val */
2291 		if (hash_contains_ip(ip, op->func_hash))
2292 			return op;
2293 	} while_for_each_ftrace_op(op);
2294 
2295 	return NULL;
2296 }
2297 
2298 /**
2299  * ftrace_get_addr_new - Get the call address to set to
2300  * @rec:  The ftrace record descriptor
2301  *
2302  * If the record has the FTRACE_FL_REGS set, that means that it
2303  * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
2304  * is not not set, then it wants to convert to the normal callback.
2305  *
2306  * Returns the address of the trampoline to set to
2307  */
2308 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2309 {
2310 	struct ftrace_ops *ops;
2311 
2312 	/* Trampolines take precedence over regs */
2313 	if (rec->flags & FTRACE_FL_TRAMP) {
2314 		ops = ftrace_find_tramp_ops_new(rec);
2315 		if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
2316 			pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
2317 				(void *)rec->ip, (void *)rec->ip, rec->flags);
2318 			/* Ftrace is shutting down, return anything */
2319 			return (unsigned long)FTRACE_ADDR;
2320 		}
2321 		return ops->trampoline;
2322 	}
2323 
2324 	if (rec->flags & FTRACE_FL_REGS)
2325 		return (unsigned long)FTRACE_REGS_ADDR;
2326 	else
2327 		return (unsigned long)FTRACE_ADDR;
2328 }
2329 
2330 /**
2331  * ftrace_get_addr_curr - Get the call address that is already there
2332  * @rec:  The ftrace record descriptor
2333  *
2334  * The FTRACE_FL_REGS_EN is set when the record already points to
2335  * a function that saves all the regs. Basically the '_EN' version
2336  * represents the current state of the function.
2337  *
2338  * Returns the address of the trampoline that is currently being called
2339  */
2340 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2341 {
2342 	struct ftrace_ops *ops;
2343 
2344 	/* Trampolines take precedence over regs */
2345 	if (rec->flags & FTRACE_FL_TRAMP_EN) {
2346 		ops = ftrace_find_tramp_ops_curr(rec);
2347 		if (FTRACE_WARN_ON(!ops)) {
2348 			pr_warn("Bad trampoline accounting at: %p (%pS)\n",
2349 				(void *)rec->ip, (void *)rec->ip);
2350 			/* Ftrace is shutting down, return anything */
2351 			return (unsigned long)FTRACE_ADDR;
2352 		}
2353 		return ops->trampoline;
2354 	}
2355 
2356 	if (rec->flags & FTRACE_FL_REGS_EN)
2357 		return (unsigned long)FTRACE_REGS_ADDR;
2358 	else
2359 		return (unsigned long)FTRACE_ADDR;
2360 }
2361 
2362 static int
2363 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
2364 {
2365 	unsigned long ftrace_old_addr;
2366 	unsigned long ftrace_addr;
2367 	int ret;
2368 
2369 	ftrace_addr = ftrace_get_addr_new(rec);
2370 
2371 	/* This needs to be done before we call ftrace_update_record */
2372 	ftrace_old_addr = ftrace_get_addr_curr(rec);
2373 
2374 	ret = ftrace_update_record(rec, enable);
2375 
2376 	ftrace_bug_type = FTRACE_BUG_UNKNOWN;
2377 
2378 	switch (ret) {
2379 	case FTRACE_UPDATE_IGNORE:
2380 		return 0;
2381 
2382 	case FTRACE_UPDATE_MAKE_CALL:
2383 		ftrace_bug_type = FTRACE_BUG_CALL;
2384 		return ftrace_make_call(rec, ftrace_addr);
2385 
2386 	case FTRACE_UPDATE_MAKE_NOP:
2387 		ftrace_bug_type = FTRACE_BUG_NOP;
2388 		return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2389 
2390 	case FTRACE_UPDATE_MODIFY_CALL:
2391 		ftrace_bug_type = FTRACE_BUG_UPDATE;
2392 		return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2393 	}
2394 
2395 	return -1; /* unknown ftrace bug */
2396 }
2397 
2398 void __weak ftrace_replace_code(int mod_flags)
2399 {
2400 	struct dyn_ftrace *rec;
2401 	struct ftrace_page *pg;
2402 	int enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
2403 	int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
2404 	int failed;
2405 
2406 	if (unlikely(ftrace_disabled))
2407 		return;
2408 
2409 	do_for_each_ftrace_rec(pg, rec) {
2410 
2411 		if (rec->flags & FTRACE_FL_DISABLED)
2412 			continue;
2413 
2414 		failed = __ftrace_replace_code(rec, enable);
2415 		if (failed) {
2416 			ftrace_bug(failed, rec);
2417 			/* Stop processing */
2418 			return;
2419 		}
2420 		if (schedulable)
2421 			cond_resched();
2422 	} while_for_each_ftrace_rec();
2423 }
2424 
2425 struct ftrace_rec_iter {
2426 	struct ftrace_page	*pg;
2427 	int			index;
2428 };
2429 
2430 /**
2431  * ftrace_rec_iter_start, start up iterating over traced functions
2432  *
2433  * Returns an iterator handle that is used to iterate over all
2434  * the records that represent address locations where functions
2435  * are traced.
2436  *
2437  * May return NULL if no records are available.
2438  */
2439 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
2440 {
2441 	/*
2442 	 * We only use a single iterator.
2443 	 * Protected by the ftrace_lock mutex.
2444 	 */
2445 	static struct ftrace_rec_iter ftrace_rec_iter;
2446 	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
2447 
2448 	iter->pg = ftrace_pages_start;
2449 	iter->index = 0;
2450 
2451 	/* Could have empty pages */
2452 	while (iter->pg && !iter->pg->index)
2453 		iter->pg = iter->pg->next;
2454 
2455 	if (!iter->pg)
2456 		return NULL;
2457 
2458 	return iter;
2459 }
2460 
2461 /**
2462  * ftrace_rec_iter_next, get the next record to process.
2463  * @iter: The handle to the iterator.
2464  *
2465  * Returns the next iterator after the given iterator @iter.
2466  */
2467 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
2468 {
2469 	iter->index++;
2470 
2471 	if (iter->index >= iter->pg->index) {
2472 		iter->pg = iter->pg->next;
2473 		iter->index = 0;
2474 
2475 		/* Could have empty pages */
2476 		while (iter->pg && !iter->pg->index)
2477 			iter->pg = iter->pg->next;
2478 	}
2479 
2480 	if (!iter->pg)
2481 		return NULL;
2482 
2483 	return iter;
2484 }
2485 
2486 /**
2487  * ftrace_rec_iter_record, get the record at the iterator location
2488  * @iter: The current iterator location
2489  *
2490  * Returns the record that the current @iter is at.
2491  */
2492 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
2493 {
2494 	return &iter->pg->records[iter->index];
2495 }
2496 
2497 static int
2498 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
2499 {
2500 	int ret;
2501 
2502 	if (unlikely(ftrace_disabled))
2503 		return 0;
2504 
2505 	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
2506 	if (ret) {
2507 		ftrace_bug_type = FTRACE_BUG_INIT;
2508 		ftrace_bug(ret, rec);
2509 		return 0;
2510 	}
2511 	return 1;
2512 }
2513 
2514 /*
2515  * archs can override this function if they must do something
2516  * before the modifying code is performed.
2517  */
2518 int __weak ftrace_arch_code_modify_prepare(void)
2519 {
2520 	return 0;
2521 }
2522 
2523 /*
2524  * archs can override this function if they must do something
2525  * after the modifying code is performed.
2526  */
2527 int __weak ftrace_arch_code_modify_post_process(void)
2528 {
2529 	return 0;
2530 }
2531 
2532 void ftrace_modify_all_code(int command)
2533 {
2534 	int update = command & FTRACE_UPDATE_TRACE_FUNC;
2535 	int mod_flags = 0;
2536 	int err = 0;
2537 
2538 	if (command & FTRACE_MAY_SLEEP)
2539 		mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
2540 
2541 	/*
2542 	 * If the ftrace_caller calls a ftrace_ops func directly,
2543 	 * we need to make sure that it only traces functions it
2544 	 * expects to trace. When doing the switch of functions,
2545 	 * we need to update to the ftrace_ops_list_func first
2546 	 * before the transition between old and new calls are set,
2547 	 * as the ftrace_ops_list_func will check the ops hashes
2548 	 * to make sure the ops are having the right functions
2549 	 * traced.
2550 	 */
2551 	if (update) {
2552 		err = ftrace_update_ftrace_func(ftrace_ops_list_func);
2553 		if (FTRACE_WARN_ON(err))
2554 			return;
2555 	}
2556 
2557 	if (command & FTRACE_UPDATE_CALLS)
2558 		ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
2559 	else if (command & FTRACE_DISABLE_CALLS)
2560 		ftrace_replace_code(mod_flags);
2561 
2562 	if (update && ftrace_trace_function != ftrace_ops_list_func) {
2563 		function_trace_op = set_function_trace_op;
2564 		smp_wmb();
2565 		/* If irqs are disabled, we are in stop machine */
2566 		if (!irqs_disabled())
2567 			smp_call_function(ftrace_sync_ipi, NULL, 1);
2568 		err = ftrace_update_ftrace_func(ftrace_trace_function);
2569 		if (FTRACE_WARN_ON(err))
2570 			return;
2571 	}
2572 
2573 	if (command & FTRACE_START_FUNC_RET)
2574 		err = ftrace_enable_ftrace_graph_caller();
2575 	else if (command & FTRACE_STOP_FUNC_RET)
2576 		err = ftrace_disable_ftrace_graph_caller();
2577 	FTRACE_WARN_ON(err);
2578 }
2579 
2580 static int __ftrace_modify_code(void *data)
2581 {
2582 	int *command = data;
2583 
2584 	ftrace_modify_all_code(*command);
2585 
2586 	return 0;
2587 }
2588 
2589 /**
2590  * ftrace_run_stop_machine, go back to the stop machine method
2591  * @command: The command to tell ftrace what to do
2592  *
2593  * If an arch needs to fall back to the stop machine method, the
2594  * it can call this function.
2595  */
2596 void ftrace_run_stop_machine(int command)
2597 {
2598 	stop_machine(__ftrace_modify_code, &command, NULL);
2599 }
2600 
2601 /**
2602  * arch_ftrace_update_code, modify the code to trace or not trace
2603  * @command: The command that needs to be done
2604  *
2605  * Archs can override this function if it does not need to
2606  * run stop_machine() to modify code.
2607  */
2608 void __weak arch_ftrace_update_code(int command)
2609 {
2610 	ftrace_run_stop_machine(command);
2611 }
2612 
2613 static void ftrace_run_update_code(int command)
2614 {
2615 	int ret;
2616 
2617 	ret = ftrace_arch_code_modify_prepare();
2618 	FTRACE_WARN_ON(ret);
2619 	if (ret)
2620 		return;
2621 
2622 	/*
2623 	 * By default we use stop_machine() to modify the code.
2624 	 * But archs can do what ever they want as long as it
2625 	 * is safe. The stop_machine() is the safest, but also
2626 	 * produces the most overhead.
2627 	 */
2628 	arch_ftrace_update_code(command);
2629 
2630 	ret = ftrace_arch_code_modify_post_process();
2631 	FTRACE_WARN_ON(ret);
2632 }
2633 
2634 static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
2635 				   struct ftrace_ops_hash *old_hash)
2636 {
2637 	ops->flags |= FTRACE_OPS_FL_MODIFYING;
2638 	ops->old_hash.filter_hash = old_hash->filter_hash;
2639 	ops->old_hash.notrace_hash = old_hash->notrace_hash;
2640 	ftrace_run_update_code(command);
2641 	ops->old_hash.filter_hash = NULL;
2642 	ops->old_hash.notrace_hash = NULL;
2643 	ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
2644 }
2645 
2646 static ftrace_func_t saved_ftrace_func;
2647 static int ftrace_start_up;
2648 
2649 void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
2650 {
2651 }
2652 
2653 static void ftrace_startup_enable(int command)
2654 {
2655 	if (saved_ftrace_func != ftrace_trace_function) {
2656 		saved_ftrace_func = ftrace_trace_function;
2657 		command |= FTRACE_UPDATE_TRACE_FUNC;
2658 	}
2659 
2660 	if (!command || !ftrace_enabled)
2661 		return;
2662 
2663 	ftrace_run_update_code(command);
2664 }
2665 
2666 static void ftrace_startup_all(int command)
2667 {
2668 	update_all_ops = true;
2669 	ftrace_startup_enable(command);
2670 	update_all_ops = false;
2671 }
2672 
2673 int ftrace_startup(struct ftrace_ops *ops, int command)
2674 {
2675 	int ret;
2676 
2677 	if (unlikely(ftrace_disabled))
2678 		return -ENODEV;
2679 
2680 	ret = __register_ftrace_function(ops);
2681 	if (ret)
2682 		return ret;
2683 
2684 	ftrace_start_up++;
2685 
2686 	/*
2687 	 * Note that ftrace probes uses this to start up
2688 	 * and modify functions it will probe. But we still
2689 	 * set the ADDING flag for modification, as probes
2690 	 * do not have trampolines. If they add them in the
2691 	 * future, then the probes will need to distinguish
2692 	 * between adding and updating probes.
2693 	 */
2694 	ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
2695 
2696 	ret = ftrace_hash_ipmodify_enable(ops);
2697 	if (ret < 0) {
2698 		/* Rollback registration process */
2699 		__unregister_ftrace_function(ops);
2700 		ftrace_start_up--;
2701 		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2702 		return ret;
2703 	}
2704 
2705 	if (ftrace_hash_rec_enable(ops, 1))
2706 		command |= FTRACE_UPDATE_CALLS;
2707 
2708 	ftrace_startup_enable(command);
2709 
2710 	ops->flags &= ~FTRACE_OPS_FL_ADDING;
2711 
2712 	return 0;
2713 }
2714 
2715 int ftrace_shutdown(struct ftrace_ops *ops, int command)
2716 {
2717 	int ret;
2718 
2719 	if (unlikely(ftrace_disabled))
2720 		return -ENODEV;
2721 
2722 	ret = __unregister_ftrace_function(ops);
2723 	if (ret)
2724 		return ret;
2725 
2726 	ftrace_start_up--;
2727 	/*
2728 	 * Just warn in case of unbalance, no need to kill ftrace, it's not
2729 	 * critical but the ftrace_call callers may be never nopped again after
2730 	 * further ftrace uses.
2731 	 */
2732 	WARN_ON_ONCE(ftrace_start_up < 0);
2733 
2734 	/* Disabling ipmodify never fails */
2735 	ftrace_hash_ipmodify_disable(ops);
2736 
2737 	if (ftrace_hash_rec_disable(ops, 1))
2738 		command |= FTRACE_UPDATE_CALLS;
2739 
2740 	ops->flags &= ~FTRACE_OPS_FL_ENABLED;
2741 
2742 	if (saved_ftrace_func != ftrace_trace_function) {
2743 		saved_ftrace_func = ftrace_trace_function;
2744 		command |= FTRACE_UPDATE_TRACE_FUNC;
2745 	}
2746 
2747 	if (!command || !ftrace_enabled) {
2748 		/*
2749 		 * If these are dynamic or per_cpu ops, they still
2750 		 * need their data freed. Since, function tracing is
2751 		 * not currently active, we can just free them
2752 		 * without synchronizing all CPUs.
2753 		 */
2754 		if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
2755 			goto free_ops;
2756 
2757 		return 0;
2758 	}
2759 
2760 	/*
2761 	 * If the ops uses a trampoline, then it needs to be
2762 	 * tested first on update.
2763 	 */
2764 	ops->flags |= FTRACE_OPS_FL_REMOVING;
2765 	removed_ops = ops;
2766 
2767 	/* The trampoline logic checks the old hashes */
2768 	ops->old_hash.filter_hash = ops->func_hash->filter_hash;
2769 	ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
2770 
2771 	ftrace_run_update_code(command);
2772 
2773 	/*
2774 	 * If there's no more ops registered with ftrace, run a
2775 	 * sanity check to make sure all rec flags are cleared.
2776 	 */
2777 	if (rcu_dereference_protected(ftrace_ops_list,
2778 			lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
2779 		struct ftrace_page *pg;
2780 		struct dyn_ftrace *rec;
2781 
2782 		do_for_each_ftrace_rec(pg, rec) {
2783 			if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_FL_DISABLED))
2784 				pr_warn("  %pS flags:%lx\n",
2785 					(void *)rec->ip, rec->flags);
2786 		} while_for_each_ftrace_rec();
2787 	}
2788 
2789 	ops->old_hash.filter_hash = NULL;
2790 	ops->old_hash.notrace_hash = NULL;
2791 
2792 	removed_ops = NULL;
2793 	ops->flags &= ~FTRACE_OPS_FL_REMOVING;
2794 
2795 	/*
2796 	 * Dynamic ops may be freed, we must make sure that all
2797 	 * callers are done before leaving this function.
2798 	 * The same goes for freeing the per_cpu data of the per_cpu
2799 	 * ops.
2800 	 */
2801 	if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
2802 		/*
2803 		 * We need to do a hard force of sched synchronization.
2804 		 * This is because we use preempt_disable() to do RCU, but
2805 		 * the function tracers can be called where RCU is not watching
2806 		 * (like before user_exit()). We can not rely on the RCU
2807 		 * infrastructure to do the synchronization, thus we must do it
2808 		 * ourselves.
2809 		 */
2810 		schedule_on_each_cpu(ftrace_sync);
2811 
2812 		/*
2813 		 * When the kernel is preeptive, tasks can be preempted
2814 		 * while on a ftrace trampoline. Just scheduling a task on
2815 		 * a CPU is not good enough to flush them. Calling
2816 		 * synchornize_rcu_tasks() will wait for those tasks to
2817 		 * execute and either schedule voluntarily or enter user space.
2818 		 */
2819 		if (IS_ENABLED(CONFIG_PREEMPT))
2820 			synchronize_rcu_tasks();
2821 
2822  free_ops:
2823 		arch_ftrace_trampoline_free(ops);
2824 	}
2825 
2826 	return 0;
2827 }
2828 
2829 static void ftrace_startup_sysctl(void)
2830 {
2831 	int command;
2832 
2833 	if (unlikely(ftrace_disabled))
2834 		return;
2835 
2836 	/* Force update next time */
2837 	saved_ftrace_func = NULL;
2838 	/* ftrace_start_up is true if we want ftrace running */
2839 	if (ftrace_start_up) {
2840 		command = FTRACE_UPDATE_CALLS;
2841 		if (ftrace_graph_active)
2842 			command |= FTRACE_START_FUNC_RET;
2843 		ftrace_startup_enable(command);
2844 	}
2845 }
2846 
2847 static void ftrace_shutdown_sysctl(void)
2848 {
2849 	int command;
2850 
2851 	if (unlikely(ftrace_disabled))
2852 		return;
2853 
2854 	/* ftrace_start_up is true if ftrace is running */
2855 	if (ftrace_start_up) {
2856 		command = FTRACE_DISABLE_CALLS;
2857 		if (ftrace_graph_active)
2858 			command |= FTRACE_STOP_FUNC_RET;
2859 		ftrace_run_update_code(command);
2860 	}
2861 }
2862 
2863 static u64		ftrace_update_time;
2864 unsigned long		ftrace_update_tot_cnt;
2865 
2866 static inline int ops_traces_mod(struct ftrace_ops *ops)
2867 {
2868 	/*
2869 	 * Filter_hash being empty will default to trace module.
2870 	 * But notrace hash requires a test of individual module functions.
2871 	 */
2872 	return ftrace_hash_empty(ops->func_hash->filter_hash) &&
2873 		ftrace_hash_empty(ops->func_hash->notrace_hash);
2874 }
2875 
2876 /*
2877  * Check if the current ops references the record.
2878  *
2879  * If the ops traces all functions, then it was already accounted for.
2880  * If the ops does not trace the current record function, skip it.
2881  * If the ops ignores the function via notrace filter, skip it.
2882  */
2883 static inline bool
2884 ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
2885 {
2886 	/* If ops isn't enabled, ignore it */
2887 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
2888 		return false;
2889 
2890 	/* If ops traces all then it includes this function */
2891 	if (ops_traces_mod(ops))
2892 		return true;
2893 
2894 	/* The function must be in the filter */
2895 	if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
2896 	    !__ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
2897 		return false;
2898 
2899 	/* If in notrace hash, we ignore it too */
2900 	if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
2901 		return false;
2902 
2903 	return true;
2904 }
2905 
2906 static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
2907 {
2908 	struct ftrace_page *pg;
2909 	struct dyn_ftrace *p;
2910 	u64 start, stop;
2911 	unsigned long update_cnt = 0;
2912 	unsigned long rec_flags = 0;
2913 	int i;
2914 
2915 	start = ftrace_now(raw_smp_processor_id());
2916 
2917 	/*
2918 	 * When a module is loaded, this function is called to convert
2919 	 * the calls to mcount in its text to nops, and also to create
2920 	 * an entry in the ftrace data. Now, if ftrace is activated
2921 	 * after this call, but before the module sets its text to
2922 	 * read-only, the modification of enabling ftrace can fail if
2923 	 * the read-only is done while ftrace is converting the calls.
2924 	 * To prevent this, the module's records are set as disabled
2925 	 * and will be enabled after the call to set the module's text
2926 	 * to read-only.
2927 	 */
2928 	if (mod)
2929 		rec_flags |= FTRACE_FL_DISABLED;
2930 
2931 	for (pg = new_pgs; pg; pg = pg->next) {
2932 
2933 		for (i = 0; i < pg->index; i++) {
2934 
2935 			/* If something went wrong, bail without enabling anything */
2936 			if (unlikely(ftrace_disabled))
2937 				return -1;
2938 
2939 			p = &pg->records[i];
2940 			p->flags = rec_flags;
2941 
2942 #ifndef CC_USING_NOP_MCOUNT
2943 			/*
2944 			 * Do the initial record conversion from mcount jump
2945 			 * to the NOP instructions.
2946 			 */
2947 			if (!ftrace_code_disable(mod, p))
2948 				break;
2949 #endif
2950 
2951 			update_cnt++;
2952 		}
2953 	}
2954 
2955 	stop = ftrace_now(raw_smp_processor_id());
2956 	ftrace_update_time = stop - start;
2957 	ftrace_update_tot_cnt += update_cnt;
2958 
2959 	return 0;
2960 }
2961 
2962 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2963 {
2964 	int order;
2965 	int cnt;
2966 
2967 	if (WARN_ON(!count))
2968 		return -EINVAL;
2969 
2970 	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2971 
2972 	/*
2973 	 * We want to fill as much as possible. No more than a page
2974 	 * may be empty.
2975 	 */
2976 	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2977 		order--;
2978 
2979  again:
2980 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2981 
2982 	if (!pg->records) {
2983 		/* if we can't allocate this size, try something smaller */
2984 		if (!order)
2985 			return -ENOMEM;
2986 		order >>= 1;
2987 		goto again;
2988 	}
2989 
2990 	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2991 	pg->size = cnt;
2992 
2993 	if (cnt > count)
2994 		cnt = count;
2995 
2996 	return cnt;
2997 }
2998 
2999 static struct ftrace_page *
3000 ftrace_allocate_pages(unsigned long num_to_init)
3001 {
3002 	struct ftrace_page *start_pg;
3003 	struct ftrace_page *pg;
3004 	int order;
3005 	int cnt;
3006 
3007 	if (!num_to_init)
3008 		return NULL;
3009 
3010 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
3011 	if (!pg)
3012 		return NULL;
3013 
3014 	/*
3015 	 * Try to allocate as much as possible in one continues
3016 	 * location that fills in all of the space. We want to
3017 	 * waste as little space as possible.
3018 	 */
3019 	for (;;) {
3020 		cnt = ftrace_allocate_records(pg, num_to_init);
3021 		if (cnt < 0)
3022 			goto free_pages;
3023 
3024 		num_to_init -= cnt;
3025 		if (!num_to_init)
3026 			break;
3027 
3028 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
3029 		if (!pg->next)
3030 			goto free_pages;
3031 
3032 		pg = pg->next;
3033 	}
3034 
3035 	return start_pg;
3036 
3037  free_pages:
3038 	pg = start_pg;
3039 	while (pg) {
3040 		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
3041 		free_pages((unsigned long)pg->records, order);
3042 		start_pg = pg->next;
3043 		kfree(pg);
3044 		pg = start_pg;
3045 	}
3046 	pr_info("ftrace: FAILED to allocate memory for functions\n");
3047 	return NULL;
3048 }
3049 
3050 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
3051 
3052 struct ftrace_iterator {
3053 	loff_t				pos;
3054 	loff_t				func_pos;
3055 	loff_t				mod_pos;
3056 	struct ftrace_page		*pg;
3057 	struct dyn_ftrace		*func;
3058 	struct ftrace_func_probe	*probe;
3059 	struct ftrace_func_entry	*probe_entry;
3060 	struct trace_parser		parser;
3061 	struct ftrace_hash		*hash;
3062 	struct ftrace_ops		*ops;
3063 	struct trace_array		*tr;
3064 	struct list_head		*mod_list;
3065 	int				pidx;
3066 	int				idx;
3067 	unsigned			flags;
3068 };
3069 
3070 static void *
3071 t_probe_next(struct seq_file *m, loff_t *pos)
3072 {
3073 	struct ftrace_iterator *iter = m->private;
3074 	struct trace_array *tr = iter->ops->private;
3075 	struct list_head *func_probes;
3076 	struct ftrace_hash *hash;
3077 	struct list_head *next;
3078 	struct hlist_node *hnd = NULL;
3079 	struct hlist_head *hhd;
3080 	int size;
3081 
3082 	(*pos)++;
3083 	iter->pos = *pos;
3084 
3085 	if (!tr)
3086 		return NULL;
3087 
3088 	func_probes = &tr->func_probes;
3089 	if (list_empty(func_probes))
3090 		return NULL;
3091 
3092 	if (!iter->probe) {
3093 		next = func_probes->next;
3094 		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3095 	}
3096 
3097 	if (iter->probe_entry)
3098 		hnd = &iter->probe_entry->hlist;
3099 
3100 	hash = iter->probe->ops.func_hash->filter_hash;
3101 	size = 1 << hash->size_bits;
3102 
3103  retry:
3104 	if (iter->pidx >= size) {
3105 		if (iter->probe->list.next == func_probes)
3106 			return NULL;
3107 		next = iter->probe->list.next;
3108 		iter->probe = list_entry(next, struct ftrace_func_probe, list);
3109 		hash = iter->probe->ops.func_hash->filter_hash;
3110 		size = 1 << hash->size_bits;
3111 		iter->pidx = 0;
3112 	}
3113 
3114 	hhd = &hash->buckets[iter->pidx];
3115 
3116 	if (hlist_empty(hhd)) {
3117 		iter->pidx++;
3118 		hnd = NULL;
3119 		goto retry;
3120 	}
3121 
3122 	if (!hnd)
3123 		hnd = hhd->first;
3124 	else {
3125 		hnd = hnd->next;
3126 		if (!hnd) {
3127 			iter->pidx++;
3128 			goto retry;
3129 		}
3130 	}
3131 
3132 	if (WARN_ON_ONCE(!hnd))
3133 		return NULL;
3134 
3135 	iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
3136 
3137 	return iter;
3138 }
3139 
3140 static void *t_probe_start(struct seq_file *m, loff_t *pos)
3141 {
3142 	struct ftrace_iterator *iter = m->private;
3143 	void *p = NULL;
3144 	loff_t l;
3145 
3146 	if (!(iter->flags & FTRACE_ITER_DO_PROBES))
3147 		return NULL;
3148 
3149 	if (iter->mod_pos > *pos)
3150 		return NULL;
3151 
3152 	iter->probe = NULL;
3153 	iter->probe_entry = NULL;
3154 	iter->pidx = 0;
3155 	for (l = 0; l <= (*pos - iter->mod_pos); ) {
3156 		p = t_probe_next(m, &l);
3157 		if (!p)
3158 			break;
3159 	}
3160 	if (!p)
3161 		return NULL;
3162 
3163 	/* Only set this if we have an item */
3164 	iter->flags |= FTRACE_ITER_PROBE;
3165 
3166 	return iter;
3167 }
3168 
3169 static int
3170 t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
3171 {
3172 	struct ftrace_func_entry *probe_entry;
3173 	struct ftrace_probe_ops *probe_ops;
3174 	struct ftrace_func_probe *probe;
3175 
3176 	probe = iter->probe;
3177 	probe_entry = iter->probe_entry;
3178 
3179 	if (WARN_ON_ONCE(!probe || !probe_entry))
3180 		return -EIO;
3181 
3182 	probe_ops = probe->probe_ops;
3183 
3184 	if (probe_ops->print)
3185 		return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
3186 
3187 	seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
3188 		   (void *)probe_ops->func);
3189 
3190 	return 0;
3191 }
3192 
3193 static void *
3194 t_mod_next(struct seq_file *m, loff_t *pos)
3195 {
3196 	struct ftrace_iterator *iter = m->private;
3197 	struct trace_array *tr = iter->tr;
3198 
3199 	(*pos)++;
3200 	iter->pos = *pos;
3201 
3202 	iter->mod_list = iter->mod_list->next;
3203 
3204 	if (iter->mod_list == &tr->mod_trace ||
3205 	    iter->mod_list == &tr->mod_notrace) {
3206 		iter->flags &= ~FTRACE_ITER_MOD;
3207 		return NULL;
3208 	}
3209 
3210 	iter->mod_pos = *pos;
3211 
3212 	return iter;
3213 }
3214 
3215 static void *t_mod_start(struct seq_file *m, loff_t *pos)
3216 {
3217 	struct ftrace_iterator *iter = m->private;
3218 	void *p = NULL;
3219 	loff_t l;
3220 
3221 	if (iter->func_pos > *pos)
3222 		return NULL;
3223 
3224 	iter->mod_pos = iter->func_pos;
3225 
3226 	/* probes are only available if tr is set */
3227 	if (!iter->tr)
3228 		return NULL;
3229 
3230 	for (l = 0; l <= (*pos - iter->func_pos); ) {
3231 		p = t_mod_next(m, &l);
3232 		if (!p)
3233 			break;
3234 	}
3235 	if (!p) {
3236 		iter->flags &= ~FTRACE_ITER_MOD;
3237 		return t_probe_start(m, pos);
3238 	}
3239 
3240 	/* Only set this if we have an item */
3241 	iter->flags |= FTRACE_ITER_MOD;
3242 
3243 	return iter;
3244 }
3245 
3246 static int
3247 t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
3248 {
3249 	struct ftrace_mod_load *ftrace_mod;
3250 	struct trace_array *tr = iter->tr;
3251 
3252 	if (WARN_ON_ONCE(!iter->mod_list) ||
3253 			 iter->mod_list == &tr->mod_trace ||
3254 			 iter->mod_list == &tr->mod_notrace)
3255 		return -EIO;
3256 
3257 	ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
3258 
3259 	if (ftrace_mod->func)
3260 		seq_printf(m, "%s", ftrace_mod->func);
3261 	else
3262 		seq_putc(m, '*');
3263 
3264 	seq_printf(m, ":mod:%s\n", ftrace_mod->module);
3265 
3266 	return 0;
3267 }
3268 
3269 static void *
3270 t_func_next(struct seq_file *m, loff_t *pos)
3271 {
3272 	struct ftrace_iterator *iter = m->private;
3273 	struct dyn_ftrace *rec = NULL;
3274 
3275 	(*pos)++;
3276 
3277  retry:
3278 	if (iter->idx >= iter->pg->index) {
3279 		if (iter->pg->next) {
3280 			iter->pg = iter->pg->next;
3281 			iter->idx = 0;
3282 			goto retry;
3283 		}
3284 	} else {
3285 		rec = &iter->pg->records[iter->idx++];
3286 		if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3287 		     !ftrace_lookup_ip(iter->hash, rec->ip)) ||
3288 
3289 		    ((iter->flags & FTRACE_ITER_ENABLED) &&
3290 		     !(rec->flags & FTRACE_FL_ENABLED))) {
3291 
3292 			rec = NULL;
3293 			goto retry;
3294 		}
3295 	}
3296 
3297 	if (!rec)
3298 		return NULL;
3299 
3300 	iter->pos = iter->func_pos = *pos;
3301 	iter->func = rec;
3302 
3303 	return iter;
3304 }
3305 
3306 static void *
3307 t_next(struct seq_file *m, void *v, loff_t *pos)
3308 {
3309 	struct ftrace_iterator *iter = m->private;
3310 	loff_t l = *pos; /* t_probe_start() must use original pos */
3311 	void *ret;
3312 
3313 	if (unlikely(ftrace_disabled))
3314 		return NULL;
3315 
3316 	if (iter->flags & FTRACE_ITER_PROBE)
3317 		return t_probe_next(m, pos);
3318 
3319 	if (iter->flags & FTRACE_ITER_MOD)
3320 		return t_mod_next(m, pos);
3321 
3322 	if (iter->flags & FTRACE_ITER_PRINTALL) {
3323 		/* next must increment pos, and t_probe_start does not */
3324 		(*pos)++;
3325 		return t_mod_start(m, &l);
3326 	}
3327 
3328 	ret = t_func_next(m, pos);
3329 
3330 	if (!ret)
3331 		return t_mod_start(m, &l);
3332 
3333 	return ret;
3334 }
3335 
3336 static void reset_iter_read(struct ftrace_iterator *iter)
3337 {
3338 	iter->pos = 0;
3339 	iter->func_pos = 0;
3340 	iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
3341 }
3342 
3343 static void *t_start(struct seq_file *m, loff_t *pos)
3344 {
3345 	struct ftrace_iterator *iter = m->private;
3346 	void *p = NULL;
3347 	loff_t l;
3348 
3349 	mutex_lock(&ftrace_lock);
3350 
3351 	if (unlikely(ftrace_disabled))
3352 		return NULL;
3353 
3354 	/*
3355 	 * If an lseek was done, then reset and start from beginning.
3356 	 */
3357 	if (*pos < iter->pos)
3358 		reset_iter_read(iter);
3359 
3360 	/*
3361 	 * For set_ftrace_filter reading, if we have the filter
3362 	 * off, we can short cut and just print out that all
3363 	 * functions are enabled.
3364 	 */
3365 	if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
3366 	    ftrace_hash_empty(iter->hash)) {
3367 		iter->func_pos = 1; /* Account for the message */
3368 		if (*pos > 0)
3369 			return t_mod_start(m, pos);
3370 		iter->flags |= FTRACE_ITER_PRINTALL;
3371 		/* reset in case of seek/pread */
3372 		iter->flags &= ~FTRACE_ITER_PROBE;
3373 		return iter;
3374 	}
3375 
3376 	if (iter->flags & FTRACE_ITER_MOD)
3377 		return t_mod_start(m, pos);
3378 
3379 	/*
3380 	 * Unfortunately, we need to restart at ftrace_pages_start
3381 	 * every time we let go of the ftrace_mutex. This is because
3382 	 * those pointers can change without the lock.
3383 	 */
3384 	iter->pg = ftrace_pages_start;
3385 	iter->idx = 0;
3386 	for (l = 0; l <= *pos; ) {
3387 		p = t_func_next(m, &l);
3388 		if (!p)
3389 			break;
3390 	}
3391 
3392 	if (!p)
3393 		return t_mod_start(m, pos);
3394 
3395 	return iter;
3396 }
3397 
3398 static void t_stop(struct seq_file *m, void *p)
3399 {
3400 	mutex_unlock(&ftrace_lock);
3401 }
3402 
3403 void * __weak
3404 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
3405 {
3406 	return NULL;
3407 }
3408 
3409 static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
3410 				struct dyn_ftrace *rec)
3411 {
3412 	void *ptr;
3413 
3414 	ptr = arch_ftrace_trampoline_func(ops, rec);
3415 	if (ptr)
3416 		seq_printf(m, " ->%pS", ptr);
3417 }
3418 
3419 static int t_show(struct seq_file *m, void *v)
3420 {
3421 	struct ftrace_iterator *iter = m->private;
3422 	struct dyn_ftrace *rec;
3423 
3424 	if (iter->flags & FTRACE_ITER_PROBE)
3425 		return t_probe_show(m, iter);
3426 
3427 	if (iter->flags & FTRACE_ITER_MOD)
3428 		return t_mod_show(m, iter);
3429 
3430 	if (iter->flags & FTRACE_ITER_PRINTALL) {
3431 		if (iter->flags & FTRACE_ITER_NOTRACE)
3432 			seq_puts(m, "#### no functions disabled ####\n");
3433 		else
3434 			seq_puts(m, "#### all functions enabled ####\n");
3435 		return 0;
3436 	}
3437 
3438 	rec = iter->func;
3439 
3440 	if (!rec)
3441 		return 0;
3442 
3443 	seq_printf(m, "%ps", (void *)rec->ip);
3444 	if (iter->flags & FTRACE_ITER_ENABLED) {
3445 		struct ftrace_ops *ops;
3446 
3447 		seq_printf(m, " (%ld)%s%s",
3448 			   ftrace_rec_count(rec),
3449 			   rec->flags & FTRACE_FL_REGS ? " R" : "  ",
3450 			   rec->flags & FTRACE_FL_IPMODIFY ? " I" : "  ");
3451 		if (rec->flags & FTRACE_FL_TRAMP_EN) {
3452 			ops = ftrace_find_tramp_ops_any(rec);
3453 			if (ops) {
3454 				do {
3455 					seq_printf(m, "\ttramp: %pS (%pS)",
3456 						   (void *)ops->trampoline,
3457 						   (void *)ops->func);
3458 					add_trampoline_func(m, ops, rec);
3459 					ops = ftrace_find_tramp_ops_next(rec, ops);
3460 				} while (ops);
3461 			} else
3462 				seq_puts(m, "\ttramp: ERROR!");
3463 		} else {
3464 			add_trampoline_func(m, NULL, rec);
3465 		}
3466 	}
3467 
3468 	seq_putc(m, '\n');
3469 
3470 	return 0;
3471 }
3472 
3473 static const struct seq_operations show_ftrace_seq_ops = {
3474 	.start = t_start,
3475 	.next = t_next,
3476 	.stop = t_stop,
3477 	.show = t_show,
3478 };
3479 
3480 static int
3481 ftrace_avail_open(struct inode *inode, struct file *file)
3482 {
3483 	struct ftrace_iterator *iter;
3484 
3485 	if (unlikely(ftrace_disabled))
3486 		return -ENODEV;
3487 
3488 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3489 	if (!iter)
3490 		return -ENOMEM;
3491 
3492 	iter->pg = ftrace_pages_start;
3493 	iter->ops = &global_ops;
3494 
3495 	return 0;
3496 }
3497 
3498 static int
3499 ftrace_enabled_open(struct inode *inode, struct file *file)
3500 {
3501 	struct ftrace_iterator *iter;
3502 
3503 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
3504 	if (!iter)
3505 		return -ENOMEM;
3506 
3507 	iter->pg = ftrace_pages_start;
3508 	iter->flags = FTRACE_ITER_ENABLED;
3509 	iter->ops = &global_ops;
3510 
3511 	return 0;
3512 }
3513 
3514 /**
3515  * ftrace_regex_open - initialize function tracer filter files
3516  * @ops: The ftrace_ops that hold the hash filters
3517  * @flag: The type of filter to process
3518  * @inode: The inode, usually passed in to your open routine
3519  * @file: The file, usually passed in to your open routine
3520  *
3521  * ftrace_regex_open() initializes the filter files for the
3522  * @ops. Depending on @flag it may process the filter hash or
3523  * the notrace hash of @ops. With this called from the open
3524  * routine, you can use ftrace_filter_write() for the write
3525  * routine if @flag has FTRACE_ITER_FILTER set, or
3526  * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
3527  * tracing_lseek() should be used as the lseek routine, and
3528  * release must call ftrace_regex_release().
3529  */
3530 int
3531 ftrace_regex_open(struct ftrace_ops *ops, int flag,
3532 		  struct inode *inode, struct file *file)
3533 {
3534 	struct ftrace_iterator *iter;
3535 	struct ftrace_hash *hash;
3536 	struct list_head *mod_head;
3537 	struct trace_array *tr = ops->private;
3538 	int ret = 0;
3539 
3540 	ftrace_ops_init(ops);
3541 
3542 	if (unlikely(ftrace_disabled))
3543 		return -ENODEV;
3544 
3545 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3546 	if (!iter)
3547 		return -ENOMEM;
3548 
3549 	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
3550 		kfree(iter);
3551 		return -ENOMEM;
3552 	}
3553 
3554 	iter->ops = ops;
3555 	iter->flags = flag;
3556 	iter->tr = tr;
3557 
3558 	mutex_lock(&ops->func_hash->regex_lock);
3559 
3560 	if (flag & FTRACE_ITER_NOTRACE) {
3561 		hash = ops->func_hash->notrace_hash;
3562 		mod_head = tr ? &tr->mod_notrace : NULL;
3563 	} else {
3564 		hash = ops->func_hash->filter_hash;
3565 		mod_head = tr ? &tr->mod_trace : NULL;
3566 	}
3567 
3568 	iter->mod_list = mod_head;
3569 
3570 	if (file->f_mode & FMODE_WRITE) {
3571 		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
3572 
3573 		if (file->f_flags & O_TRUNC) {
3574 			iter->hash = alloc_ftrace_hash(size_bits);
3575 			clear_ftrace_mod_list(mod_head);
3576 	        } else {
3577 			iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
3578 		}
3579 
3580 		if (!iter->hash) {
3581 			trace_parser_put(&iter->parser);
3582 			kfree(iter);
3583 			ret = -ENOMEM;
3584 			goto out_unlock;
3585 		}
3586 	} else
3587 		iter->hash = hash;
3588 
3589 	if (file->f_mode & FMODE_READ) {
3590 		iter->pg = ftrace_pages_start;
3591 
3592 		ret = seq_open(file, &show_ftrace_seq_ops);
3593 		if (!ret) {
3594 			struct seq_file *m = file->private_data;
3595 			m->private = iter;
3596 		} else {
3597 			/* Failed */
3598 			free_ftrace_hash(iter->hash);
3599 			trace_parser_put(&iter->parser);
3600 			kfree(iter);
3601 		}
3602 	} else
3603 		file->private_data = iter;
3604 
3605  out_unlock:
3606 	mutex_unlock(&ops->func_hash->regex_lock);
3607 
3608 	return ret;
3609 }
3610 
3611 static int
3612 ftrace_filter_open(struct inode *inode, struct file *file)
3613 {
3614 	struct ftrace_ops *ops = inode->i_private;
3615 
3616 	return ftrace_regex_open(ops,
3617 			FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
3618 			inode, file);
3619 }
3620 
3621 static int
3622 ftrace_notrace_open(struct inode *inode, struct file *file)
3623 {
3624 	struct ftrace_ops *ops = inode->i_private;
3625 
3626 	return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
3627 				 inode, file);
3628 }
3629 
3630 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
3631 struct ftrace_glob {
3632 	char *search;
3633 	unsigned len;
3634 	int type;
3635 };
3636 
3637 /*
3638  * If symbols in an architecture don't correspond exactly to the user-visible
3639  * name of what they represent, it is possible to define this function to
3640  * perform the necessary adjustments.
3641 */
3642 char * __weak arch_ftrace_match_adjust(char *str, const char *search)
3643 {
3644 	return str;
3645 }
3646 
3647 static int ftrace_match(char *str, struct ftrace_glob *g)
3648 {
3649 	int matched = 0;
3650 	int slen;
3651 
3652 	str = arch_ftrace_match_adjust(str, g->search);
3653 
3654 	switch (g->type) {
3655 	case MATCH_FULL:
3656 		if (strcmp(str, g->search) == 0)
3657 			matched = 1;
3658 		break;
3659 	case MATCH_FRONT_ONLY:
3660 		if (strncmp(str, g->search, g->len) == 0)
3661 			matched = 1;
3662 		break;
3663 	case MATCH_MIDDLE_ONLY:
3664 		if (strstr(str, g->search))
3665 			matched = 1;
3666 		break;
3667 	case MATCH_END_ONLY:
3668 		slen = strlen(str);
3669 		if (slen >= g->len &&
3670 		    memcmp(str + slen - g->len, g->search, g->len) == 0)
3671 			matched = 1;
3672 		break;
3673 	case MATCH_GLOB:
3674 		if (glob_match(g->search, str))
3675 			matched = 1;
3676 		break;
3677 	}
3678 
3679 	return matched;
3680 }
3681 
3682 static int
3683 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
3684 {
3685 	struct ftrace_func_entry *entry;
3686 	int ret = 0;
3687 
3688 	entry = ftrace_lookup_ip(hash, rec->ip);
3689 	if (clear_filter) {
3690 		/* Do nothing if it doesn't exist */
3691 		if (!entry)
3692 			return 0;
3693 
3694 		free_hash_entry(hash, entry);
3695 	} else {
3696 		/* Do nothing if it exists */
3697 		if (entry)
3698 			return 0;
3699 
3700 		ret = add_hash_entry(hash, rec->ip);
3701 	}
3702 	return ret;
3703 }
3704 
3705 static int
3706 add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
3707 		 int clear_filter)
3708 {
3709 	long index = simple_strtoul(func_g->search, NULL, 0);
3710 	struct ftrace_page *pg;
3711 	struct dyn_ftrace *rec;
3712 
3713 	/* The index starts at 1 */
3714 	if (--index < 0)
3715 		return 0;
3716 
3717 	do_for_each_ftrace_rec(pg, rec) {
3718 		if (pg->index <= index) {
3719 			index -= pg->index;
3720 			/* this is a double loop, break goes to the next page */
3721 			break;
3722 		}
3723 		rec = &pg->records[index];
3724 		enter_record(hash, rec, clear_filter);
3725 		return 1;
3726 	} while_for_each_ftrace_rec();
3727 	return 0;
3728 }
3729 
3730 static int
3731 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
3732 		struct ftrace_glob *mod_g, int exclude_mod)
3733 {
3734 	char str[KSYM_SYMBOL_LEN];
3735 	char *modname;
3736 
3737 	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
3738 
3739 	if (mod_g) {
3740 		int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
3741 
3742 		/* blank module name to match all modules */
3743 		if (!mod_g->len) {
3744 			/* blank module globbing: modname xor exclude_mod */
3745 			if (!exclude_mod != !modname)
3746 				goto func_match;
3747 			return 0;
3748 		}
3749 
3750 		/*
3751 		 * exclude_mod is set to trace everything but the given
3752 		 * module. If it is set and the module matches, then
3753 		 * return 0. If it is not set, and the module doesn't match
3754 		 * also return 0. Otherwise, check the function to see if
3755 		 * that matches.
3756 		 */
3757 		if (!mod_matches == !exclude_mod)
3758 			return 0;
3759 func_match:
3760 		/* blank search means to match all funcs in the mod */
3761 		if (!func_g->len)
3762 			return 1;
3763 	}
3764 
3765 	return ftrace_match(str, func_g);
3766 }
3767 
3768 static int
3769 match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
3770 {
3771 	struct ftrace_page *pg;
3772 	struct dyn_ftrace *rec;
3773 	struct ftrace_glob func_g = { .type = MATCH_FULL };
3774 	struct ftrace_glob mod_g = { .type = MATCH_FULL };
3775 	struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
3776 	int exclude_mod = 0;
3777 	int found = 0;
3778 	int ret;
3779 	int clear_filter = 0;
3780 
3781 	if (func) {
3782 		func_g.type = filter_parse_regex(func, len, &func_g.search,
3783 						 &clear_filter);
3784 		func_g.len = strlen(func_g.search);
3785 	}
3786 
3787 	if (mod) {
3788 		mod_g.type = filter_parse_regex(mod, strlen(mod),
3789 				&mod_g.search, &exclude_mod);
3790 		mod_g.len = strlen(mod_g.search);
3791 	}
3792 
3793 	mutex_lock(&ftrace_lock);
3794 
3795 	if (unlikely(ftrace_disabled))
3796 		goto out_unlock;
3797 
3798 	if (func_g.type == MATCH_INDEX) {
3799 		found = add_rec_by_index(hash, &func_g, clear_filter);
3800 		goto out_unlock;
3801 	}
3802 
3803 	do_for_each_ftrace_rec(pg, rec) {
3804 
3805 		if (rec->flags & FTRACE_FL_DISABLED)
3806 			continue;
3807 
3808 		if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
3809 			ret = enter_record(hash, rec, clear_filter);
3810 			if (ret < 0) {
3811 				found = ret;
3812 				goto out_unlock;
3813 			}
3814 			found = 1;
3815 		}
3816 	} while_for_each_ftrace_rec();
3817  out_unlock:
3818 	mutex_unlock(&ftrace_lock);
3819 
3820 	return found;
3821 }
3822 
3823 static int
3824 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
3825 {
3826 	return match_records(hash, buff, len, NULL);
3827 }
3828 
3829 static void ftrace_ops_update_code(struct ftrace_ops *ops,
3830 				   struct ftrace_ops_hash *old_hash)
3831 {
3832 	struct ftrace_ops *op;
3833 
3834 	if (!ftrace_enabled)
3835 		return;
3836 
3837 	if (ops->flags & FTRACE_OPS_FL_ENABLED) {
3838 		ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
3839 		return;
3840 	}
3841 
3842 	/*
3843 	 * If this is the shared global_ops filter, then we need to
3844 	 * check if there is another ops that shares it, is enabled.
3845 	 * If so, we still need to run the modify code.
3846 	 */
3847 	if (ops->func_hash != &global_ops.local_hash)
3848 		return;
3849 
3850 	do_for_each_ftrace_op(op, ftrace_ops_list) {
3851 		if (op->func_hash == &global_ops.local_hash &&
3852 		    op->flags & FTRACE_OPS_FL_ENABLED) {
3853 			ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
3854 			/* Only need to do this once */
3855 			return;
3856 		}
3857 	} while_for_each_ftrace_op(op);
3858 }
3859 
3860 static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
3861 					   struct ftrace_hash **orig_hash,
3862 					   struct ftrace_hash *hash,
3863 					   int enable)
3864 {
3865 	struct ftrace_ops_hash old_hash_ops;
3866 	struct ftrace_hash *old_hash;
3867 	int ret;
3868 
3869 	old_hash = *orig_hash;
3870 	old_hash_ops.filter_hash = ops->func_hash->filter_hash;
3871 	old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
3872 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3873 	if (!ret) {
3874 		ftrace_ops_update_code(ops, &old_hash_ops);
3875 		free_ftrace_hash_rcu(old_hash);
3876 	}
3877 	return ret;
3878 }
3879 
3880 static bool module_exists(const char *module)
3881 {
3882 	/* All modules have the symbol __this_module */
3883 	const char this_mod[] = "__this_module";
3884 	char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
3885 	unsigned long val;
3886 	int n;
3887 
3888 	n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
3889 
3890 	if (n > sizeof(modname) - 1)
3891 		return false;
3892 
3893 	val = module_kallsyms_lookup_name(modname);
3894 	return val != 0;
3895 }
3896 
3897 static int cache_mod(struct trace_array *tr,
3898 		     const char *func, char *module, int enable)
3899 {
3900 	struct ftrace_mod_load *ftrace_mod, *n;
3901 	struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
3902 	int ret;
3903 
3904 	mutex_lock(&ftrace_lock);
3905 
3906 	/* We do not cache inverse filters */
3907 	if (func[0] == '!') {
3908 		func++;
3909 		ret = -EINVAL;
3910 
3911 		/* Look to remove this hash */
3912 		list_for_each_entry_safe(ftrace_mod, n, head, list) {
3913 			if (strcmp(ftrace_mod->module, module) != 0)
3914 				continue;
3915 
3916 			/* no func matches all */
3917 			if (strcmp(func, "*") == 0 ||
3918 			    (ftrace_mod->func &&
3919 			     strcmp(ftrace_mod->func, func) == 0)) {
3920 				ret = 0;
3921 				free_ftrace_mod(ftrace_mod);
3922 				continue;
3923 			}
3924 		}
3925 		goto out;
3926 	}
3927 
3928 	ret = -EINVAL;
3929 	/* We only care about modules that have not been loaded yet */
3930 	if (module_exists(module))
3931 		goto out;
3932 
3933 	/* Save this string off, and execute it when the module is loaded */
3934 	ret = ftrace_add_mod(tr, func, module, enable);
3935  out:
3936 	mutex_unlock(&ftrace_lock);
3937 
3938 	return ret;
3939 }
3940 
3941 static int
3942 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
3943 		 int reset, int enable);
3944 
3945 #ifdef CONFIG_MODULES
3946 static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
3947 			     char *mod, bool enable)
3948 {
3949 	struct ftrace_mod_load *ftrace_mod, *n;
3950 	struct ftrace_hash **orig_hash, *new_hash;
3951 	LIST_HEAD(process_mods);
3952 	char *func;
3953 	int ret;
3954 
3955 	mutex_lock(&ops->func_hash->regex_lock);
3956 
3957 	if (enable)
3958 		orig_hash = &ops->func_hash->filter_hash;
3959 	else
3960 		orig_hash = &ops->func_hash->notrace_hash;
3961 
3962 	new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
3963 					      *orig_hash);
3964 	if (!new_hash)
3965 		goto out; /* warn? */
3966 
3967 	mutex_lock(&ftrace_lock);
3968 
3969 	list_for_each_entry_safe(ftrace_mod, n, head, list) {
3970 
3971 		if (strcmp(ftrace_mod->module, mod) != 0)
3972 			continue;
3973 
3974 		if (ftrace_mod->func)
3975 			func = kstrdup(ftrace_mod->func, GFP_KERNEL);
3976 		else
3977 			func = kstrdup("*", GFP_KERNEL);
3978 
3979 		if (!func) /* warn? */
3980 			continue;
3981 
3982 		list_del(&ftrace_mod->list);
3983 		list_add(&ftrace_mod->list, &process_mods);
3984 
3985 		/* Use the newly allocated func, as it may be "*" */
3986 		kfree(ftrace_mod->func);
3987 		ftrace_mod->func = func;
3988 	}
3989 
3990 	mutex_unlock(&ftrace_lock);
3991 
3992 	list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
3993 
3994 		func = ftrace_mod->func;
3995 
3996 		/* Grabs ftrace_lock, which is why we have this extra step */
3997 		match_records(new_hash, func, strlen(func), mod);
3998 		free_ftrace_mod(ftrace_mod);
3999 	}
4000 
4001 	if (enable && list_empty(head))
4002 		new_hash->flags &= ~FTRACE_HASH_FL_MOD;
4003 
4004 	mutex_lock(&ftrace_lock);
4005 
4006 	ret = ftrace_hash_move_and_update_ops(ops, orig_hash,
4007 					      new_hash, enable);
4008 	mutex_unlock(&ftrace_lock);
4009 
4010  out:
4011 	mutex_unlock(&ops->func_hash->regex_lock);
4012 
4013 	free_ftrace_hash(new_hash);
4014 }
4015 
4016 static void process_cached_mods(const char *mod_name)
4017 {
4018 	struct trace_array *tr;
4019 	char *mod;
4020 
4021 	mod = kstrdup(mod_name, GFP_KERNEL);
4022 	if (!mod)
4023 		return;
4024 
4025 	mutex_lock(&trace_types_lock);
4026 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
4027 		if (!list_empty(&tr->mod_trace))
4028 			process_mod_list(&tr->mod_trace, tr->ops, mod, true);
4029 		if (!list_empty(&tr->mod_notrace))
4030 			process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
4031 	}
4032 	mutex_unlock(&trace_types_lock);
4033 
4034 	kfree(mod);
4035 }
4036 #endif
4037 
4038 /*
4039  * We register the module command as a template to show others how
4040  * to register the a command as well.
4041  */
4042 
4043 static int
4044 ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
4045 		    char *func_orig, char *cmd, char *module, int enable)
4046 {
4047 	char *func;
4048 	int ret;
4049 
4050 	/* match_records() modifies func, and we need the original */
4051 	func = kstrdup(func_orig, GFP_KERNEL);
4052 	if (!func)
4053 		return -ENOMEM;
4054 
4055 	/*
4056 	 * cmd == 'mod' because we only registered this func
4057 	 * for the 'mod' ftrace_func_command.
4058 	 * But if you register one func with multiple commands,
4059 	 * you can tell which command was used by the cmd
4060 	 * parameter.
4061 	 */
4062 	ret = match_records(hash, func, strlen(func), module);
4063 	kfree(func);
4064 
4065 	if (!ret)
4066 		return cache_mod(tr, func_orig, module, enable);
4067 	if (ret < 0)
4068 		return ret;
4069 	return 0;
4070 }
4071 
4072 static struct ftrace_func_command ftrace_mod_cmd = {
4073 	.name			= "mod",
4074 	.func			= ftrace_mod_callback,
4075 };
4076 
4077 static int __init ftrace_mod_cmd_init(void)
4078 {
4079 	return register_ftrace_command(&ftrace_mod_cmd);
4080 }
4081 core_initcall(ftrace_mod_cmd_init);
4082 
4083 static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
4084 				      struct ftrace_ops *op, struct pt_regs *pt_regs)
4085 {
4086 	struct ftrace_probe_ops *probe_ops;
4087 	struct ftrace_func_probe *probe;
4088 
4089 	probe = container_of(op, struct ftrace_func_probe, ops);
4090 	probe_ops = probe->probe_ops;
4091 
4092 	/*
4093 	 * Disable preemption for these calls to prevent a RCU grace
4094 	 * period. This syncs the hash iteration and freeing of items
4095 	 * on the hash. rcu_read_lock is too dangerous here.
4096 	 */
4097 	preempt_disable_notrace();
4098 	probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
4099 	preempt_enable_notrace();
4100 }
4101 
4102 struct ftrace_func_map {
4103 	struct ftrace_func_entry	entry;
4104 	void				*data;
4105 };
4106 
4107 struct ftrace_func_mapper {
4108 	struct ftrace_hash		hash;
4109 };
4110 
4111 /**
4112  * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
4113  *
4114  * Returns a ftrace_func_mapper descriptor that can be used to map ips to data.
4115  */
4116 struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
4117 {
4118 	struct ftrace_hash *hash;
4119 
4120 	/*
4121 	 * The mapper is simply a ftrace_hash, but since the entries
4122 	 * in the hash are not ftrace_func_entry type, we define it
4123 	 * as a separate structure.
4124 	 */
4125 	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4126 	return (struct ftrace_func_mapper *)hash;
4127 }
4128 
4129 /**
4130  * ftrace_func_mapper_find_ip - Find some data mapped to an ip
4131  * @mapper: The mapper that has the ip maps
4132  * @ip: the instruction pointer to find the data for
4133  *
4134  * Returns the data mapped to @ip if found otherwise NULL. The return
4135  * is actually the address of the mapper data pointer. The address is
4136  * returned for use cases where the data is no bigger than a long, and
4137  * the user can use the data pointer as its data instead of having to
4138  * allocate more memory for the reference.
4139  */
4140 void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
4141 				  unsigned long ip)
4142 {
4143 	struct ftrace_func_entry *entry;
4144 	struct ftrace_func_map *map;
4145 
4146 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4147 	if (!entry)
4148 		return NULL;
4149 
4150 	map = (struct ftrace_func_map *)entry;
4151 	return &map->data;
4152 }
4153 
4154 /**
4155  * ftrace_func_mapper_add_ip - Map some data to an ip
4156  * @mapper: The mapper that has the ip maps
4157  * @ip: The instruction pointer address to map @data to
4158  * @data: The data to map to @ip
4159  *
4160  * Returns 0 on succes otherwise an error.
4161  */
4162 int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
4163 			      unsigned long ip, void *data)
4164 {
4165 	struct ftrace_func_entry *entry;
4166 	struct ftrace_func_map *map;
4167 
4168 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4169 	if (entry)
4170 		return -EBUSY;
4171 
4172 	map = kmalloc(sizeof(*map), GFP_KERNEL);
4173 	if (!map)
4174 		return -ENOMEM;
4175 
4176 	map->entry.ip = ip;
4177 	map->data = data;
4178 
4179 	__add_hash_entry(&mapper->hash, &map->entry);
4180 
4181 	return 0;
4182 }
4183 
4184 /**
4185  * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
4186  * @mapper: The mapper that has the ip maps
4187  * @ip: The instruction pointer address to remove the data from
4188  *
4189  * Returns the data if it is found, otherwise NULL.
4190  * Note, if the data pointer is used as the data itself, (see
4191  * ftrace_func_mapper_find_ip(), then the return value may be meaningless,
4192  * if the data pointer was set to zero.
4193  */
4194 void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
4195 				   unsigned long ip)
4196 {
4197 	struct ftrace_func_entry *entry;
4198 	struct ftrace_func_map *map;
4199 	void *data;
4200 
4201 	entry = ftrace_lookup_ip(&mapper->hash, ip);
4202 	if (!entry)
4203 		return NULL;
4204 
4205 	map = (struct ftrace_func_map *)entry;
4206 	data = map->data;
4207 
4208 	remove_hash_entry(&mapper->hash, entry);
4209 	kfree(entry);
4210 
4211 	return data;
4212 }
4213 
4214 /**
4215  * free_ftrace_func_mapper - free a mapping of ips and data
4216  * @mapper: The mapper that has the ip maps
4217  * @free_func: A function to be called on each data item.
4218  *
4219  * This is used to free the function mapper. The @free_func is optional
4220  * and can be used if the data needs to be freed as well.
4221  */
4222 void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
4223 			     ftrace_mapper_func free_func)
4224 {
4225 	struct ftrace_func_entry *entry;
4226 	struct ftrace_func_map *map;
4227 	struct hlist_head *hhd;
4228 	int size = 1 << mapper->hash.size_bits;
4229 	int i;
4230 
4231 	if (free_func && mapper->hash.count) {
4232 		for (i = 0; i < size; i++) {
4233 			hhd = &mapper->hash.buckets[i];
4234 			hlist_for_each_entry(entry, hhd, hlist) {
4235 				map = (struct ftrace_func_map *)entry;
4236 				free_func(map);
4237 			}
4238 		}
4239 	}
4240 	free_ftrace_hash(&mapper->hash);
4241 }
4242 
4243 static void release_probe(struct ftrace_func_probe *probe)
4244 {
4245 	struct ftrace_probe_ops *probe_ops;
4246 
4247 	mutex_lock(&ftrace_lock);
4248 
4249 	WARN_ON(probe->ref <= 0);
4250 
4251 	/* Subtract the ref that was used to protect this instance */
4252 	probe->ref--;
4253 
4254 	if (!probe->ref) {
4255 		probe_ops = probe->probe_ops;
4256 		/*
4257 		 * Sending zero as ip tells probe_ops to free
4258 		 * the probe->data itself
4259 		 */
4260 		if (probe_ops->free)
4261 			probe_ops->free(probe_ops, probe->tr, 0, probe->data);
4262 		list_del(&probe->list);
4263 		kfree(probe);
4264 	}
4265 	mutex_unlock(&ftrace_lock);
4266 }
4267 
4268 static void acquire_probe_locked(struct ftrace_func_probe *probe)
4269 {
4270 	/*
4271 	 * Add one ref to keep it from being freed when releasing the
4272 	 * ftrace_lock mutex.
4273 	 */
4274 	probe->ref++;
4275 }
4276 
4277 int
4278 register_ftrace_function_probe(char *glob, struct trace_array *tr,
4279 			       struct ftrace_probe_ops *probe_ops,
4280 			       void *data)
4281 {
4282 	struct ftrace_func_entry *entry;
4283 	struct ftrace_func_probe *probe;
4284 	struct ftrace_hash **orig_hash;
4285 	struct ftrace_hash *old_hash;
4286 	struct ftrace_hash *hash;
4287 	int count = 0;
4288 	int size;
4289 	int ret;
4290 	int i;
4291 
4292 	if (WARN_ON(!tr))
4293 		return -EINVAL;
4294 
4295 	/* We do not support '!' for function probes */
4296 	if (WARN_ON(glob[0] == '!'))
4297 		return -EINVAL;
4298 
4299 
4300 	mutex_lock(&ftrace_lock);
4301 	/* Check if the probe_ops is already registered */
4302 	list_for_each_entry(probe, &tr->func_probes, list) {
4303 		if (probe->probe_ops == probe_ops)
4304 			break;
4305 	}
4306 	if (&probe->list == &tr->func_probes) {
4307 		probe = kzalloc(sizeof(*probe), GFP_KERNEL);
4308 		if (!probe) {
4309 			mutex_unlock(&ftrace_lock);
4310 			return -ENOMEM;
4311 		}
4312 		probe->probe_ops = probe_ops;
4313 		probe->ops.func = function_trace_probe_call;
4314 		probe->tr = tr;
4315 		ftrace_ops_init(&probe->ops);
4316 		list_add(&probe->list, &tr->func_probes);
4317 	}
4318 
4319 	acquire_probe_locked(probe);
4320 
4321 	mutex_unlock(&ftrace_lock);
4322 
4323 	mutex_lock(&probe->ops.func_hash->regex_lock);
4324 
4325 	orig_hash = &probe->ops.func_hash->filter_hash;
4326 	old_hash = *orig_hash;
4327 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4328 
4329 	ret = ftrace_match_records(hash, glob, strlen(glob));
4330 
4331 	/* Nothing found? */
4332 	if (!ret)
4333 		ret = -EINVAL;
4334 
4335 	if (ret < 0)
4336 		goto out;
4337 
4338 	size = 1 << hash->size_bits;
4339 	for (i = 0; i < size; i++) {
4340 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4341 			if (ftrace_lookup_ip(old_hash, entry->ip))
4342 				continue;
4343 			/*
4344 			 * The caller might want to do something special
4345 			 * for each function we find. We call the callback
4346 			 * to give the caller an opportunity to do so.
4347 			 */
4348 			if (probe_ops->init) {
4349 				ret = probe_ops->init(probe_ops, tr,
4350 						      entry->ip, data,
4351 						      &probe->data);
4352 				if (ret < 0) {
4353 					if (probe_ops->free && count)
4354 						probe_ops->free(probe_ops, tr,
4355 								0, probe->data);
4356 					probe->data = NULL;
4357 					goto out;
4358 				}
4359 			}
4360 			count++;
4361 		}
4362 	}
4363 
4364 	mutex_lock(&ftrace_lock);
4365 
4366 	if (!count) {
4367 		/* Nothing was added? */
4368 		ret = -EINVAL;
4369 		goto out_unlock;
4370 	}
4371 
4372 	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4373 					      hash, 1);
4374 	if (ret < 0)
4375 		goto err_unlock;
4376 
4377 	/* One ref for each new function traced */
4378 	probe->ref += count;
4379 
4380 	if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
4381 		ret = ftrace_startup(&probe->ops, 0);
4382 
4383  out_unlock:
4384 	mutex_unlock(&ftrace_lock);
4385 
4386 	if (!ret)
4387 		ret = count;
4388  out:
4389 	mutex_unlock(&probe->ops.func_hash->regex_lock);
4390 	free_ftrace_hash(hash);
4391 
4392 	release_probe(probe);
4393 
4394 	return ret;
4395 
4396  err_unlock:
4397 	if (!probe_ops->free || !count)
4398 		goto out_unlock;
4399 
4400 	/* Failed to do the move, need to call the free functions */
4401 	for (i = 0; i < size; i++) {
4402 		hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
4403 			if (ftrace_lookup_ip(old_hash, entry->ip))
4404 				continue;
4405 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4406 		}
4407 	}
4408 	goto out_unlock;
4409 }
4410 
4411 int
4412 unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
4413 				      struct ftrace_probe_ops *probe_ops)
4414 {
4415 	struct ftrace_ops_hash old_hash_ops;
4416 	struct ftrace_func_entry *entry;
4417 	struct ftrace_func_probe *probe;
4418 	struct ftrace_glob func_g;
4419 	struct ftrace_hash **orig_hash;
4420 	struct ftrace_hash *old_hash;
4421 	struct ftrace_hash *hash = NULL;
4422 	struct hlist_node *tmp;
4423 	struct hlist_head hhd;
4424 	char str[KSYM_SYMBOL_LEN];
4425 	int count = 0;
4426 	int i, ret = -ENODEV;
4427 	int size;
4428 
4429 	if (!glob || !strlen(glob) || !strcmp(glob, "*"))
4430 		func_g.search = NULL;
4431 	else {
4432 		int not;
4433 
4434 		func_g.type = filter_parse_regex(glob, strlen(glob),
4435 						 &func_g.search, &not);
4436 		func_g.len = strlen(func_g.search);
4437 
4438 		/* we do not support '!' for function probes */
4439 		if (WARN_ON(not))
4440 			return -EINVAL;
4441 	}
4442 
4443 	mutex_lock(&ftrace_lock);
4444 	/* Check if the probe_ops is already registered */
4445 	list_for_each_entry(probe, &tr->func_probes, list) {
4446 		if (probe->probe_ops == probe_ops)
4447 			break;
4448 	}
4449 	if (&probe->list == &tr->func_probes)
4450 		goto err_unlock_ftrace;
4451 
4452 	ret = -EINVAL;
4453 	if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
4454 		goto err_unlock_ftrace;
4455 
4456 	acquire_probe_locked(probe);
4457 
4458 	mutex_unlock(&ftrace_lock);
4459 
4460 	mutex_lock(&probe->ops.func_hash->regex_lock);
4461 
4462 	orig_hash = &probe->ops.func_hash->filter_hash;
4463 	old_hash = *orig_hash;
4464 
4465 	if (ftrace_hash_empty(old_hash))
4466 		goto out_unlock;
4467 
4468 	old_hash_ops.filter_hash = old_hash;
4469 	/* Probes only have filters */
4470 	old_hash_ops.notrace_hash = NULL;
4471 
4472 	ret = -ENOMEM;
4473 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
4474 	if (!hash)
4475 		goto out_unlock;
4476 
4477 	INIT_HLIST_HEAD(&hhd);
4478 
4479 	size = 1 << hash->size_bits;
4480 	for (i = 0; i < size; i++) {
4481 		hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
4482 
4483 			if (func_g.search) {
4484 				kallsyms_lookup(entry->ip, NULL, NULL,
4485 						NULL, str);
4486 				if (!ftrace_match(str, &func_g))
4487 					continue;
4488 			}
4489 			count++;
4490 			remove_hash_entry(hash, entry);
4491 			hlist_add_head(&entry->hlist, &hhd);
4492 		}
4493 	}
4494 
4495 	/* Nothing found? */
4496 	if (!count) {
4497 		ret = -EINVAL;
4498 		goto out_unlock;
4499 	}
4500 
4501 	mutex_lock(&ftrace_lock);
4502 
4503 	WARN_ON(probe->ref < count);
4504 
4505 	probe->ref -= count;
4506 
4507 	if (ftrace_hash_empty(hash))
4508 		ftrace_shutdown(&probe->ops, 0);
4509 
4510 	ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
4511 					      hash, 1);
4512 
4513 	/* still need to update the function call sites */
4514 	if (ftrace_enabled && !ftrace_hash_empty(hash))
4515 		ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
4516 				       &old_hash_ops);
4517 	synchronize_rcu();
4518 
4519 	hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
4520 		hlist_del(&entry->hlist);
4521 		if (probe_ops->free)
4522 			probe_ops->free(probe_ops, tr, entry->ip, probe->data);
4523 		kfree(entry);
4524 	}
4525 	mutex_unlock(&ftrace_lock);
4526 
4527  out_unlock:
4528 	mutex_unlock(&probe->ops.func_hash->regex_lock);
4529 	free_ftrace_hash(hash);
4530 
4531 	release_probe(probe);
4532 
4533 	return ret;
4534 
4535  err_unlock_ftrace:
4536 	mutex_unlock(&ftrace_lock);
4537 	return ret;
4538 }
4539 
4540 void clear_ftrace_function_probes(struct trace_array *tr)
4541 {
4542 	struct ftrace_func_probe *probe, *n;
4543 
4544 	list_for_each_entry_safe(probe, n, &tr->func_probes, list)
4545 		unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
4546 }
4547 
4548 static LIST_HEAD(ftrace_commands);
4549 static DEFINE_MUTEX(ftrace_cmd_mutex);
4550 
4551 /*
4552  * Currently we only register ftrace commands from __init, so mark this
4553  * __init too.
4554  */
4555 __init int register_ftrace_command(struct ftrace_func_command *cmd)
4556 {
4557 	struct ftrace_func_command *p;
4558 	int ret = 0;
4559 
4560 	mutex_lock(&ftrace_cmd_mutex);
4561 	list_for_each_entry(p, &ftrace_commands, list) {
4562 		if (strcmp(cmd->name, p->name) == 0) {
4563 			ret = -EBUSY;
4564 			goto out_unlock;
4565 		}
4566 	}
4567 	list_add(&cmd->list, &ftrace_commands);
4568  out_unlock:
4569 	mutex_unlock(&ftrace_cmd_mutex);
4570 
4571 	return ret;
4572 }
4573 
4574 /*
4575  * Currently we only unregister ftrace commands from __init, so mark
4576  * this __init too.
4577  */
4578 __init int unregister_ftrace_command(struct ftrace_func_command *cmd)
4579 {
4580 	struct ftrace_func_command *p, *n;
4581 	int ret = -ENODEV;
4582 
4583 	mutex_lock(&ftrace_cmd_mutex);
4584 	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
4585 		if (strcmp(cmd->name, p->name) == 0) {
4586 			ret = 0;
4587 			list_del_init(&p->list);
4588 			goto out_unlock;
4589 		}
4590 	}
4591  out_unlock:
4592 	mutex_unlock(&ftrace_cmd_mutex);
4593 
4594 	return ret;
4595 }
4596 
4597 static int ftrace_process_regex(struct ftrace_iterator *iter,
4598 				char *buff, int len, int enable)
4599 {
4600 	struct ftrace_hash *hash = iter->hash;
4601 	struct trace_array *tr = iter->ops->private;
4602 	char *func, *command, *next = buff;
4603 	struct ftrace_func_command *p;
4604 	int ret = -EINVAL;
4605 
4606 	func = strsep(&next, ":");
4607 
4608 	if (!next) {
4609 		ret = ftrace_match_records(hash, func, len);
4610 		if (!ret)
4611 			ret = -EINVAL;
4612 		if (ret < 0)
4613 			return ret;
4614 		return 0;
4615 	}
4616 
4617 	/* command found */
4618 
4619 	command = strsep(&next, ":");
4620 
4621 	mutex_lock(&ftrace_cmd_mutex);
4622 	list_for_each_entry(p, &ftrace_commands, list) {
4623 		if (strcmp(p->name, command) == 0) {
4624 			ret = p->func(tr, hash, func, command, next, enable);
4625 			goto out_unlock;
4626 		}
4627 	}
4628  out_unlock:
4629 	mutex_unlock(&ftrace_cmd_mutex);
4630 
4631 	return ret;
4632 }
4633 
4634 static ssize_t
4635 ftrace_regex_write(struct file *file, const char __user *ubuf,
4636 		   size_t cnt, loff_t *ppos, int enable)
4637 {
4638 	struct ftrace_iterator *iter;
4639 	struct trace_parser *parser;
4640 	ssize_t ret, read;
4641 
4642 	if (!cnt)
4643 		return 0;
4644 
4645 	if (file->f_mode & FMODE_READ) {
4646 		struct seq_file *m = file->private_data;
4647 		iter = m->private;
4648 	} else
4649 		iter = file->private_data;
4650 
4651 	if (unlikely(ftrace_disabled))
4652 		return -ENODEV;
4653 
4654 	/* iter->hash is a local copy, so we don't need regex_lock */
4655 
4656 	parser = &iter->parser;
4657 	read = trace_get_user(parser, ubuf, cnt, ppos);
4658 
4659 	if (read >= 0 && trace_parser_loaded(parser) &&
4660 	    !trace_parser_cont(parser)) {
4661 		ret = ftrace_process_regex(iter, parser->buffer,
4662 					   parser->idx, enable);
4663 		trace_parser_clear(parser);
4664 		if (ret < 0)
4665 			goto out;
4666 	}
4667 
4668 	ret = read;
4669  out:
4670 	return ret;
4671 }
4672 
4673 ssize_t
4674 ftrace_filter_write(struct file *file, const char __user *ubuf,
4675 		    size_t cnt, loff_t *ppos)
4676 {
4677 	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
4678 }
4679 
4680 ssize_t
4681 ftrace_notrace_write(struct file *file, const char __user *ubuf,
4682 		     size_t cnt, loff_t *ppos)
4683 {
4684 	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
4685 }
4686 
4687 static int
4688 ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
4689 {
4690 	struct ftrace_func_entry *entry;
4691 
4692 	if (!ftrace_location(ip))
4693 		return -EINVAL;
4694 
4695 	if (remove) {
4696 		entry = ftrace_lookup_ip(hash, ip);
4697 		if (!entry)
4698 			return -ENOENT;
4699 		free_hash_entry(hash, entry);
4700 		return 0;
4701 	}
4702 
4703 	return add_hash_entry(hash, ip);
4704 }
4705 
4706 static int
4707 ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
4708 		unsigned long ip, int remove, int reset, int enable)
4709 {
4710 	struct ftrace_hash **orig_hash;
4711 	struct ftrace_hash *hash;
4712 	int ret;
4713 
4714 	if (unlikely(ftrace_disabled))
4715 		return -ENODEV;
4716 
4717 	mutex_lock(&ops->func_hash->regex_lock);
4718 
4719 	if (enable)
4720 		orig_hash = &ops->func_hash->filter_hash;
4721 	else
4722 		orig_hash = &ops->func_hash->notrace_hash;
4723 
4724 	if (reset)
4725 		hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4726 	else
4727 		hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
4728 
4729 	if (!hash) {
4730 		ret = -ENOMEM;
4731 		goto out_regex_unlock;
4732 	}
4733 
4734 	if (buf && !ftrace_match_records(hash, buf, len)) {
4735 		ret = -EINVAL;
4736 		goto out_regex_unlock;
4737 	}
4738 	if (ip) {
4739 		ret = ftrace_match_addr(hash, ip, remove);
4740 		if (ret < 0)
4741 			goto out_regex_unlock;
4742 	}
4743 
4744 	mutex_lock(&ftrace_lock);
4745 	ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
4746 	mutex_unlock(&ftrace_lock);
4747 
4748  out_regex_unlock:
4749 	mutex_unlock(&ops->func_hash->regex_lock);
4750 
4751 	free_ftrace_hash(hash);
4752 	return ret;
4753 }
4754 
4755 static int
4756 ftrace_set_addr(struct ftrace_ops *ops, unsigned long ip, int remove,
4757 		int reset, int enable)
4758 {
4759 	return ftrace_set_hash(ops, NULL, 0, ip, remove, reset, enable);
4760 }
4761 
4762 /**
4763  * ftrace_set_filter_ip - set a function to filter on in ftrace by address
4764  * @ops - the ops to set the filter with
4765  * @ip - the address to add to or remove from the filter.
4766  * @remove - non zero to remove the ip from the filter
4767  * @reset - non zero to reset all filters before applying this filter.
4768  *
4769  * Filters denote which functions should be enabled when tracing is enabled
4770  * If @ip is NULL, it failes to update filter.
4771  */
4772 int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
4773 			 int remove, int reset)
4774 {
4775 	ftrace_ops_init(ops);
4776 	return ftrace_set_addr(ops, ip, remove, reset, 1);
4777 }
4778 EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
4779 
4780 /**
4781  * ftrace_ops_set_global_filter - setup ops to use global filters
4782  * @ops - the ops which will use the global filters
4783  *
4784  * ftrace users who need global function trace filtering should call this.
4785  * It can set the global filter only if ops were not initialized before.
4786  */
4787 void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
4788 {
4789 	if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
4790 		return;
4791 
4792 	ftrace_ops_init(ops);
4793 	ops->func_hash = &global_ops.local_hash;
4794 }
4795 EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
4796 
4797 static int
4798 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
4799 		 int reset, int enable)
4800 {
4801 	return ftrace_set_hash(ops, buf, len, 0, 0, reset, enable);
4802 }
4803 
4804 /**
4805  * ftrace_set_filter - set a function to filter on in ftrace
4806  * @ops - the ops to set the filter with
4807  * @buf - the string that holds the function filter text.
4808  * @len - the length of the string.
4809  * @reset - non zero to reset all filters before applying this filter.
4810  *
4811  * Filters denote which functions should be enabled when tracing is enabled.
4812  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4813  */
4814 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
4815 		       int len, int reset)
4816 {
4817 	ftrace_ops_init(ops);
4818 	return ftrace_set_regex(ops, buf, len, reset, 1);
4819 }
4820 EXPORT_SYMBOL_GPL(ftrace_set_filter);
4821 
4822 /**
4823  * ftrace_set_notrace - set a function to not trace in ftrace
4824  * @ops - the ops to set the notrace filter with
4825  * @buf - the string that holds the function notrace text.
4826  * @len - the length of the string.
4827  * @reset - non zero to reset all filters before applying this filter.
4828  *
4829  * Notrace Filters denote which functions should not be enabled when tracing
4830  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4831  * for tracing.
4832  */
4833 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
4834 			int len, int reset)
4835 {
4836 	ftrace_ops_init(ops);
4837 	return ftrace_set_regex(ops, buf, len, reset, 0);
4838 }
4839 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
4840 /**
4841  * ftrace_set_global_filter - set a function to filter on with global tracers
4842  * @buf - the string that holds the function filter text.
4843  * @len - the length of the string.
4844  * @reset - non zero to reset all filters before applying this filter.
4845  *
4846  * Filters denote which functions should be enabled when tracing is enabled.
4847  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
4848  */
4849 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
4850 {
4851 	ftrace_set_regex(&global_ops, buf, len, reset, 1);
4852 }
4853 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
4854 
4855 /**
4856  * ftrace_set_global_notrace - set a function to not trace with global tracers
4857  * @buf - the string that holds the function notrace text.
4858  * @len - the length of the string.
4859  * @reset - non zero to reset all filters before applying this filter.
4860  *
4861  * Notrace Filters denote which functions should not be enabled when tracing
4862  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
4863  * for tracing.
4864  */
4865 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
4866 {
4867 	ftrace_set_regex(&global_ops, buf, len, reset, 0);
4868 }
4869 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
4870 
4871 /*
4872  * command line interface to allow users to set filters on boot up.
4873  */
4874 #define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
4875 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4876 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
4877 
4878 /* Used by function selftest to not test if filter is set */
4879 bool ftrace_filter_param __initdata;
4880 
4881 static int __init set_ftrace_notrace(char *str)
4882 {
4883 	ftrace_filter_param = true;
4884 	strlcpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
4885 	return 1;
4886 }
4887 __setup("ftrace_notrace=", set_ftrace_notrace);
4888 
4889 static int __init set_ftrace_filter(char *str)
4890 {
4891 	ftrace_filter_param = true;
4892 	strlcpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
4893 	return 1;
4894 }
4895 __setup("ftrace_filter=", set_ftrace_filter);
4896 
4897 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4898 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
4899 static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
4900 static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
4901 
4902 static int __init set_graph_function(char *str)
4903 {
4904 	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
4905 	return 1;
4906 }
4907 __setup("ftrace_graph_filter=", set_graph_function);
4908 
4909 static int __init set_graph_notrace_function(char *str)
4910 {
4911 	strlcpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
4912 	return 1;
4913 }
4914 __setup("ftrace_graph_notrace=", set_graph_notrace_function);
4915 
4916 static int __init set_graph_max_depth_function(char *str)
4917 {
4918 	if (!str)
4919 		return 0;
4920 	fgraph_max_depth = simple_strtoul(str, NULL, 0);
4921 	return 1;
4922 }
4923 __setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
4924 
4925 static void __init set_ftrace_early_graph(char *buf, int enable)
4926 {
4927 	int ret;
4928 	char *func;
4929 	struct ftrace_hash *hash;
4930 
4931 	hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
4932 	if (WARN_ON(!hash))
4933 		return;
4934 
4935 	while (buf) {
4936 		func = strsep(&buf, ",");
4937 		/* we allow only one expression at a time */
4938 		ret = ftrace_graph_set_hash(hash, func);
4939 		if (ret)
4940 			printk(KERN_DEBUG "ftrace: function %s not "
4941 					  "traceable\n", func);
4942 	}
4943 
4944 	if (enable)
4945 		ftrace_graph_hash = hash;
4946 	else
4947 		ftrace_graph_notrace_hash = hash;
4948 }
4949 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4950 
4951 void __init
4952 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
4953 {
4954 	char *func;
4955 
4956 	ftrace_ops_init(ops);
4957 
4958 	while (buf) {
4959 		func = strsep(&buf, ",");
4960 		ftrace_set_regex(ops, func, strlen(func), 0, enable);
4961 	}
4962 }
4963 
4964 static void __init set_ftrace_early_filters(void)
4965 {
4966 	if (ftrace_filter_buf[0])
4967 		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
4968 	if (ftrace_notrace_buf[0])
4969 		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
4970 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4971 	if (ftrace_graph_buf[0])
4972 		set_ftrace_early_graph(ftrace_graph_buf, 1);
4973 	if (ftrace_graph_notrace_buf[0])
4974 		set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
4975 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
4976 }
4977 
4978 int ftrace_regex_release(struct inode *inode, struct file *file)
4979 {
4980 	struct seq_file *m = (struct seq_file *)file->private_data;
4981 	struct ftrace_iterator *iter;
4982 	struct ftrace_hash **orig_hash;
4983 	struct trace_parser *parser;
4984 	int filter_hash;
4985 	int ret;
4986 
4987 	if (file->f_mode & FMODE_READ) {
4988 		iter = m->private;
4989 		seq_release(inode, file);
4990 	} else
4991 		iter = file->private_data;
4992 
4993 	parser = &iter->parser;
4994 	if (trace_parser_loaded(parser)) {
4995 		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
4996 	}
4997 
4998 	trace_parser_put(parser);
4999 
5000 	mutex_lock(&iter->ops->func_hash->regex_lock);
5001 
5002 	if (file->f_mode & FMODE_WRITE) {
5003 		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
5004 
5005 		if (filter_hash) {
5006 			orig_hash = &iter->ops->func_hash->filter_hash;
5007 			if (iter->tr && !list_empty(&iter->tr->mod_trace))
5008 				iter->hash->flags |= FTRACE_HASH_FL_MOD;
5009 		} else
5010 			orig_hash = &iter->ops->func_hash->notrace_hash;
5011 
5012 		mutex_lock(&ftrace_lock);
5013 		ret = ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
5014 						      iter->hash, filter_hash);
5015 		mutex_unlock(&ftrace_lock);
5016 	} else {
5017 		/* For read only, the hash is the ops hash */
5018 		iter->hash = NULL;
5019 	}
5020 
5021 	mutex_unlock(&iter->ops->func_hash->regex_lock);
5022 	free_ftrace_hash(iter->hash);
5023 	kfree(iter);
5024 
5025 	return 0;
5026 }
5027 
5028 static const struct file_operations ftrace_avail_fops = {
5029 	.open = ftrace_avail_open,
5030 	.read = seq_read,
5031 	.llseek = seq_lseek,
5032 	.release = seq_release_private,
5033 };
5034 
5035 static const struct file_operations ftrace_enabled_fops = {
5036 	.open = ftrace_enabled_open,
5037 	.read = seq_read,
5038 	.llseek = seq_lseek,
5039 	.release = seq_release_private,
5040 };
5041 
5042 static const struct file_operations ftrace_filter_fops = {
5043 	.open = ftrace_filter_open,
5044 	.read = seq_read,
5045 	.write = ftrace_filter_write,
5046 	.llseek = tracing_lseek,
5047 	.release = ftrace_regex_release,
5048 };
5049 
5050 static const struct file_operations ftrace_notrace_fops = {
5051 	.open = ftrace_notrace_open,
5052 	.read = seq_read,
5053 	.write = ftrace_notrace_write,
5054 	.llseek = tracing_lseek,
5055 	.release = ftrace_regex_release,
5056 };
5057 
5058 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5059 
5060 static DEFINE_MUTEX(graph_lock);
5061 
5062 struct ftrace_hash *ftrace_graph_hash = EMPTY_HASH;
5063 struct ftrace_hash *ftrace_graph_notrace_hash = EMPTY_HASH;
5064 
5065 enum graph_filter_type {
5066 	GRAPH_FILTER_NOTRACE	= 0,
5067 	GRAPH_FILTER_FUNCTION,
5068 };
5069 
5070 #define FTRACE_GRAPH_EMPTY	((void *)1)
5071 
5072 struct ftrace_graph_data {
5073 	struct ftrace_hash		*hash;
5074 	struct ftrace_func_entry	*entry;
5075 	int				idx;   /* for hash table iteration */
5076 	enum graph_filter_type		type;
5077 	struct ftrace_hash		*new_hash;
5078 	const struct seq_operations	*seq_ops;
5079 	struct trace_parser		parser;
5080 };
5081 
5082 static void *
5083 __g_next(struct seq_file *m, loff_t *pos)
5084 {
5085 	struct ftrace_graph_data *fgd = m->private;
5086 	struct ftrace_func_entry *entry = fgd->entry;
5087 	struct hlist_head *head;
5088 	int i, idx = fgd->idx;
5089 
5090 	if (*pos >= fgd->hash->count)
5091 		return NULL;
5092 
5093 	if (entry) {
5094 		hlist_for_each_entry_continue(entry, hlist) {
5095 			fgd->entry = entry;
5096 			return entry;
5097 		}
5098 
5099 		idx++;
5100 	}
5101 
5102 	for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
5103 		head = &fgd->hash->buckets[i];
5104 		hlist_for_each_entry(entry, head, hlist) {
5105 			fgd->entry = entry;
5106 			fgd->idx = i;
5107 			return entry;
5108 		}
5109 	}
5110 	return NULL;
5111 }
5112 
5113 static void *
5114 g_next(struct seq_file *m, void *v, loff_t *pos)
5115 {
5116 	(*pos)++;
5117 	return __g_next(m, pos);
5118 }
5119 
5120 static void *g_start(struct seq_file *m, loff_t *pos)
5121 {
5122 	struct ftrace_graph_data *fgd = m->private;
5123 
5124 	mutex_lock(&graph_lock);
5125 
5126 	if (fgd->type == GRAPH_FILTER_FUNCTION)
5127 		fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
5128 					lockdep_is_held(&graph_lock));
5129 	else
5130 		fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
5131 					lockdep_is_held(&graph_lock));
5132 
5133 	/* Nothing, tell g_show to print all functions are enabled */
5134 	if (ftrace_hash_empty(fgd->hash) && !*pos)
5135 		return FTRACE_GRAPH_EMPTY;
5136 
5137 	fgd->idx = 0;
5138 	fgd->entry = NULL;
5139 	return __g_next(m, pos);
5140 }
5141 
5142 static void g_stop(struct seq_file *m, void *p)
5143 {
5144 	mutex_unlock(&graph_lock);
5145 }
5146 
5147 static int g_show(struct seq_file *m, void *v)
5148 {
5149 	struct ftrace_func_entry *entry = v;
5150 
5151 	if (!entry)
5152 		return 0;
5153 
5154 	if (entry == FTRACE_GRAPH_EMPTY) {
5155 		struct ftrace_graph_data *fgd = m->private;
5156 
5157 		if (fgd->type == GRAPH_FILTER_FUNCTION)
5158 			seq_puts(m, "#### all functions enabled ####\n");
5159 		else
5160 			seq_puts(m, "#### no functions disabled ####\n");
5161 		return 0;
5162 	}
5163 
5164 	seq_printf(m, "%ps\n", (void *)entry->ip);
5165 
5166 	return 0;
5167 }
5168 
5169 static const struct seq_operations ftrace_graph_seq_ops = {
5170 	.start = g_start,
5171 	.next = g_next,
5172 	.stop = g_stop,
5173 	.show = g_show,
5174 };
5175 
5176 static int
5177 __ftrace_graph_open(struct inode *inode, struct file *file,
5178 		    struct ftrace_graph_data *fgd)
5179 {
5180 	int ret = 0;
5181 	struct ftrace_hash *new_hash = NULL;
5182 
5183 	if (file->f_mode & FMODE_WRITE) {
5184 		const int size_bits = FTRACE_HASH_DEFAULT_BITS;
5185 
5186 		if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
5187 			return -ENOMEM;
5188 
5189 		if (file->f_flags & O_TRUNC)
5190 			new_hash = alloc_ftrace_hash(size_bits);
5191 		else
5192 			new_hash = alloc_and_copy_ftrace_hash(size_bits,
5193 							      fgd->hash);
5194 		if (!new_hash) {
5195 			ret = -ENOMEM;
5196 			goto out;
5197 		}
5198 	}
5199 
5200 	if (file->f_mode & FMODE_READ) {
5201 		ret = seq_open(file, &ftrace_graph_seq_ops);
5202 		if (!ret) {
5203 			struct seq_file *m = file->private_data;
5204 			m->private = fgd;
5205 		} else {
5206 			/* Failed */
5207 			free_ftrace_hash(new_hash);
5208 			new_hash = NULL;
5209 		}
5210 	} else
5211 		file->private_data = fgd;
5212 
5213 out:
5214 	if (ret < 0 && file->f_mode & FMODE_WRITE)
5215 		trace_parser_put(&fgd->parser);
5216 
5217 	fgd->new_hash = new_hash;
5218 
5219 	/*
5220 	 * All uses of fgd->hash must be taken with the graph_lock
5221 	 * held. The graph_lock is going to be released, so force
5222 	 * fgd->hash to be reinitialized when it is taken again.
5223 	 */
5224 	fgd->hash = NULL;
5225 
5226 	return ret;
5227 }
5228 
5229 static int
5230 ftrace_graph_open(struct inode *inode, struct file *file)
5231 {
5232 	struct ftrace_graph_data *fgd;
5233 	int ret;
5234 
5235 	if (unlikely(ftrace_disabled))
5236 		return -ENODEV;
5237 
5238 	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
5239 	if (fgd == NULL)
5240 		return -ENOMEM;
5241 
5242 	mutex_lock(&graph_lock);
5243 
5244 	fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
5245 					lockdep_is_held(&graph_lock));
5246 	fgd->type = GRAPH_FILTER_FUNCTION;
5247 	fgd->seq_ops = &ftrace_graph_seq_ops;
5248 
5249 	ret = __ftrace_graph_open(inode, file, fgd);
5250 	if (ret < 0)
5251 		kfree(fgd);
5252 
5253 	mutex_unlock(&graph_lock);
5254 	return ret;
5255 }
5256 
5257 static int
5258 ftrace_graph_notrace_open(struct inode *inode, struct file *file)
5259 {
5260 	struct ftrace_graph_data *fgd;
5261 	int ret;
5262 
5263 	if (unlikely(ftrace_disabled))
5264 		return -ENODEV;
5265 
5266 	fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
5267 	if (fgd == NULL)
5268 		return -ENOMEM;
5269 
5270 	mutex_lock(&graph_lock);
5271 
5272 	fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
5273 					lockdep_is_held(&graph_lock));
5274 	fgd->type = GRAPH_FILTER_NOTRACE;
5275 	fgd->seq_ops = &ftrace_graph_seq_ops;
5276 
5277 	ret = __ftrace_graph_open(inode, file, fgd);
5278 	if (ret < 0)
5279 		kfree(fgd);
5280 
5281 	mutex_unlock(&graph_lock);
5282 	return ret;
5283 }
5284 
5285 static int
5286 ftrace_graph_release(struct inode *inode, struct file *file)
5287 {
5288 	struct ftrace_graph_data *fgd;
5289 	struct ftrace_hash *old_hash, *new_hash;
5290 	struct trace_parser *parser;
5291 	int ret = 0;
5292 
5293 	if (file->f_mode & FMODE_READ) {
5294 		struct seq_file *m = file->private_data;
5295 
5296 		fgd = m->private;
5297 		seq_release(inode, file);
5298 	} else {
5299 		fgd = file->private_data;
5300 	}
5301 
5302 
5303 	if (file->f_mode & FMODE_WRITE) {
5304 
5305 		parser = &fgd->parser;
5306 
5307 		if (trace_parser_loaded((parser))) {
5308 			ret = ftrace_graph_set_hash(fgd->new_hash,
5309 						    parser->buffer);
5310 		}
5311 
5312 		trace_parser_put(parser);
5313 
5314 		new_hash = __ftrace_hash_move(fgd->new_hash);
5315 		if (!new_hash) {
5316 			ret = -ENOMEM;
5317 			goto out;
5318 		}
5319 
5320 		mutex_lock(&graph_lock);
5321 
5322 		if (fgd->type == GRAPH_FILTER_FUNCTION) {
5323 			old_hash = rcu_dereference_protected(ftrace_graph_hash,
5324 					lockdep_is_held(&graph_lock));
5325 			rcu_assign_pointer(ftrace_graph_hash, new_hash);
5326 		} else {
5327 			old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
5328 					lockdep_is_held(&graph_lock));
5329 			rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
5330 		}
5331 
5332 		mutex_unlock(&graph_lock);
5333 
5334 		/* Wait till all users are no longer using the old hash */
5335 		synchronize_rcu();
5336 
5337 		free_ftrace_hash(old_hash);
5338 	}
5339 
5340  out:
5341 	free_ftrace_hash(fgd->new_hash);
5342 	kfree(fgd);
5343 
5344 	return ret;
5345 }
5346 
5347 static int
5348 ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
5349 {
5350 	struct ftrace_glob func_g;
5351 	struct dyn_ftrace *rec;
5352 	struct ftrace_page *pg;
5353 	struct ftrace_func_entry *entry;
5354 	int fail = 1;
5355 	int not;
5356 
5357 	/* decode regex */
5358 	func_g.type = filter_parse_regex(buffer, strlen(buffer),
5359 					 &func_g.search, &not);
5360 
5361 	func_g.len = strlen(func_g.search);
5362 
5363 	mutex_lock(&ftrace_lock);
5364 
5365 	if (unlikely(ftrace_disabled)) {
5366 		mutex_unlock(&ftrace_lock);
5367 		return -ENODEV;
5368 	}
5369 
5370 	do_for_each_ftrace_rec(pg, rec) {
5371 
5372 		if (rec->flags & FTRACE_FL_DISABLED)
5373 			continue;
5374 
5375 		if (ftrace_match_record(rec, &func_g, NULL, 0)) {
5376 			entry = ftrace_lookup_ip(hash, rec->ip);
5377 
5378 			if (!not) {
5379 				fail = 0;
5380 
5381 				if (entry)
5382 					continue;
5383 				if (add_hash_entry(hash, rec->ip) < 0)
5384 					goto out;
5385 			} else {
5386 				if (entry) {
5387 					free_hash_entry(hash, entry);
5388 					fail = 0;
5389 				}
5390 			}
5391 		}
5392 	} while_for_each_ftrace_rec();
5393 out:
5394 	mutex_unlock(&ftrace_lock);
5395 
5396 	if (fail)
5397 		return -EINVAL;
5398 
5399 	return 0;
5400 }
5401 
5402 static ssize_t
5403 ftrace_graph_write(struct file *file, const char __user *ubuf,
5404 		   size_t cnt, loff_t *ppos)
5405 {
5406 	ssize_t read, ret = 0;
5407 	struct ftrace_graph_data *fgd = file->private_data;
5408 	struct trace_parser *parser;
5409 
5410 	if (!cnt)
5411 		return 0;
5412 
5413 	/* Read mode uses seq functions */
5414 	if (file->f_mode & FMODE_READ) {
5415 		struct seq_file *m = file->private_data;
5416 		fgd = m->private;
5417 	}
5418 
5419 	parser = &fgd->parser;
5420 
5421 	read = trace_get_user(parser, ubuf, cnt, ppos);
5422 
5423 	if (read >= 0 && trace_parser_loaded(parser) &&
5424 	    !trace_parser_cont(parser)) {
5425 
5426 		ret = ftrace_graph_set_hash(fgd->new_hash,
5427 					    parser->buffer);
5428 		trace_parser_clear(parser);
5429 	}
5430 
5431 	if (!ret)
5432 		ret = read;
5433 
5434 	return ret;
5435 }
5436 
5437 static const struct file_operations ftrace_graph_fops = {
5438 	.open		= ftrace_graph_open,
5439 	.read		= seq_read,
5440 	.write		= ftrace_graph_write,
5441 	.llseek		= tracing_lseek,
5442 	.release	= ftrace_graph_release,
5443 };
5444 
5445 static const struct file_operations ftrace_graph_notrace_fops = {
5446 	.open		= ftrace_graph_notrace_open,
5447 	.read		= seq_read,
5448 	.write		= ftrace_graph_write,
5449 	.llseek		= tracing_lseek,
5450 	.release	= ftrace_graph_release,
5451 };
5452 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5453 
5454 void ftrace_create_filter_files(struct ftrace_ops *ops,
5455 				struct dentry *parent)
5456 {
5457 
5458 	trace_create_file("set_ftrace_filter", 0644, parent,
5459 			  ops, &ftrace_filter_fops);
5460 
5461 	trace_create_file("set_ftrace_notrace", 0644, parent,
5462 			  ops, &ftrace_notrace_fops);
5463 }
5464 
5465 /*
5466  * The name "destroy_filter_files" is really a misnomer. Although
5467  * in the future, it may actually delete the files, but this is
5468  * really intended to make sure the ops passed in are disabled
5469  * and that when this function returns, the caller is free to
5470  * free the ops.
5471  *
5472  * The "destroy" name is only to match the "create" name that this
5473  * should be paired with.
5474  */
5475 void ftrace_destroy_filter_files(struct ftrace_ops *ops)
5476 {
5477 	mutex_lock(&ftrace_lock);
5478 	if (ops->flags & FTRACE_OPS_FL_ENABLED)
5479 		ftrace_shutdown(ops, 0);
5480 	ops->flags |= FTRACE_OPS_FL_DELETED;
5481 	ftrace_free_filter(ops);
5482 	mutex_unlock(&ftrace_lock);
5483 }
5484 
5485 static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
5486 {
5487 
5488 	trace_create_file("available_filter_functions", 0444,
5489 			d_tracer, NULL, &ftrace_avail_fops);
5490 
5491 	trace_create_file("enabled_functions", 0444,
5492 			d_tracer, NULL, &ftrace_enabled_fops);
5493 
5494 	ftrace_create_filter_files(&global_ops, d_tracer);
5495 
5496 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
5497 	trace_create_file("set_graph_function", 0644, d_tracer,
5498 				    NULL,
5499 				    &ftrace_graph_fops);
5500 	trace_create_file("set_graph_notrace", 0644, d_tracer,
5501 				    NULL,
5502 				    &ftrace_graph_notrace_fops);
5503 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
5504 
5505 	return 0;
5506 }
5507 
5508 static int ftrace_cmp_ips(const void *a, const void *b)
5509 {
5510 	const unsigned long *ipa = a;
5511 	const unsigned long *ipb = b;
5512 
5513 	if (*ipa > *ipb)
5514 		return 1;
5515 	if (*ipa < *ipb)
5516 		return -1;
5517 	return 0;
5518 }
5519 
5520 static int ftrace_process_locs(struct module *mod,
5521 			       unsigned long *start,
5522 			       unsigned long *end)
5523 {
5524 	struct ftrace_page *start_pg;
5525 	struct ftrace_page *pg;
5526 	struct dyn_ftrace *rec;
5527 	unsigned long count;
5528 	unsigned long *p;
5529 	unsigned long addr;
5530 	unsigned long flags = 0; /* Shut up gcc */
5531 	int ret = -ENOMEM;
5532 
5533 	count = end - start;
5534 
5535 	if (!count)
5536 		return 0;
5537 
5538 	sort(start, count, sizeof(*start),
5539 	     ftrace_cmp_ips, NULL);
5540 
5541 	start_pg = ftrace_allocate_pages(count);
5542 	if (!start_pg)
5543 		return -ENOMEM;
5544 
5545 	mutex_lock(&ftrace_lock);
5546 
5547 	/*
5548 	 * Core and each module needs their own pages, as
5549 	 * modules will free them when they are removed.
5550 	 * Force a new page to be allocated for modules.
5551 	 */
5552 	if (!mod) {
5553 		WARN_ON(ftrace_pages || ftrace_pages_start);
5554 		/* First initialization */
5555 		ftrace_pages = ftrace_pages_start = start_pg;
5556 	} else {
5557 		if (!ftrace_pages)
5558 			goto out;
5559 
5560 		if (WARN_ON(ftrace_pages->next)) {
5561 			/* Hmm, we have free pages? */
5562 			while (ftrace_pages->next)
5563 				ftrace_pages = ftrace_pages->next;
5564 		}
5565 
5566 		ftrace_pages->next = start_pg;
5567 	}
5568 
5569 	p = start;
5570 	pg = start_pg;
5571 	while (p < end) {
5572 		addr = ftrace_call_adjust(*p++);
5573 		/*
5574 		 * Some architecture linkers will pad between
5575 		 * the different mcount_loc sections of different
5576 		 * object files to satisfy alignments.
5577 		 * Skip any NULL pointers.
5578 		 */
5579 		if (!addr)
5580 			continue;
5581 
5582 		if (pg->index == pg->size) {
5583 			/* We should have allocated enough */
5584 			if (WARN_ON(!pg->next))
5585 				break;
5586 			pg = pg->next;
5587 		}
5588 
5589 		rec = &pg->records[pg->index++];
5590 		rec->ip = addr;
5591 	}
5592 
5593 	/* We should have used all pages */
5594 	WARN_ON(pg->next);
5595 
5596 	/* Assign the last page to ftrace_pages */
5597 	ftrace_pages = pg;
5598 
5599 	/*
5600 	 * We only need to disable interrupts on start up
5601 	 * because we are modifying code that an interrupt
5602 	 * may execute, and the modification is not atomic.
5603 	 * But for modules, nothing runs the code we modify
5604 	 * until we are finished with it, and there's no
5605 	 * reason to cause large interrupt latencies while we do it.
5606 	 */
5607 	if (!mod)
5608 		local_irq_save(flags);
5609 	ftrace_update_code(mod, start_pg);
5610 	if (!mod)
5611 		local_irq_restore(flags);
5612 	ret = 0;
5613  out:
5614 	mutex_unlock(&ftrace_lock);
5615 
5616 	return ret;
5617 }
5618 
5619 struct ftrace_mod_func {
5620 	struct list_head	list;
5621 	char			*name;
5622 	unsigned long		ip;
5623 	unsigned int		size;
5624 };
5625 
5626 struct ftrace_mod_map {
5627 	struct rcu_head		rcu;
5628 	struct list_head	list;
5629 	struct module		*mod;
5630 	unsigned long		start_addr;
5631 	unsigned long		end_addr;
5632 	struct list_head	funcs;
5633 	unsigned int		num_funcs;
5634 };
5635 
5636 #ifdef CONFIG_MODULES
5637 
5638 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
5639 
5640 static LIST_HEAD(ftrace_mod_maps);
5641 
5642 static int referenced_filters(struct dyn_ftrace *rec)
5643 {
5644 	struct ftrace_ops *ops;
5645 	int cnt = 0;
5646 
5647 	for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
5648 		if (ops_references_rec(ops, rec))
5649 		    cnt++;
5650 	}
5651 
5652 	return cnt;
5653 }
5654 
5655 static void
5656 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
5657 {
5658 	struct ftrace_func_entry *entry;
5659 	struct dyn_ftrace *rec;
5660 	int i;
5661 
5662 	if (ftrace_hash_empty(hash))
5663 		return;
5664 
5665 	for (i = 0; i < pg->index; i++) {
5666 		rec = &pg->records[i];
5667 		entry = __ftrace_lookup_ip(hash, rec->ip);
5668 		/*
5669 		 * Do not allow this rec to match again.
5670 		 * Yeah, it may waste some memory, but will be removed
5671 		 * if/when the hash is modified again.
5672 		 */
5673 		if (entry)
5674 			entry->ip = 0;
5675 	}
5676 }
5677 
5678 /* Clear any records from hashs */
5679 static void clear_mod_from_hashes(struct ftrace_page *pg)
5680 {
5681 	struct trace_array *tr;
5682 
5683 	mutex_lock(&trace_types_lock);
5684 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5685 		if (!tr->ops || !tr->ops->func_hash)
5686 			continue;
5687 		mutex_lock(&tr->ops->func_hash->regex_lock);
5688 		clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
5689 		clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
5690 		mutex_unlock(&tr->ops->func_hash->regex_lock);
5691 	}
5692 	mutex_unlock(&trace_types_lock);
5693 }
5694 
5695 static void ftrace_free_mod_map(struct rcu_head *rcu)
5696 {
5697 	struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
5698 	struct ftrace_mod_func *mod_func;
5699 	struct ftrace_mod_func *n;
5700 
5701 	/* All the contents of mod_map are now not visible to readers */
5702 	list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
5703 		kfree(mod_func->name);
5704 		list_del(&mod_func->list);
5705 		kfree(mod_func);
5706 	}
5707 
5708 	kfree(mod_map);
5709 }
5710 
5711 void ftrace_release_mod(struct module *mod)
5712 {
5713 	struct ftrace_mod_map *mod_map;
5714 	struct ftrace_mod_map *n;
5715 	struct dyn_ftrace *rec;
5716 	struct ftrace_page **last_pg;
5717 	struct ftrace_page *tmp_page = NULL;
5718 	struct ftrace_page *pg;
5719 	int order;
5720 
5721 	mutex_lock(&ftrace_lock);
5722 
5723 	if (ftrace_disabled)
5724 		goto out_unlock;
5725 
5726 	list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
5727 		if (mod_map->mod == mod) {
5728 			list_del_rcu(&mod_map->list);
5729 			call_rcu(&mod_map->rcu, ftrace_free_mod_map);
5730 			break;
5731 		}
5732 	}
5733 
5734 	/*
5735 	 * Each module has its own ftrace_pages, remove
5736 	 * them from the list.
5737 	 */
5738 	last_pg = &ftrace_pages_start;
5739 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
5740 		rec = &pg->records[0];
5741 		if (within_module_core(rec->ip, mod) ||
5742 		    within_module_init(rec->ip, mod)) {
5743 			/*
5744 			 * As core pages are first, the first
5745 			 * page should never be a module page.
5746 			 */
5747 			if (WARN_ON(pg == ftrace_pages_start))
5748 				goto out_unlock;
5749 
5750 			/* Check if we are deleting the last page */
5751 			if (pg == ftrace_pages)
5752 				ftrace_pages = next_to_ftrace_page(last_pg);
5753 
5754 			ftrace_update_tot_cnt -= pg->index;
5755 			*last_pg = pg->next;
5756 
5757 			pg->next = tmp_page;
5758 			tmp_page = pg;
5759 		} else
5760 			last_pg = &pg->next;
5761 	}
5762  out_unlock:
5763 	mutex_unlock(&ftrace_lock);
5764 
5765 	for (pg = tmp_page; pg; pg = tmp_page) {
5766 
5767 		/* Needs to be called outside of ftrace_lock */
5768 		clear_mod_from_hashes(pg);
5769 
5770 		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
5771 		free_pages((unsigned long)pg->records, order);
5772 		tmp_page = pg->next;
5773 		kfree(pg);
5774 	}
5775 }
5776 
5777 void ftrace_module_enable(struct module *mod)
5778 {
5779 	struct dyn_ftrace *rec;
5780 	struct ftrace_page *pg;
5781 
5782 	mutex_lock(&ftrace_lock);
5783 
5784 	if (ftrace_disabled)
5785 		goto out_unlock;
5786 
5787 	/*
5788 	 * If the tracing is enabled, go ahead and enable the record.
5789 	 *
5790 	 * The reason not to enable the record immediately is the
5791 	 * inherent check of ftrace_make_nop/ftrace_make_call for
5792 	 * correct previous instructions.  Making first the NOP
5793 	 * conversion puts the module to the correct state, thus
5794 	 * passing the ftrace_make_call check.
5795 	 *
5796 	 * We also delay this to after the module code already set the
5797 	 * text to read-only, as we now need to set it back to read-write
5798 	 * so that we can modify the text.
5799 	 */
5800 	if (ftrace_start_up)
5801 		ftrace_arch_code_modify_prepare();
5802 
5803 	do_for_each_ftrace_rec(pg, rec) {
5804 		int cnt;
5805 		/*
5806 		 * do_for_each_ftrace_rec() is a double loop.
5807 		 * module text shares the pg. If a record is
5808 		 * not part of this module, then skip this pg,
5809 		 * which the "break" will do.
5810 		 */
5811 		if (!within_module_core(rec->ip, mod) &&
5812 		    !within_module_init(rec->ip, mod))
5813 			break;
5814 
5815 		cnt = 0;
5816 
5817 		/*
5818 		 * When adding a module, we need to check if tracers are
5819 		 * currently enabled and if they are, and can trace this record,
5820 		 * we need to enable the module functions as well as update the
5821 		 * reference counts for those function records.
5822 		 */
5823 		if (ftrace_start_up)
5824 			cnt += referenced_filters(rec);
5825 
5826 		/* This clears FTRACE_FL_DISABLED */
5827 		rec->flags = cnt;
5828 
5829 		if (ftrace_start_up && cnt) {
5830 			int failed = __ftrace_replace_code(rec, 1);
5831 			if (failed) {
5832 				ftrace_bug(failed, rec);
5833 				goto out_loop;
5834 			}
5835 		}
5836 
5837 	} while_for_each_ftrace_rec();
5838 
5839  out_loop:
5840 	if (ftrace_start_up)
5841 		ftrace_arch_code_modify_post_process();
5842 
5843  out_unlock:
5844 	mutex_unlock(&ftrace_lock);
5845 
5846 	process_cached_mods(mod->name);
5847 }
5848 
5849 void ftrace_module_init(struct module *mod)
5850 {
5851 	if (ftrace_disabled || !mod->num_ftrace_callsites)
5852 		return;
5853 
5854 	ftrace_process_locs(mod, mod->ftrace_callsites,
5855 			    mod->ftrace_callsites + mod->num_ftrace_callsites);
5856 }
5857 
5858 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
5859 				struct dyn_ftrace *rec)
5860 {
5861 	struct ftrace_mod_func *mod_func;
5862 	unsigned long symsize;
5863 	unsigned long offset;
5864 	char str[KSYM_SYMBOL_LEN];
5865 	char *modname;
5866 	const char *ret;
5867 
5868 	ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
5869 	if (!ret)
5870 		return;
5871 
5872 	mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
5873 	if (!mod_func)
5874 		return;
5875 
5876 	mod_func->name = kstrdup(str, GFP_KERNEL);
5877 	if (!mod_func->name) {
5878 		kfree(mod_func);
5879 		return;
5880 	}
5881 
5882 	mod_func->ip = rec->ip - offset;
5883 	mod_func->size = symsize;
5884 
5885 	mod_map->num_funcs++;
5886 
5887 	list_add_rcu(&mod_func->list, &mod_map->funcs);
5888 }
5889 
5890 static struct ftrace_mod_map *
5891 allocate_ftrace_mod_map(struct module *mod,
5892 			unsigned long start, unsigned long end)
5893 {
5894 	struct ftrace_mod_map *mod_map;
5895 
5896 	mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
5897 	if (!mod_map)
5898 		return NULL;
5899 
5900 	mod_map->mod = mod;
5901 	mod_map->start_addr = start;
5902 	mod_map->end_addr = end;
5903 	mod_map->num_funcs = 0;
5904 
5905 	INIT_LIST_HEAD_RCU(&mod_map->funcs);
5906 
5907 	list_add_rcu(&mod_map->list, &ftrace_mod_maps);
5908 
5909 	return mod_map;
5910 }
5911 
5912 static const char *
5913 ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
5914 			   unsigned long addr, unsigned long *size,
5915 			   unsigned long *off, char *sym)
5916 {
5917 	struct ftrace_mod_func *found_func =  NULL;
5918 	struct ftrace_mod_func *mod_func;
5919 
5920 	list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
5921 		if (addr >= mod_func->ip &&
5922 		    addr < mod_func->ip + mod_func->size) {
5923 			found_func = mod_func;
5924 			break;
5925 		}
5926 	}
5927 
5928 	if (found_func) {
5929 		if (size)
5930 			*size = found_func->size;
5931 		if (off)
5932 			*off = addr - found_func->ip;
5933 		if (sym)
5934 			strlcpy(sym, found_func->name, KSYM_NAME_LEN);
5935 
5936 		return found_func->name;
5937 	}
5938 
5939 	return NULL;
5940 }
5941 
5942 const char *
5943 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
5944 		   unsigned long *off, char **modname, char *sym)
5945 {
5946 	struct ftrace_mod_map *mod_map;
5947 	const char *ret = NULL;
5948 
5949 	/* mod_map is freed via call_rcu() */
5950 	preempt_disable();
5951 	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
5952 		ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
5953 		if (ret) {
5954 			if (modname)
5955 				*modname = mod_map->mod->name;
5956 			break;
5957 		}
5958 	}
5959 	preempt_enable();
5960 
5961 	return ret;
5962 }
5963 
5964 int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
5965 			   char *type, char *name,
5966 			   char *module_name, int *exported)
5967 {
5968 	struct ftrace_mod_map *mod_map;
5969 	struct ftrace_mod_func *mod_func;
5970 
5971 	preempt_disable();
5972 	list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
5973 
5974 		if (symnum >= mod_map->num_funcs) {
5975 			symnum -= mod_map->num_funcs;
5976 			continue;
5977 		}
5978 
5979 		list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
5980 			if (symnum > 1) {
5981 				symnum--;
5982 				continue;
5983 			}
5984 
5985 			*value = mod_func->ip;
5986 			*type = 'T';
5987 			strlcpy(name, mod_func->name, KSYM_NAME_LEN);
5988 			strlcpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
5989 			*exported = 1;
5990 			preempt_enable();
5991 			return 0;
5992 		}
5993 		WARN_ON(1);
5994 		break;
5995 	}
5996 	preempt_enable();
5997 	return -ERANGE;
5998 }
5999 
6000 #else
6001 static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
6002 				struct dyn_ftrace *rec) { }
6003 static inline struct ftrace_mod_map *
6004 allocate_ftrace_mod_map(struct module *mod,
6005 			unsigned long start, unsigned long end)
6006 {
6007 	return NULL;
6008 }
6009 #endif /* CONFIG_MODULES */
6010 
6011 struct ftrace_init_func {
6012 	struct list_head list;
6013 	unsigned long ip;
6014 };
6015 
6016 /* Clear any init ips from hashes */
6017 static void
6018 clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
6019 {
6020 	struct ftrace_func_entry *entry;
6021 
6022 	if (ftrace_hash_empty(hash))
6023 		return;
6024 
6025 	entry = __ftrace_lookup_ip(hash, func->ip);
6026 
6027 	/*
6028 	 * Do not allow this rec to match again.
6029 	 * Yeah, it may waste some memory, but will be removed
6030 	 * if/when the hash is modified again.
6031 	 */
6032 	if (entry)
6033 		entry->ip = 0;
6034 }
6035 
6036 static void
6037 clear_func_from_hashes(struct ftrace_init_func *func)
6038 {
6039 	struct trace_array *tr;
6040 
6041 	mutex_lock(&trace_types_lock);
6042 	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6043 		if (!tr->ops || !tr->ops->func_hash)
6044 			continue;
6045 		mutex_lock(&tr->ops->func_hash->regex_lock);
6046 		clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
6047 		clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
6048 		mutex_unlock(&tr->ops->func_hash->regex_lock);
6049 	}
6050 	mutex_unlock(&trace_types_lock);
6051 }
6052 
6053 static void add_to_clear_hash_list(struct list_head *clear_list,
6054 				   struct dyn_ftrace *rec)
6055 {
6056 	struct ftrace_init_func *func;
6057 
6058 	func = kmalloc(sizeof(*func), GFP_KERNEL);
6059 	if (!func) {
6060 		WARN_ONCE(1, "alloc failure, ftrace filter could be stale\n");
6061 		return;
6062 	}
6063 
6064 	func->ip = rec->ip;
6065 	list_add(&func->list, clear_list);
6066 }
6067 
6068 void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
6069 {
6070 	unsigned long start = (unsigned long)(start_ptr);
6071 	unsigned long end = (unsigned long)(end_ptr);
6072 	struct ftrace_page **last_pg = &ftrace_pages_start;
6073 	struct ftrace_page *pg;
6074 	struct dyn_ftrace *rec;
6075 	struct dyn_ftrace key;
6076 	struct ftrace_mod_map *mod_map = NULL;
6077 	struct ftrace_init_func *func, *func_next;
6078 	struct list_head clear_hash;
6079 	int order;
6080 
6081 	INIT_LIST_HEAD(&clear_hash);
6082 
6083 	key.ip = start;
6084 	key.flags = end;	/* overload flags, as it is unsigned long */
6085 
6086 	mutex_lock(&ftrace_lock);
6087 
6088 	/*
6089 	 * If we are freeing module init memory, then check if
6090 	 * any tracer is active. If so, we need to save a mapping of
6091 	 * the module functions being freed with the address.
6092 	 */
6093 	if (mod && ftrace_ops_list != &ftrace_list_end)
6094 		mod_map = allocate_ftrace_mod_map(mod, start, end);
6095 
6096 	for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
6097 		if (end < pg->records[0].ip ||
6098 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
6099 			continue;
6100  again:
6101 		rec = bsearch(&key, pg->records, pg->index,
6102 			      sizeof(struct dyn_ftrace),
6103 			      ftrace_cmp_recs);
6104 		if (!rec)
6105 			continue;
6106 
6107 		/* rec will be cleared from hashes after ftrace_lock unlock */
6108 		add_to_clear_hash_list(&clear_hash, rec);
6109 
6110 		if (mod_map)
6111 			save_ftrace_mod_rec(mod_map, rec);
6112 
6113 		pg->index--;
6114 		ftrace_update_tot_cnt--;
6115 		if (!pg->index) {
6116 			*last_pg = pg->next;
6117 			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
6118 			free_pages((unsigned long)pg->records, order);
6119 			kfree(pg);
6120 			pg = container_of(last_pg, struct ftrace_page, next);
6121 			if (!(*last_pg))
6122 				ftrace_pages = pg;
6123 			continue;
6124 		}
6125 		memmove(rec, rec + 1,
6126 			(pg->index - (rec - pg->records)) * sizeof(*rec));
6127 		/* More than one function may be in this block */
6128 		goto again;
6129 	}
6130 	mutex_unlock(&ftrace_lock);
6131 
6132 	list_for_each_entry_safe(func, func_next, &clear_hash, list) {
6133 		clear_func_from_hashes(func);
6134 		kfree(func);
6135 	}
6136 }
6137 
6138 void __init ftrace_free_init_mem(void)
6139 {
6140 	void *start = (void *)(&__init_begin);
6141 	void *end = (void *)(&__init_end);
6142 
6143 	ftrace_free_mem(NULL, start, end);
6144 }
6145 
6146 void __init ftrace_init(void)
6147 {
6148 	extern unsigned long __start_mcount_loc[];
6149 	extern unsigned long __stop_mcount_loc[];
6150 	unsigned long count, flags;
6151 	int ret;
6152 
6153 	local_irq_save(flags);
6154 	ret = ftrace_dyn_arch_init();
6155 	local_irq_restore(flags);
6156 	if (ret)
6157 		goto failed;
6158 
6159 	count = __stop_mcount_loc - __start_mcount_loc;
6160 	if (!count) {
6161 		pr_info("ftrace: No functions to be traced?\n");
6162 		goto failed;
6163 	}
6164 
6165 	pr_info("ftrace: allocating %ld entries in %ld pages\n",
6166 		count, count / ENTRIES_PER_PAGE + 1);
6167 
6168 	last_ftrace_enabled = ftrace_enabled = 1;
6169 
6170 	ret = ftrace_process_locs(NULL,
6171 				  __start_mcount_loc,
6172 				  __stop_mcount_loc);
6173 
6174 	set_ftrace_early_filters();
6175 
6176 	return;
6177  failed:
6178 	ftrace_disabled = 1;
6179 }
6180 
6181 /* Do nothing if arch does not support this */
6182 void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
6183 {
6184 }
6185 
6186 static void ftrace_update_trampoline(struct ftrace_ops *ops)
6187 {
6188 	arch_ftrace_update_trampoline(ops);
6189 }
6190 
6191 void ftrace_init_trace_array(struct trace_array *tr)
6192 {
6193 	INIT_LIST_HEAD(&tr->func_probes);
6194 	INIT_LIST_HEAD(&tr->mod_trace);
6195 	INIT_LIST_HEAD(&tr->mod_notrace);
6196 }
6197 #else
6198 
6199 struct ftrace_ops global_ops = {
6200 	.func			= ftrace_stub,
6201 	.flags			= FTRACE_OPS_FL_RECURSION_SAFE |
6202 				  FTRACE_OPS_FL_INITIALIZED |
6203 				  FTRACE_OPS_FL_PID,
6204 };
6205 
6206 static int __init ftrace_nodyn_init(void)
6207 {
6208 	ftrace_enabled = 1;
6209 	return 0;
6210 }
6211 core_initcall(ftrace_nodyn_init);
6212 
6213 static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
6214 static inline void ftrace_startup_enable(int command) { }
6215 static inline void ftrace_startup_all(int command) { }
6216 
6217 # define ftrace_startup_sysctl()	do { } while (0)
6218 # define ftrace_shutdown_sysctl()	do { } while (0)
6219 
6220 static void ftrace_update_trampoline(struct ftrace_ops *ops)
6221 {
6222 }
6223 
6224 #endif /* CONFIG_DYNAMIC_FTRACE */
6225 
6226 __init void ftrace_init_global_array_ops(struct trace_array *tr)
6227 {
6228 	tr->ops = &global_ops;
6229 	tr->ops->private = tr;
6230 	ftrace_init_trace_array(tr);
6231 }
6232 
6233 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
6234 {
6235 	/* If we filter on pids, update to use the pid function */
6236 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
6237 		if (WARN_ON(tr->ops->func != ftrace_stub))
6238 			printk("ftrace ops had %pS for function\n",
6239 			       tr->ops->func);
6240 	}
6241 	tr->ops->func = func;
6242 	tr->ops->private = tr;
6243 }
6244 
6245 void ftrace_reset_array_ops(struct trace_array *tr)
6246 {
6247 	tr->ops->func = ftrace_stub;
6248 }
6249 
6250 static nokprobe_inline void
6251 __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6252 		       struct ftrace_ops *ignored, struct pt_regs *regs)
6253 {
6254 	struct ftrace_ops *op;
6255 	int bit;
6256 
6257 	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
6258 	if (bit < 0)
6259 		return;
6260 
6261 	/*
6262 	 * Some of the ops may be dynamically allocated,
6263 	 * they must be freed after a synchronize_rcu().
6264 	 */
6265 	preempt_disable_notrace();
6266 
6267 	do_for_each_ftrace_op(op, ftrace_ops_list) {
6268 		/*
6269 		 * Check the following for each ops before calling their func:
6270 		 *  if RCU flag is set, then rcu_is_watching() must be true
6271 		 *  if PER_CPU is set, then ftrace_function_local_disable()
6272 		 *                          must be false
6273 		 *  Otherwise test if the ip matches the ops filter
6274 		 *
6275 		 * If any of the above fails then the op->func() is not executed.
6276 		 */
6277 		if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
6278 		    ftrace_ops_test(op, ip, regs)) {
6279 			if (FTRACE_WARN_ON(!op->func)) {
6280 				pr_warn("op=%p %pS\n", op, op);
6281 				goto out;
6282 			}
6283 			op->func(ip, parent_ip, op, regs);
6284 		}
6285 	} while_for_each_ftrace_op(op);
6286 out:
6287 	preempt_enable_notrace();
6288 	trace_clear_recursion(bit);
6289 }
6290 
6291 /*
6292  * Some archs only support passing ip and parent_ip. Even though
6293  * the list function ignores the op parameter, we do not want any
6294  * C side effects, where a function is called without the caller
6295  * sending a third parameter.
6296  * Archs are to support both the regs and ftrace_ops at the same time.
6297  * If they support ftrace_ops, it is assumed they support regs.
6298  * If call backs want to use regs, they must either check for regs
6299  * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS.
6300  * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved.
6301  * An architecture can pass partial regs with ftrace_ops and still
6302  * set the ARCH_SUPPORTS_FTRACE_OPS.
6303  */
6304 #if ARCH_SUPPORTS_FTRACE_OPS
6305 static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
6306 				 struct ftrace_ops *op, struct pt_regs *regs)
6307 {
6308 	__ftrace_ops_list_func(ip, parent_ip, NULL, regs);
6309 }
6310 NOKPROBE_SYMBOL(ftrace_ops_list_func);
6311 #else
6312 static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip)
6313 {
6314 	__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
6315 }
6316 NOKPROBE_SYMBOL(ftrace_ops_no_ops);
6317 #endif
6318 
6319 /*
6320  * If there's only one function registered but it does not support
6321  * recursion, needs RCU protection and/or requires per cpu handling, then
6322  * this function will be called by the mcount trampoline.
6323  */
6324 static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
6325 				   struct ftrace_ops *op, struct pt_regs *regs)
6326 {
6327 	int bit;
6328 
6329 	if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
6330 		return;
6331 
6332 	bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
6333 	if (bit < 0)
6334 		return;
6335 
6336 	preempt_disable_notrace();
6337 
6338 	op->func(ip, parent_ip, op, regs);
6339 
6340 	preempt_enable_notrace();
6341 	trace_clear_recursion(bit);
6342 }
6343 NOKPROBE_SYMBOL(ftrace_ops_assist_func);
6344 
6345 /**
6346  * ftrace_ops_get_func - get the function a trampoline should call
6347  * @ops: the ops to get the function for
6348  *
6349  * Normally the mcount trampoline will call the ops->func, but there
6350  * are times that it should not. For example, if the ops does not
6351  * have its own recursion protection, then it should call the
6352  * ftrace_ops_assist_func() instead.
6353  *
6354  * Returns the function that the trampoline should call for @ops.
6355  */
6356 ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
6357 {
6358 	/*
6359 	 * If the function does not handle recursion, needs to be RCU safe,
6360 	 * or does per cpu logic, then we need to call the assist handler.
6361 	 */
6362 	if (!(ops->flags & FTRACE_OPS_FL_RECURSION_SAFE) ||
6363 	    ops->flags & FTRACE_OPS_FL_RCU)
6364 		return ftrace_ops_assist_func;
6365 
6366 	return ops->func;
6367 }
6368 
6369 static void
6370 ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
6371 		    struct task_struct *prev, struct task_struct *next)
6372 {
6373 	struct trace_array *tr = data;
6374 	struct trace_pid_list *pid_list;
6375 
6376 	pid_list = rcu_dereference_sched(tr->function_pids);
6377 
6378 	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
6379 		       trace_ignore_this_task(pid_list, next));
6380 }
6381 
6382 static void
6383 ftrace_pid_follow_sched_process_fork(void *data,
6384 				     struct task_struct *self,
6385 				     struct task_struct *task)
6386 {
6387 	struct trace_pid_list *pid_list;
6388 	struct trace_array *tr = data;
6389 
6390 	pid_list = rcu_dereference_sched(tr->function_pids);
6391 	trace_filter_add_remove_task(pid_list, self, task);
6392 }
6393 
6394 static void
6395 ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
6396 {
6397 	struct trace_pid_list *pid_list;
6398 	struct trace_array *tr = data;
6399 
6400 	pid_list = rcu_dereference_sched(tr->function_pids);
6401 	trace_filter_add_remove_task(pid_list, NULL, task);
6402 }
6403 
6404 void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
6405 {
6406 	if (enable) {
6407 		register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
6408 						  tr);
6409 		register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
6410 						  tr);
6411 	} else {
6412 		unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
6413 						    tr);
6414 		unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit,
6415 						    tr);
6416 	}
6417 }
6418 
6419 static void clear_ftrace_pids(struct trace_array *tr)
6420 {
6421 	struct trace_pid_list *pid_list;
6422 	int cpu;
6423 
6424 	pid_list = rcu_dereference_protected(tr->function_pids,
6425 					     lockdep_is_held(&ftrace_lock));
6426 	if (!pid_list)
6427 		return;
6428 
6429 	unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
6430 
6431 	for_each_possible_cpu(cpu)
6432 		per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
6433 
6434 	rcu_assign_pointer(tr->function_pids, NULL);
6435 
6436 	/* Wait till all users are no longer using pid filtering */
6437 	synchronize_rcu();
6438 
6439 	trace_free_pid_list(pid_list);
6440 }
6441 
6442 void ftrace_clear_pids(struct trace_array *tr)
6443 {
6444 	mutex_lock(&ftrace_lock);
6445 
6446 	clear_ftrace_pids(tr);
6447 
6448 	mutex_unlock(&ftrace_lock);
6449 }
6450 
6451 static void ftrace_pid_reset(struct trace_array *tr)
6452 {
6453 	mutex_lock(&ftrace_lock);
6454 	clear_ftrace_pids(tr);
6455 
6456 	ftrace_update_pid_func();
6457 	ftrace_startup_all(0);
6458 
6459 	mutex_unlock(&ftrace_lock);
6460 }
6461 
6462 /* Greater than any max PID */
6463 #define FTRACE_NO_PIDS		(void *)(PID_MAX_LIMIT + 1)
6464 
6465 static void *fpid_start(struct seq_file *m, loff_t *pos)
6466 	__acquires(RCU)
6467 {
6468 	struct trace_pid_list *pid_list;
6469 	struct trace_array *tr = m->private;
6470 
6471 	mutex_lock(&ftrace_lock);
6472 	rcu_read_lock_sched();
6473 
6474 	pid_list = rcu_dereference_sched(tr->function_pids);
6475 
6476 	if (!pid_list)
6477 		return !(*pos) ? FTRACE_NO_PIDS : NULL;
6478 
6479 	return trace_pid_start(pid_list, pos);
6480 }
6481 
6482 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
6483 {
6484 	struct trace_array *tr = m->private;
6485 	struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
6486 
6487 	if (v == FTRACE_NO_PIDS)
6488 		return NULL;
6489 
6490 	return trace_pid_next(pid_list, v, pos);
6491 }
6492 
6493 static void fpid_stop(struct seq_file *m, void *p)
6494 	__releases(RCU)
6495 {
6496 	rcu_read_unlock_sched();
6497 	mutex_unlock(&ftrace_lock);
6498 }
6499 
6500 static int fpid_show(struct seq_file *m, void *v)
6501 {
6502 	if (v == FTRACE_NO_PIDS) {
6503 		seq_puts(m, "no pid\n");
6504 		return 0;
6505 	}
6506 
6507 	return trace_pid_show(m, v);
6508 }
6509 
6510 static const struct seq_operations ftrace_pid_sops = {
6511 	.start = fpid_start,
6512 	.next = fpid_next,
6513 	.stop = fpid_stop,
6514 	.show = fpid_show,
6515 };
6516 
6517 static int
6518 ftrace_pid_open(struct inode *inode, struct file *file)
6519 {
6520 	struct trace_array *tr = inode->i_private;
6521 	struct seq_file *m;
6522 	int ret = 0;
6523 
6524 	if (trace_array_get(tr) < 0)
6525 		return -ENODEV;
6526 
6527 	if ((file->f_mode & FMODE_WRITE) &&
6528 	    (file->f_flags & O_TRUNC))
6529 		ftrace_pid_reset(tr);
6530 
6531 	ret = seq_open(file, &ftrace_pid_sops);
6532 	if (ret < 0) {
6533 		trace_array_put(tr);
6534 	} else {
6535 		m = file->private_data;
6536 		/* copy tr over to seq ops */
6537 		m->private = tr;
6538 	}
6539 
6540 	return ret;
6541 }
6542 
6543 static void ignore_task_cpu(void *data)
6544 {
6545 	struct trace_array *tr = data;
6546 	struct trace_pid_list *pid_list;
6547 
6548 	/*
6549 	 * This function is called by on_each_cpu() while the
6550 	 * event_mutex is held.
6551 	 */
6552 	pid_list = rcu_dereference_protected(tr->function_pids,
6553 					     mutex_is_locked(&ftrace_lock));
6554 
6555 	this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
6556 		       trace_ignore_this_task(pid_list, current));
6557 }
6558 
6559 static ssize_t
6560 ftrace_pid_write(struct file *filp, const char __user *ubuf,
6561 		   size_t cnt, loff_t *ppos)
6562 {
6563 	struct seq_file *m = filp->private_data;
6564 	struct trace_array *tr = m->private;
6565 	struct trace_pid_list *filtered_pids = NULL;
6566 	struct trace_pid_list *pid_list;
6567 	ssize_t ret;
6568 
6569 	if (!cnt)
6570 		return 0;
6571 
6572 	mutex_lock(&ftrace_lock);
6573 
6574 	filtered_pids = rcu_dereference_protected(tr->function_pids,
6575 					     lockdep_is_held(&ftrace_lock));
6576 
6577 	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
6578 	if (ret < 0)
6579 		goto out;
6580 
6581 	rcu_assign_pointer(tr->function_pids, pid_list);
6582 
6583 	if (filtered_pids) {
6584 		synchronize_rcu();
6585 		trace_free_pid_list(filtered_pids);
6586 	} else if (pid_list) {
6587 		/* Register a probe to set whether to ignore the tracing of a task */
6588 		register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
6589 	}
6590 
6591 	/*
6592 	 * Ignoring of pids is done at task switch. But we have to
6593 	 * check for those tasks that are currently running.
6594 	 * Always do this in case a pid was appended or removed.
6595 	 */
6596 	on_each_cpu(ignore_task_cpu, tr, 1);
6597 
6598 	ftrace_update_pid_func();
6599 	ftrace_startup_all(0);
6600  out:
6601 	mutex_unlock(&ftrace_lock);
6602 
6603 	if (ret > 0)
6604 		*ppos += ret;
6605 
6606 	return ret;
6607 }
6608 
6609 static int
6610 ftrace_pid_release(struct inode *inode, struct file *file)
6611 {
6612 	struct trace_array *tr = inode->i_private;
6613 
6614 	trace_array_put(tr);
6615 
6616 	return seq_release(inode, file);
6617 }
6618 
6619 static const struct file_operations ftrace_pid_fops = {
6620 	.open		= ftrace_pid_open,
6621 	.write		= ftrace_pid_write,
6622 	.read		= seq_read,
6623 	.llseek		= tracing_lseek,
6624 	.release	= ftrace_pid_release,
6625 };
6626 
6627 void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6628 {
6629 	trace_create_file("set_ftrace_pid", 0644, d_tracer,
6630 			    tr, &ftrace_pid_fops);
6631 }
6632 
6633 void __init ftrace_init_tracefs_toplevel(struct trace_array *tr,
6634 					 struct dentry *d_tracer)
6635 {
6636 	/* Only the top level directory has the dyn_tracefs and profile */
6637 	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
6638 
6639 	ftrace_init_dyn_tracefs(d_tracer);
6640 	ftrace_profile_tracefs(d_tracer);
6641 }
6642 
6643 /**
6644  * ftrace_kill - kill ftrace
6645  *
6646  * This function should be used by panic code. It stops ftrace
6647  * but in a not so nice way. If you need to simply kill ftrace
6648  * from a non-atomic section, use ftrace_kill.
6649  */
6650 void ftrace_kill(void)
6651 {
6652 	ftrace_disabled = 1;
6653 	ftrace_enabled = 0;
6654 	ftrace_trace_function = ftrace_stub;
6655 }
6656 
6657 /**
6658  * Test if ftrace is dead or not.
6659  */
6660 int ftrace_is_dead(void)
6661 {
6662 	return ftrace_disabled;
6663 }
6664 
6665 /**
6666  * register_ftrace_function - register a function for profiling
6667  * @ops - ops structure that holds the function for profiling.
6668  *
6669  * Register a function to be called by all functions in the
6670  * kernel.
6671  *
6672  * Note: @ops->func and all the functions it calls must be labeled
6673  *       with "notrace", otherwise it will go into a
6674  *       recursive loop.
6675  */
6676 int register_ftrace_function(struct ftrace_ops *ops)
6677 {
6678 	int ret = -1;
6679 
6680 	ftrace_ops_init(ops);
6681 
6682 	mutex_lock(&ftrace_lock);
6683 
6684 	ret = ftrace_startup(ops, 0);
6685 
6686 	mutex_unlock(&ftrace_lock);
6687 
6688 	return ret;
6689 }
6690 EXPORT_SYMBOL_GPL(register_ftrace_function);
6691 
6692 /**
6693  * unregister_ftrace_function - unregister a function for profiling.
6694  * @ops - ops structure that holds the function to unregister
6695  *
6696  * Unregister a function that was added to be called by ftrace profiling.
6697  */
6698 int unregister_ftrace_function(struct ftrace_ops *ops)
6699 {
6700 	int ret;
6701 
6702 	mutex_lock(&ftrace_lock);
6703 	ret = ftrace_shutdown(ops, 0);
6704 	mutex_unlock(&ftrace_lock);
6705 
6706 	return ret;
6707 }
6708 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
6709 
6710 int
6711 ftrace_enable_sysctl(struct ctl_table *table, int write,
6712 		     void __user *buffer, size_t *lenp,
6713 		     loff_t *ppos)
6714 {
6715 	int ret = -ENODEV;
6716 
6717 	mutex_lock(&ftrace_lock);
6718 
6719 	if (unlikely(ftrace_disabled))
6720 		goto out;
6721 
6722 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
6723 
6724 	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
6725 		goto out;
6726 
6727 	last_ftrace_enabled = !!ftrace_enabled;
6728 
6729 	if (ftrace_enabled) {
6730 
6731 		/* we are starting ftrace again */
6732 		if (rcu_dereference_protected(ftrace_ops_list,
6733 			lockdep_is_held(&ftrace_lock)) != &ftrace_list_end)
6734 			update_ftrace_function();
6735 
6736 		ftrace_startup_sysctl();
6737 
6738 	} else {
6739 		/* stopping ftrace calls (just send to ftrace_stub) */
6740 		ftrace_trace_function = ftrace_stub;
6741 
6742 		ftrace_shutdown_sysctl();
6743 	}
6744 
6745  out:
6746 	mutex_unlock(&ftrace_lock);
6747 	return ret;
6748 }
6749