xref: /linux/kernel/trace/ftrace.c (revision 26b0d14106954ae46d2f4f7eec3481828a210f7d)
1 /*
2  * Infrastructure for profiling code inserted by 'gcc -pg'.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally ported from the -rt patch by:
8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code in the latency_tracer, that is:
11  *
12  *  Copyright (C) 2004-2006 Ingo Molnar
13  *  Copyright (C) 2004 William Lee Irwin III
14  */
15 
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/bsearch.h>
26 #include <linux/module.h>
27 #include <linux/ftrace.h>
28 #include <linux/sysctl.h>
29 #include <linux/slab.h>
30 #include <linux/ctype.h>
31 #include <linux/sort.h>
32 #include <linux/list.h>
33 #include <linux/hash.h>
34 #include <linux/rcupdate.h>
35 
36 #include <trace/events/sched.h>
37 
38 #include <asm/setup.h>
39 
40 #include "trace_output.h"
41 #include "trace_stat.h"
42 
43 #define FTRACE_WARN_ON(cond)			\
44 	({					\
45 		int ___r = cond;		\
46 		if (WARN_ON(___r))		\
47 			ftrace_kill();		\
48 		___r;				\
49 	})
50 
51 #define FTRACE_WARN_ON_ONCE(cond)		\
52 	({					\
53 		int ___r = cond;		\
54 		if (WARN_ON_ONCE(___r))		\
55 			ftrace_kill();		\
56 		___r;				\
57 	})
58 
59 /* hash bits for specific function selection */
60 #define FTRACE_HASH_BITS 7
61 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
62 #define FTRACE_HASH_DEFAULT_BITS 10
63 #define FTRACE_HASH_MAX_BITS 12
64 
65 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
66 
67 /* ftrace_enabled is a method to turn ftrace on or off */
68 int ftrace_enabled __read_mostly;
69 static int last_ftrace_enabled;
70 
71 /* Quick disabling of function tracer. */
72 int function_trace_stop;
73 
74 /* List for set_ftrace_pid's pids. */
75 LIST_HEAD(ftrace_pids);
76 struct ftrace_pid {
77 	struct list_head list;
78 	struct pid *pid;
79 };
80 
81 /*
82  * ftrace_disabled is set when an anomaly is discovered.
83  * ftrace_disabled is much stronger than ftrace_enabled.
84  */
85 static int ftrace_disabled __read_mostly;
86 
87 static DEFINE_MUTEX(ftrace_lock);
88 
89 static struct ftrace_ops ftrace_list_end __read_mostly = {
90 	.func		= ftrace_stub,
91 };
92 
93 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
94 static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
95 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
96 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
97 static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
98 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
99 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
100 static struct ftrace_ops global_ops;
101 static struct ftrace_ops control_ops;
102 
103 static void
104 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
105 
106 /*
107  * Traverse the ftrace_global_list, invoking all entries.  The reason that we
108  * can use rcu_dereference_raw() is that elements removed from this list
109  * are simply leaked, so there is no need to interact with a grace-period
110  * mechanism.  The rcu_dereference_raw() calls are needed to handle
111  * concurrent insertions into the ftrace_global_list.
112  *
113  * Silly Alpha and silly pointer-speculation compiler optimizations!
114  */
115 static void ftrace_global_list_func(unsigned long ip,
116 				    unsigned long parent_ip)
117 {
118 	struct ftrace_ops *op;
119 
120 	if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
121 		return;
122 
123 	trace_recursion_set(TRACE_GLOBAL_BIT);
124 	op = rcu_dereference_raw(ftrace_global_list); /*see above*/
125 	while (op != &ftrace_list_end) {
126 		op->func(ip, parent_ip);
127 		op = rcu_dereference_raw(op->next); /*see above*/
128 	};
129 	trace_recursion_clear(TRACE_GLOBAL_BIT);
130 }
131 
132 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
133 {
134 	if (!test_tsk_trace_trace(current))
135 		return;
136 
137 	ftrace_pid_function(ip, parent_ip);
138 }
139 
140 static void set_ftrace_pid_function(ftrace_func_t func)
141 {
142 	/* do not set ftrace_pid_function to itself! */
143 	if (func != ftrace_pid_func)
144 		ftrace_pid_function = func;
145 }
146 
147 /**
148  * clear_ftrace_function - reset the ftrace function
149  *
150  * This NULLs the ftrace function and in essence stops
151  * tracing.  There may be lag
152  */
153 void clear_ftrace_function(void)
154 {
155 	ftrace_trace_function = ftrace_stub;
156 	__ftrace_trace_function = ftrace_stub;
157 	__ftrace_trace_function_delay = ftrace_stub;
158 	ftrace_pid_function = ftrace_stub;
159 }
160 
161 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
162 /*
163  * For those archs that do not test ftrace_trace_stop in their
164  * mcount call site, we need to do it from C.
165  */
166 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
167 {
168 	if (function_trace_stop)
169 		return;
170 
171 	__ftrace_trace_function(ip, parent_ip);
172 }
173 #endif
174 
175 static void control_ops_disable_all(struct ftrace_ops *ops)
176 {
177 	int cpu;
178 
179 	for_each_possible_cpu(cpu)
180 		*per_cpu_ptr(ops->disabled, cpu) = 1;
181 }
182 
183 static int control_ops_alloc(struct ftrace_ops *ops)
184 {
185 	int __percpu *disabled;
186 
187 	disabled = alloc_percpu(int);
188 	if (!disabled)
189 		return -ENOMEM;
190 
191 	ops->disabled = disabled;
192 	control_ops_disable_all(ops);
193 	return 0;
194 }
195 
196 static void control_ops_free(struct ftrace_ops *ops)
197 {
198 	free_percpu(ops->disabled);
199 }
200 
201 static void update_global_ops(void)
202 {
203 	ftrace_func_t func;
204 
205 	/*
206 	 * If there's only one function registered, then call that
207 	 * function directly. Otherwise, we need to iterate over the
208 	 * registered callers.
209 	 */
210 	if (ftrace_global_list == &ftrace_list_end ||
211 	    ftrace_global_list->next == &ftrace_list_end)
212 		func = ftrace_global_list->func;
213 	else
214 		func = ftrace_global_list_func;
215 
216 	/* If we filter on pids, update to use the pid function */
217 	if (!list_empty(&ftrace_pids)) {
218 		set_ftrace_pid_function(func);
219 		func = ftrace_pid_func;
220 	}
221 
222 	global_ops.func = func;
223 }
224 
225 static void update_ftrace_function(void)
226 {
227 	ftrace_func_t func;
228 
229 	update_global_ops();
230 
231 	/*
232 	 * If we are at the end of the list and this ops is
233 	 * not dynamic, then have the mcount trampoline call
234 	 * the function directly
235 	 */
236 	if (ftrace_ops_list == &ftrace_list_end ||
237 	    (ftrace_ops_list->next == &ftrace_list_end &&
238 	     !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
239 		func = ftrace_ops_list->func;
240 	else
241 		func = ftrace_ops_list_func;
242 
243 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
244 	ftrace_trace_function = func;
245 #else
246 #ifdef CONFIG_DYNAMIC_FTRACE
247 	/* do not update till all functions have been modified */
248 	__ftrace_trace_function_delay = func;
249 #else
250 	__ftrace_trace_function = func;
251 #endif
252 	ftrace_trace_function =
253 		(func == ftrace_stub) ? func : ftrace_test_stop_func;
254 #endif
255 }
256 
257 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
258 {
259 	ops->next = *list;
260 	/*
261 	 * We are entering ops into the list but another
262 	 * CPU might be walking that list. We need to make sure
263 	 * the ops->next pointer is valid before another CPU sees
264 	 * the ops pointer included into the list.
265 	 */
266 	rcu_assign_pointer(*list, ops);
267 }
268 
269 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
270 {
271 	struct ftrace_ops **p;
272 
273 	/*
274 	 * If we are removing the last function, then simply point
275 	 * to the ftrace_stub.
276 	 */
277 	if (*list == ops && ops->next == &ftrace_list_end) {
278 		*list = &ftrace_list_end;
279 		return 0;
280 	}
281 
282 	for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
283 		if (*p == ops)
284 			break;
285 
286 	if (*p != ops)
287 		return -1;
288 
289 	*p = (*p)->next;
290 	return 0;
291 }
292 
293 static void add_ftrace_list_ops(struct ftrace_ops **list,
294 				struct ftrace_ops *main_ops,
295 				struct ftrace_ops *ops)
296 {
297 	int first = *list == &ftrace_list_end;
298 	add_ftrace_ops(list, ops);
299 	if (first)
300 		add_ftrace_ops(&ftrace_ops_list, main_ops);
301 }
302 
303 static int remove_ftrace_list_ops(struct ftrace_ops **list,
304 				  struct ftrace_ops *main_ops,
305 				  struct ftrace_ops *ops)
306 {
307 	int ret = remove_ftrace_ops(list, ops);
308 	if (!ret && *list == &ftrace_list_end)
309 		ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
310 	return ret;
311 }
312 
313 static int __register_ftrace_function(struct ftrace_ops *ops)
314 {
315 	if (ftrace_disabled)
316 		return -ENODEV;
317 
318 	if (FTRACE_WARN_ON(ops == &global_ops))
319 		return -EINVAL;
320 
321 	if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
322 		return -EBUSY;
323 
324 	/* We don't support both control and global flags set. */
325 	if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
326 		return -EINVAL;
327 
328 	if (!core_kernel_data((unsigned long)ops))
329 		ops->flags |= FTRACE_OPS_FL_DYNAMIC;
330 
331 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
332 		add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
333 		ops->flags |= FTRACE_OPS_FL_ENABLED;
334 	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
335 		if (control_ops_alloc(ops))
336 			return -ENOMEM;
337 		add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
338 	} else
339 		add_ftrace_ops(&ftrace_ops_list, ops);
340 
341 	if (ftrace_enabled)
342 		update_ftrace_function();
343 
344 	return 0;
345 }
346 
347 static int __unregister_ftrace_function(struct ftrace_ops *ops)
348 {
349 	int ret;
350 
351 	if (ftrace_disabled)
352 		return -ENODEV;
353 
354 	if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
355 		return -EBUSY;
356 
357 	if (FTRACE_WARN_ON(ops == &global_ops))
358 		return -EINVAL;
359 
360 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
361 		ret = remove_ftrace_list_ops(&ftrace_global_list,
362 					     &global_ops, ops);
363 		if (!ret)
364 			ops->flags &= ~FTRACE_OPS_FL_ENABLED;
365 	} else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
366 		ret = remove_ftrace_list_ops(&ftrace_control_list,
367 					     &control_ops, ops);
368 		if (!ret) {
369 			/*
370 			 * The ftrace_ops is now removed from the list,
371 			 * so there'll be no new users. We must ensure
372 			 * all current users are done before we free
373 			 * the control data.
374 			 */
375 			synchronize_sched();
376 			control_ops_free(ops);
377 		}
378 	} else
379 		ret = remove_ftrace_ops(&ftrace_ops_list, ops);
380 
381 	if (ret < 0)
382 		return ret;
383 
384 	if (ftrace_enabled)
385 		update_ftrace_function();
386 
387 	/*
388 	 * Dynamic ops may be freed, we must make sure that all
389 	 * callers are done before leaving this function.
390 	 */
391 	if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
392 		synchronize_sched();
393 
394 	return 0;
395 }
396 
397 static void ftrace_update_pid_func(void)
398 {
399 	/* Only do something if we are tracing something */
400 	if (ftrace_trace_function == ftrace_stub)
401 		return;
402 
403 	update_ftrace_function();
404 }
405 
406 #ifdef CONFIG_FUNCTION_PROFILER
407 struct ftrace_profile {
408 	struct hlist_node		node;
409 	unsigned long			ip;
410 	unsigned long			counter;
411 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
412 	unsigned long long		time;
413 	unsigned long long		time_squared;
414 #endif
415 };
416 
417 struct ftrace_profile_page {
418 	struct ftrace_profile_page	*next;
419 	unsigned long			index;
420 	struct ftrace_profile		records[];
421 };
422 
423 struct ftrace_profile_stat {
424 	atomic_t			disabled;
425 	struct hlist_head		*hash;
426 	struct ftrace_profile_page	*pages;
427 	struct ftrace_profile_page	*start;
428 	struct tracer_stat		stat;
429 };
430 
431 #define PROFILE_RECORDS_SIZE						\
432 	(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
433 
434 #define PROFILES_PER_PAGE					\
435 	(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
436 
437 static int ftrace_profile_bits __read_mostly;
438 static int ftrace_profile_enabled __read_mostly;
439 
440 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
441 static DEFINE_MUTEX(ftrace_profile_lock);
442 
443 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
444 
445 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
446 
447 static void *
448 function_stat_next(void *v, int idx)
449 {
450 	struct ftrace_profile *rec = v;
451 	struct ftrace_profile_page *pg;
452 
453 	pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
454 
455  again:
456 	if (idx != 0)
457 		rec++;
458 
459 	if ((void *)rec >= (void *)&pg->records[pg->index]) {
460 		pg = pg->next;
461 		if (!pg)
462 			return NULL;
463 		rec = &pg->records[0];
464 		if (!rec->counter)
465 			goto again;
466 	}
467 
468 	return rec;
469 }
470 
471 static void *function_stat_start(struct tracer_stat *trace)
472 {
473 	struct ftrace_profile_stat *stat =
474 		container_of(trace, struct ftrace_profile_stat, stat);
475 
476 	if (!stat || !stat->start)
477 		return NULL;
478 
479 	return function_stat_next(&stat->start->records[0], 0);
480 }
481 
482 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
483 /* function graph compares on total time */
484 static int function_stat_cmp(void *p1, void *p2)
485 {
486 	struct ftrace_profile *a = p1;
487 	struct ftrace_profile *b = p2;
488 
489 	if (a->time < b->time)
490 		return -1;
491 	if (a->time > b->time)
492 		return 1;
493 	else
494 		return 0;
495 }
496 #else
497 /* not function graph compares against hits */
498 static int function_stat_cmp(void *p1, void *p2)
499 {
500 	struct ftrace_profile *a = p1;
501 	struct ftrace_profile *b = p2;
502 
503 	if (a->counter < b->counter)
504 		return -1;
505 	if (a->counter > b->counter)
506 		return 1;
507 	else
508 		return 0;
509 }
510 #endif
511 
512 static int function_stat_headers(struct seq_file *m)
513 {
514 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
515 	seq_printf(m, "  Function                               "
516 		   "Hit    Time            Avg             s^2\n"
517 		      "  --------                               "
518 		   "---    ----            ---             ---\n");
519 #else
520 	seq_printf(m, "  Function                               Hit\n"
521 		      "  --------                               ---\n");
522 #endif
523 	return 0;
524 }
525 
526 static int function_stat_show(struct seq_file *m, void *v)
527 {
528 	struct ftrace_profile *rec = v;
529 	char str[KSYM_SYMBOL_LEN];
530 	int ret = 0;
531 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
532 	static struct trace_seq s;
533 	unsigned long long avg;
534 	unsigned long long stddev;
535 #endif
536 	mutex_lock(&ftrace_profile_lock);
537 
538 	/* we raced with function_profile_reset() */
539 	if (unlikely(rec->counter == 0)) {
540 		ret = -EBUSY;
541 		goto out;
542 	}
543 
544 	kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
545 	seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
546 
547 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
548 	seq_printf(m, "    ");
549 	avg = rec->time;
550 	do_div(avg, rec->counter);
551 
552 	/* Sample standard deviation (s^2) */
553 	if (rec->counter <= 1)
554 		stddev = 0;
555 	else {
556 		stddev = rec->time_squared - rec->counter * avg * avg;
557 		/*
558 		 * Divide only 1000 for ns^2 -> us^2 conversion.
559 		 * trace_print_graph_duration will divide 1000 again.
560 		 */
561 		do_div(stddev, (rec->counter - 1) * 1000);
562 	}
563 
564 	trace_seq_init(&s);
565 	trace_print_graph_duration(rec->time, &s);
566 	trace_seq_puts(&s, "    ");
567 	trace_print_graph_duration(avg, &s);
568 	trace_seq_puts(&s, "    ");
569 	trace_print_graph_duration(stddev, &s);
570 	trace_print_seq(m, &s);
571 #endif
572 	seq_putc(m, '\n');
573 out:
574 	mutex_unlock(&ftrace_profile_lock);
575 
576 	return ret;
577 }
578 
579 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
580 {
581 	struct ftrace_profile_page *pg;
582 
583 	pg = stat->pages = stat->start;
584 
585 	while (pg) {
586 		memset(pg->records, 0, PROFILE_RECORDS_SIZE);
587 		pg->index = 0;
588 		pg = pg->next;
589 	}
590 
591 	memset(stat->hash, 0,
592 	       FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
593 }
594 
595 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
596 {
597 	struct ftrace_profile_page *pg;
598 	int functions;
599 	int pages;
600 	int i;
601 
602 	/* If we already allocated, do nothing */
603 	if (stat->pages)
604 		return 0;
605 
606 	stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
607 	if (!stat->pages)
608 		return -ENOMEM;
609 
610 #ifdef CONFIG_DYNAMIC_FTRACE
611 	functions = ftrace_update_tot_cnt;
612 #else
613 	/*
614 	 * We do not know the number of functions that exist because
615 	 * dynamic tracing is what counts them. With past experience
616 	 * we have around 20K functions. That should be more than enough.
617 	 * It is highly unlikely we will execute every function in
618 	 * the kernel.
619 	 */
620 	functions = 20000;
621 #endif
622 
623 	pg = stat->start = stat->pages;
624 
625 	pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
626 
627 	for (i = 0; i < pages; i++) {
628 		pg->next = (void *)get_zeroed_page(GFP_KERNEL);
629 		if (!pg->next)
630 			goto out_free;
631 		pg = pg->next;
632 	}
633 
634 	return 0;
635 
636  out_free:
637 	pg = stat->start;
638 	while (pg) {
639 		unsigned long tmp = (unsigned long)pg;
640 
641 		pg = pg->next;
642 		free_page(tmp);
643 	}
644 
645 	free_page((unsigned long)stat->pages);
646 	stat->pages = NULL;
647 	stat->start = NULL;
648 
649 	return -ENOMEM;
650 }
651 
652 static int ftrace_profile_init_cpu(int cpu)
653 {
654 	struct ftrace_profile_stat *stat;
655 	int size;
656 
657 	stat = &per_cpu(ftrace_profile_stats, cpu);
658 
659 	if (stat->hash) {
660 		/* If the profile is already created, simply reset it */
661 		ftrace_profile_reset(stat);
662 		return 0;
663 	}
664 
665 	/*
666 	 * We are profiling all functions, but usually only a few thousand
667 	 * functions are hit. We'll make a hash of 1024 items.
668 	 */
669 	size = FTRACE_PROFILE_HASH_SIZE;
670 
671 	stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
672 
673 	if (!stat->hash)
674 		return -ENOMEM;
675 
676 	if (!ftrace_profile_bits) {
677 		size--;
678 
679 		for (; size; size >>= 1)
680 			ftrace_profile_bits++;
681 	}
682 
683 	/* Preallocate the function profiling pages */
684 	if (ftrace_profile_pages_init(stat) < 0) {
685 		kfree(stat->hash);
686 		stat->hash = NULL;
687 		return -ENOMEM;
688 	}
689 
690 	return 0;
691 }
692 
693 static int ftrace_profile_init(void)
694 {
695 	int cpu;
696 	int ret = 0;
697 
698 	for_each_online_cpu(cpu) {
699 		ret = ftrace_profile_init_cpu(cpu);
700 		if (ret)
701 			break;
702 	}
703 
704 	return ret;
705 }
706 
707 /* interrupts must be disabled */
708 static struct ftrace_profile *
709 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
710 {
711 	struct ftrace_profile *rec;
712 	struct hlist_head *hhd;
713 	struct hlist_node *n;
714 	unsigned long key;
715 
716 	key = hash_long(ip, ftrace_profile_bits);
717 	hhd = &stat->hash[key];
718 
719 	if (hlist_empty(hhd))
720 		return NULL;
721 
722 	hlist_for_each_entry_rcu(rec, n, hhd, node) {
723 		if (rec->ip == ip)
724 			return rec;
725 	}
726 
727 	return NULL;
728 }
729 
730 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
731 			       struct ftrace_profile *rec)
732 {
733 	unsigned long key;
734 
735 	key = hash_long(rec->ip, ftrace_profile_bits);
736 	hlist_add_head_rcu(&rec->node, &stat->hash[key]);
737 }
738 
739 /*
740  * The memory is already allocated, this simply finds a new record to use.
741  */
742 static struct ftrace_profile *
743 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
744 {
745 	struct ftrace_profile *rec = NULL;
746 
747 	/* prevent recursion (from NMIs) */
748 	if (atomic_inc_return(&stat->disabled) != 1)
749 		goto out;
750 
751 	/*
752 	 * Try to find the function again since an NMI
753 	 * could have added it
754 	 */
755 	rec = ftrace_find_profiled_func(stat, ip);
756 	if (rec)
757 		goto out;
758 
759 	if (stat->pages->index == PROFILES_PER_PAGE) {
760 		if (!stat->pages->next)
761 			goto out;
762 		stat->pages = stat->pages->next;
763 	}
764 
765 	rec = &stat->pages->records[stat->pages->index++];
766 	rec->ip = ip;
767 	ftrace_add_profile(stat, rec);
768 
769  out:
770 	atomic_dec(&stat->disabled);
771 
772 	return rec;
773 }
774 
775 static void
776 function_profile_call(unsigned long ip, unsigned long parent_ip)
777 {
778 	struct ftrace_profile_stat *stat;
779 	struct ftrace_profile *rec;
780 	unsigned long flags;
781 
782 	if (!ftrace_profile_enabled)
783 		return;
784 
785 	local_irq_save(flags);
786 
787 	stat = &__get_cpu_var(ftrace_profile_stats);
788 	if (!stat->hash || !ftrace_profile_enabled)
789 		goto out;
790 
791 	rec = ftrace_find_profiled_func(stat, ip);
792 	if (!rec) {
793 		rec = ftrace_profile_alloc(stat, ip);
794 		if (!rec)
795 			goto out;
796 	}
797 
798 	rec->counter++;
799  out:
800 	local_irq_restore(flags);
801 }
802 
803 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
804 static int profile_graph_entry(struct ftrace_graph_ent *trace)
805 {
806 	function_profile_call(trace->func, 0);
807 	return 1;
808 }
809 
810 static void profile_graph_return(struct ftrace_graph_ret *trace)
811 {
812 	struct ftrace_profile_stat *stat;
813 	unsigned long long calltime;
814 	struct ftrace_profile *rec;
815 	unsigned long flags;
816 
817 	local_irq_save(flags);
818 	stat = &__get_cpu_var(ftrace_profile_stats);
819 	if (!stat->hash || !ftrace_profile_enabled)
820 		goto out;
821 
822 	/* If the calltime was zero'd ignore it */
823 	if (!trace->calltime)
824 		goto out;
825 
826 	calltime = trace->rettime - trace->calltime;
827 
828 	if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
829 		int index;
830 
831 		index = trace->depth;
832 
833 		/* Append this call time to the parent time to subtract */
834 		if (index)
835 			current->ret_stack[index - 1].subtime += calltime;
836 
837 		if (current->ret_stack[index].subtime < calltime)
838 			calltime -= current->ret_stack[index].subtime;
839 		else
840 			calltime = 0;
841 	}
842 
843 	rec = ftrace_find_profiled_func(stat, trace->func);
844 	if (rec) {
845 		rec->time += calltime;
846 		rec->time_squared += calltime * calltime;
847 	}
848 
849  out:
850 	local_irq_restore(flags);
851 }
852 
853 static int register_ftrace_profiler(void)
854 {
855 	return register_ftrace_graph(&profile_graph_return,
856 				     &profile_graph_entry);
857 }
858 
859 static void unregister_ftrace_profiler(void)
860 {
861 	unregister_ftrace_graph();
862 }
863 #else
864 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
865 	.func		= function_profile_call,
866 };
867 
868 static int register_ftrace_profiler(void)
869 {
870 	return register_ftrace_function(&ftrace_profile_ops);
871 }
872 
873 static void unregister_ftrace_profiler(void)
874 {
875 	unregister_ftrace_function(&ftrace_profile_ops);
876 }
877 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
878 
879 static ssize_t
880 ftrace_profile_write(struct file *filp, const char __user *ubuf,
881 		     size_t cnt, loff_t *ppos)
882 {
883 	unsigned long val;
884 	int ret;
885 
886 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
887 	if (ret)
888 		return ret;
889 
890 	val = !!val;
891 
892 	mutex_lock(&ftrace_profile_lock);
893 	if (ftrace_profile_enabled ^ val) {
894 		if (val) {
895 			ret = ftrace_profile_init();
896 			if (ret < 0) {
897 				cnt = ret;
898 				goto out;
899 			}
900 
901 			ret = register_ftrace_profiler();
902 			if (ret < 0) {
903 				cnt = ret;
904 				goto out;
905 			}
906 			ftrace_profile_enabled = 1;
907 		} else {
908 			ftrace_profile_enabled = 0;
909 			/*
910 			 * unregister_ftrace_profiler calls stop_machine
911 			 * so this acts like an synchronize_sched.
912 			 */
913 			unregister_ftrace_profiler();
914 		}
915 	}
916  out:
917 	mutex_unlock(&ftrace_profile_lock);
918 
919 	*ppos += cnt;
920 
921 	return cnt;
922 }
923 
924 static ssize_t
925 ftrace_profile_read(struct file *filp, char __user *ubuf,
926 		     size_t cnt, loff_t *ppos)
927 {
928 	char buf[64];		/* big enough to hold a number */
929 	int r;
930 
931 	r = sprintf(buf, "%u\n", ftrace_profile_enabled);
932 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
933 }
934 
935 static const struct file_operations ftrace_profile_fops = {
936 	.open		= tracing_open_generic,
937 	.read		= ftrace_profile_read,
938 	.write		= ftrace_profile_write,
939 	.llseek		= default_llseek,
940 };
941 
942 /* used to initialize the real stat files */
943 static struct tracer_stat function_stats __initdata = {
944 	.name		= "functions",
945 	.stat_start	= function_stat_start,
946 	.stat_next	= function_stat_next,
947 	.stat_cmp	= function_stat_cmp,
948 	.stat_headers	= function_stat_headers,
949 	.stat_show	= function_stat_show
950 };
951 
952 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
953 {
954 	struct ftrace_profile_stat *stat;
955 	struct dentry *entry;
956 	char *name;
957 	int ret;
958 	int cpu;
959 
960 	for_each_possible_cpu(cpu) {
961 		stat = &per_cpu(ftrace_profile_stats, cpu);
962 
963 		/* allocate enough for function name + cpu number */
964 		name = kmalloc(32, GFP_KERNEL);
965 		if (!name) {
966 			/*
967 			 * The files created are permanent, if something happens
968 			 * we still do not free memory.
969 			 */
970 			WARN(1,
971 			     "Could not allocate stat file for cpu %d\n",
972 			     cpu);
973 			return;
974 		}
975 		stat->stat = function_stats;
976 		snprintf(name, 32, "function%d", cpu);
977 		stat->stat.name = name;
978 		ret = register_stat_tracer(&stat->stat);
979 		if (ret) {
980 			WARN(1,
981 			     "Could not register function stat for cpu %d\n",
982 			     cpu);
983 			kfree(name);
984 			return;
985 		}
986 	}
987 
988 	entry = debugfs_create_file("function_profile_enabled", 0644,
989 				    d_tracer, NULL, &ftrace_profile_fops);
990 	if (!entry)
991 		pr_warning("Could not create debugfs "
992 			   "'function_profile_enabled' entry\n");
993 }
994 
995 #else /* CONFIG_FUNCTION_PROFILER */
996 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
997 {
998 }
999 #endif /* CONFIG_FUNCTION_PROFILER */
1000 
1001 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1002 
1003 #ifdef CONFIG_DYNAMIC_FTRACE
1004 
1005 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
1006 # error Dynamic ftrace depends on MCOUNT_RECORD
1007 #endif
1008 
1009 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1010 
1011 struct ftrace_func_probe {
1012 	struct hlist_node	node;
1013 	struct ftrace_probe_ops	*ops;
1014 	unsigned long		flags;
1015 	unsigned long		ip;
1016 	void			*data;
1017 	struct rcu_head		rcu;
1018 };
1019 
1020 struct ftrace_func_entry {
1021 	struct hlist_node hlist;
1022 	unsigned long ip;
1023 };
1024 
1025 struct ftrace_hash {
1026 	unsigned long		size_bits;
1027 	struct hlist_head	*buckets;
1028 	unsigned long		count;
1029 	struct rcu_head		rcu;
1030 };
1031 
1032 /*
1033  * We make these constant because no one should touch them,
1034  * but they are used as the default "empty hash", to avoid allocating
1035  * it all the time. These are in a read only section such that if
1036  * anyone does try to modify it, it will cause an exception.
1037  */
1038 static const struct hlist_head empty_buckets[1];
1039 static const struct ftrace_hash empty_hash = {
1040 	.buckets = (struct hlist_head *)empty_buckets,
1041 };
1042 #define EMPTY_HASH	((struct ftrace_hash *)&empty_hash)
1043 
1044 static struct ftrace_ops global_ops = {
1045 	.func			= ftrace_stub,
1046 	.notrace_hash		= EMPTY_HASH,
1047 	.filter_hash		= EMPTY_HASH,
1048 };
1049 
1050 static DEFINE_MUTEX(ftrace_regex_lock);
1051 
1052 struct ftrace_page {
1053 	struct ftrace_page	*next;
1054 	struct dyn_ftrace	*records;
1055 	int			index;
1056 	int			size;
1057 };
1058 
1059 static struct ftrace_page *ftrace_new_pgs;
1060 
1061 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
1062 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1063 
1064 /* estimate from running different kernels */
1065 #define NR_TO_INIT		10000
1066 
1067 static struct ftrace_page	*ftrace_pages_start;
1068 static struct ftrace_page	*ftrace_pages;
1069 
1070 static bool ftrace_hash_empty(struct ftrace_hash *hash)
1071 {
1072 	return !hash || !hash->count;
1073 }
1074 
1075 static struct ftrace_func_entry *
1076 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1077 {
1078 	unsigned long key;
1079 	struct ftrace_func_entry *entry;
1080 	struct hlist_head *hhd;
1081 	struct hlist_node *n;
1082 
1083 	if (ftrace_hash_empty(hash))
1084 		return NULL;
1085 
1086 	if (hash->size_bits > 0)
1087 		key = hash_long(ip, hash->size_bits);
1088 	else
1089 		key = 0;
1090 
1091 	hhd = &hash->buckets[key];
1092 
1093 	hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1094 		if (entry->ip == ip)
1095 			return entry;
1096 	}
1097 	return NULL;
1098 }
1099 
1100 static void __add_hash_entry(struct ftrace_hash *hash,
1101 			     struct ftrace_func_entry *entry)
1102 {
1103 	struct hlist_head *hhd;
1104 	unsigned long key;
1105 
1106 	if (hash->size_bits)
1107 		key = hash_long(entry->ip, hash->size_bits);
1108 	else
1109 		key = 0;
1110 
1111 	hhd = &hash->buckets[key];
1112 	hlist_add_head(&entry->hlist, hhd);
1113 	hash->count++;
1114 }
1115 
1116 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1117 {
1118 	struct ftrace_func_entry *entry;
1119 
1120 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1121 	if (!entry)
1122 		return -ENOMEM;
1123 
1124 	entry->ip = ip;
1125 	__add_hash_entry(hash, entry);
1126 
1127 	return 0;
1128 }
1129 
1130 static void
1131 free_hash_entry(struct ftrace_hash *hash,
1132 		  struct ftrace_func_entry *entry)
1133 {
1134 	hlist_del(&entry->hlist);
1135 	kfree(entry);
1136 	hash->count--;
1137 }
1138 
1139 static void
1140 remove_hash_entry(struct ftrace_hash *hash,
1141 		  struct ftrace_func_entry *entry)
1142 {
1143 	hlist_del(&entry->hlist);
1144 	hash->count--;
1145 }
1146 
1147 static void ftrace_hash_clear(struct ftrace_hash *hash)
1148 {
1149 	struct hlist_head *hhd;
1150 	struct hlist_node *tp, *tn;
1151 	struct ftrace_func_entry *entry;
1152 	int size = 1 << hash->size_bits;
1153 	int i;
1154 
1155 	if (!hash->count)
1156 		return;
1157 
1158 	for (i = 0; i < size; i++) {
1159 		hhd = &hash->buckets[i];
1160 		hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1161 			free_hash_entry(hash, entry);
1162 	}
1163 	FTRACE_WARN_ON(hash->count);
1164 }
1165 
1166 static void free_ftrace_hash(struct ftrace_hash *hash)
1167 {
1168 	if (!hash || hash == EMPTY_HASH)
1169 		return;
1170 	ftrace_hash_clear(hash);
1171 	kfree(hash->buckets);
1172 	kfree(hash);
1173 }
1174 
1175 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1176 {
1177 	struct ftrace_hash *hash;
1178 
1179 	hash = container_of(rcu, struct ftrace_hash, rcu);
1180 	free_ftrace_hash(hash);
1181 }
1182 
1183 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1184 {
1185 	if (!hash || hash == EMPTY_HASH)
1186 		return;
1187 	call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1188 }
1189 
1190 void ftrace_free_filter(struct ftrace_ops *ops)
1191 {
1192 	free_ftrace_hash(ops->filter_hash);
1193 	free_ftrace_hash(ops->notrace_hash);
1194 }
1195 
1196 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1197 {
1198 	struct ftrace_hash *hash;
1199 	int size;
1200 
1201 	hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1202 	if (!hash)
1203 		return NULL;
1204 
1205 	size = 1 << size_bits;
1206 	hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1207 
1208 	if (!hash->buckets) {
1209 		kfree(hash);
1210 		return NULL;
1211 	}
1212 
1213 	hash->size_bits = size_bits;
1214 
1215 	return hash;
1216 }
1217 
1218 static struct ftrace_hash *
1219 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1220 {
1221 	struct ftrace_func_entry *entry;
1222 	struct ftrace_hash *new_hash;
1223 	struct hlist_node *tp;
1224 	int size;
1225 	int ret;
1226 	int i;
1227 
1228 	new_hash = alloc_ftrace_hash(size_bits);
1229 	if (!new_hash)
1230 		return NULL;
1231 
1232 	/* Empty hash? */
1233 	if (ftrace_hash_empty(hash))
1234 		return new_hash;
1235 
1236 	size = 1 << hash->size_bits;
1237 	for (i = 0; i < size; i++) {
1238 		hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1239 			ret = add_hash_entry(new_hash, entry->ip);
1240 			if (ret < 0)
1241 				goto free_hash;
1242 		}
1243 	}
1244 
1245 	FTRACE_WARN_ON(new_hash->count != hash->count);
1246 
1247 	return new_hash;
1248 
1249  free_hash:
1250 	free_ftrace_hash(new_hash);
1251 	return NULL;
1252 }
1253 
1254 static void
1255 ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1256 static void
1257 ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1258 
1259 static int
1260 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1261 		 struct ftrace_hash **dst, struct ftrace_hash *src)
1262 {
1263 	struct ftrace_func_entry *entry;
1264 	struct hlist_node *tp, *tn;
1265 	struct hlist_head *hhd;
1266 	struct ftrace_hash *old_hash;
1267 	struct ftrace_hash *new_hash;
1268 	unsigned long key;
1269 	int size = src->count;
1270 	int bits = 0;
1271 	int ret;
1272 	int i;
1273 
1274 	/*
1275 	 * Remove the current set, update the hash and add
1276 	 * them back.
1277 	 */
1278 	ftrace_hash_rec_disable(ops, enable);
1279 
1280 	/*
1281 	 * If the new source is empty, just free dst and assign it
1282 	 * the empty_hash.
1283 	 */
1284 	if (!src->count) {
1285 		free_ftrace_hash_rcu(*dst);
1286 		rcu_assign_pointer(*dst, EMPTY_HASH);
1287 		/* still need to update the function records */
1288 		ret = 0;
1289 		goto out;
1290 	}
1291 
1292 	/*
1293 	 * Make the hash size about 1/2 the # found
1294 	 */
1295 	for (size /= 2; size; size >>= 1)
1296 		bits++;
1297 
1298 	/* Don't allocate too much */
1299 	if (bits > FTRACE_HASH_MAX_BITS)
1300 		bits = FTRACE_HASH_MAX_BITS;
1301 
1302 	ret = -ENOMEM;
1303 	new_hash = alloc_ftrace_hash(bits);
1304 	if (!new_hash)
1305 		goto out;
1306 
1307 	size = 1 << src->size_bits;
1308 	for (i = 0; i < size; i++) {
1309 		hhd = &src->buckets[i];
1310 		hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1311 			if (bits > 0)
1312 				key = hash_long(entry->ip, bits);
1313 			else
1314 				key = 0;
1315 			remove_hash_entry(src, entry);
1316 			__add_hash_entry(new_hash, entry);
1317 		}
1318 	}
1319 
1320 	old_hash = *dst;
1321 	rcu_assign_pointer(*dst, new_hash);
1322 	free_ftrace_hash_rcu(old_hash);
1323 
1324 	ret = 0;
1325  out:
1326 	/*
1327 	 * Enable regardless of ret:
1328 	 *  On success, we enable the new hash.
1329 	 *  On failure, we re-enable the original hash.
1330 	 */
1331 	ftrace_hash_rec_enable(ops, enable);
1332 
1333 	return ret;
1334 }
1335 
1336 /*
1337  * Test the hashes for this ops to see if we want to call
1338  * the ops->func or not.
1339  *
1340  * It's a match if the ip is in the ops->filter_hash or
1341  * the filter_hash does not exist or is empty,
1342  *  AND
1343  * the ip is not in the ops->notrace_hash.
1344  *
1345  * This needs to be called with preemption disabled as
1346  * the hashes are freed with call_rcu_sched().
1347  */
1348 static int
1349 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1350 {
1351 	struct ftrace_hash *filter_hash;
1352 	struct ftrace_hash *notrace_hash;
1353 	int ret;
1354 
1355 	filter_hash = rcu_dereference_raw(ops->filter_hash);
1356 	notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1357 
1358 	if ((ftrace_hash_empty(filter_hash) ||
1359 	     ftrace_lookup_ip(filter_hash, ip)) &&
1360 	    (ftrace_hash_empty(notrace_hash) ||
1361 	     !ftrace_lookup_ip(notrace_hash, ip)))
1362 		ret = 1;
1363 	else
1364 		ret = 0;
1365 
1366 	return ret;
1367 }
1368 
1369 /*
1370  * This is a double for. Do not use 'break' to break out of the loop,
1371  * you must use a goto.
1372  */
1373 #define do_for_each_ftrace_rec(pg, rec)					\
1374 	for (pg = ftrace_pages_start; pg; pg = pg->next) {		\
1375 		int _____i;						\
1376 		for (_____i = 0; _____i < pg->index; _____i++) {	\
1377 			rec = &pg->records[_____i];
1378 
1379 #define while_for_each_ftrace_rec()		\
1380 		}				\
1381 	}
1382 
1383 
1384 static int ftrace_cmp_recs(const void *a, const void *b)
1385 {
1386 	const struct dyn_ftrace *key = a;
1387 	const struct dyn_ftrace *rec = b;
1388 
1389 	if (key->flags < rec->ip)
1390 		return -1;
1391 	if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1392 		return 1;
1393 	return 0;
1394 }
1395 
1396 static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1397 {
1398 	struct ftrace_page *pg;
1399 	struct dyn_ftrace *rec;
1400 	struct dyn_ftrace key;
1401 
1402 	key.ip = start;
1403 	key.flags = end;	/* overload flags, as it is unsigned long */
1404 
1405 	for (pg = ftrace_pages_start; pg; pg = pg->next) {
1406 		if (end < pg->records[0].ip ||
1407 		    start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1408 			continue;
1409 		rec = bsearch(&key, pg->records, pg->index,
1410 			      sizeof(struct dyn_ftrace),
1411 			      ftrace_cmp_recs);
1412 		if (rec)
1413 			return rec->ip;
1414 	}
1415 
1416 	return 0;
1417 }
1418 
1419 /**
1420  * ftrace_location - return true if the ip giving is a traced location
1421  * @ip: the instruction pointer to check
1422  *
1423  * Returns rec->ip if @ip given is a pointer to a ftrace location.
1424  * That is, the instruction that is either a NOP or call to
1425  * the function tracer. It checks the ftrace internal tables to
1426  * determine if the address belongs or not.
1427  */
1428 unsigned long ftrace_location(unsigned long ip)
1429 {
1430 	return ftrace_location_range(ip, ip);
1431 }
1432 
1433 /**
1434  * ftrace_text_reserved - return true if range contains an ftrace location
1435  * @start: start of range to search
1436  * @end: end of range to search (inclusive). @end points to the last byte to check.
1437  *
1438  * Returns 1 if @start and @end contains a ftrace location.
1439  * That is, the instruction that is either a NOP or call to
1440  * the function tracer. It checks the ftrace internal tables to
1441  * determine if the address belongs or not.
1442  */
1443 int ftrace_text_reserved(void *start, void *end)
1444 {
1445 	unsigned long ret;
1446 
1447 	ret = ftrace_location_range((unsigned long)start,
1448 				    (unsigned long)end);
1449 
1450 	return (int)!!ret;
1451 }
1452 
1453 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1454 				     int filter_hash,
1455 				     bool inc)
1456 {
1457 	struct ftrace_hash *hash;
1458 	struct ftrace_hash *other_hash;
1459 	struct ftrace_page *pg;
1460 	struct dyn_ftrace *rec;
1461 	int count = 0;
1462 	int all = 0;
1463 
1464 	/* Only update if the ops has been registered */
1465 	if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1466 		return;
1467 
1468 	/*
1469 	 * In the filter_hash case:
1470 	 *   If the count is zero, we update all records.
1471 	 *   Otherwise we just update the items in the hash.
1472 	 *
1473 	 * In the notrace_hash case:
1474 	 *   We enable the update in the hash.
1475 	 *   As disabling notrace means enabling the tracing,
1476 	 *   and enabling notrace means disabling, the inc variable
1477 	 *   gets inversed.
1478 	 */
1479 	if (filter_hash) {
1480 		hash = ops->filter_hash;
1481 		other_hash = ops->notrace_hash;
1482 		if (ftrace_hash_empty(hash))
1483 			all = 1;
1484 	} else {
1485 		inc = !inc;
1486 		hash = ops->notrace_hash;
1487 		other_hash = ops->filter_hash;
1488 		/*
1489 		 * If the notrace hash has no items,
1490 		 * then there's nothing to do.
1491 		 */
1492 		if (ftrace_hash_empty(hash))
1493 			return;
1494 	}
1495 
1496 	do_for_each_ftrace_rec(pg, rec) {
1497 		int in_other_hash = 0;
1498 		int in_hash = 0;
1499 		int match = 0;
1500 
1501 		if (all) {
1502 			/*
1503 			 * Only the filter_hash affects all records.
1504 			 * Update if the record is not in the notrace hash.
1505 			 */
1506 			if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1507 				match = 1;
1508 		} else {
1509 			in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1510 			in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1511 
1512 			/*
1513 			 *
1514 			 */
1515 			if (filter_hash && in_hash && !in_other_hash)
1516 				match = 1;
1517 			else if (!filter_hash && in_hash &&
1518 				 (in_other_hash || ftrace_hash_empty(other_hash)))
1519 				match = 1;
1520 		}
1521 		if (!match)
1522 			continue;
1523 
1524 		if (inc) {
1525 			rec->flags++;
1526 			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1527 				return;
1528 		} else {
1529 			if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1530 				return;
1531 			rec->flags--;
1532 		}
1533 		count++;
1534 		/* Shortcut, if we handled all records, we are done. */
1535 		if (!all && count == hash->count)
1536 			return;
1537 	} while_for_each_ftrace_rec();
1538 }
1539 
1540 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1541 				    int filter_hash)
1542 {
1543 	__ftrace_hash_rec_update(ops, filter_hash, 0);
1544 }
1545 
1546 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1547 				   int filter_hash)
1548 {
1549 	__ftrace_hash_rec_update(ops, filter_hash, 1);
1550 }
1551 
1552 static void print_ip_ins(const char *fmt, unsigned char *p)
1553 {
1554 	int i;
1555 
1556 	printk(KERN_CONT "%s", fmt);
1557 
1558 	for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1559 		printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1560 }
1561 
1562 /**
1563  * ftrace_bug - report and shutdown function tracer
1564  * @failed: The failed type (EFAULT, EINVAL, EPERM)
1565  * @ip: The address that failed
1566  *
1567  * The arch code that enables or disables the function tracing
1568  * can call ftrace_bug() when it has detected a problem in
1569  * modifying the code. @failed should be one of either:
1570  * EFAULT - if the problem happens on reading the @ip address
1571  * EINVAL - if what is read at @ip is not what was expected
1572  * EPERM - if the problem happens on writting to the @ip address
1573  */
1574 void ftrace_bug(int failed, unsigned long ip)
1575 {
1576 	switch (failed) {
1577 	case -EFAULT:
1578 		FTRACE_WARN_ON_ONCE(1);
1579 		pr_info("ftrace faulted on modifying ");
1580 		print_ip_sym(ip);
1581 		break;
1582 	case -EINVAL:
1583 		FTRACE_WARN_ON_ONCE(1);
1584 		pr_info("ftrace failed to modify ");
1585 		print_ip_sym(ip);
1586 		print_ip_ins(" actual: ", (unsigned char *)ip);
1587 		printk(KERN_CONT "\n");
1588 		break;
1589 	case -EPERM:
1590 		FTRACE_WARN_ON_ONCE(1);
1591 		pr_info("ftrace faulted on writing ");
1592 		print_ip_sym(ip);
1593 		break;
1594 	default:
1595 		FTRACE_WARN_ON_ONCE(1);
1596 		pr_info("ftrace faulted on unknown error ");
1597 		print_ip_sym(ip);
1598 	}
1599 }
1600 
1601 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1602 {
1603 	unsigned long flag = 0UL;
1604 
1605 	/*
1606 	 * If we are updating calls:
1607 	 *
1608 	 *   If the record has a ref count, then we need to enable it
1609 	 *   because someone is using it.
1610 	 *
1611 	 *   Otherwise we make sure its disabled.
1612 	 *
1613 	 * If we are disabling calls, then disable all records that
1614 	 * are enabled.
1615 	 */
1616 	if (enable && (rec->flags & ~FTRACE_FL_MASK))
1617 		flag = FTRACE_FL_ENABLED;
1618 
1619 	/* If the state of this record hasn't changed, then do nothing */
1620 	if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1621 		return FTRACE_UPDATE_IGNORE;
1622 
1623 	if (flag) {
1624 		if (update)
1625 			rec->flags |= FTRACE_FL_ENABLED;
1626 		return FTRACE_UPDATE_MAKE_CALL;
1627 	}
1628 
1629 	if (update)
1630 		rec->flags &= ~FTRACE_FL_ENABLED;
1631 
1632 	return FTRACE_UPDATE_MAKE_NOP;
1633 }
1634 
1635 /**
1636  * ftrace_update_record, set a record that now is tracing or not
1637  * @rec: the record to update
1638  * @enable: set to 1 if the record is tracing, zero to force disable
1639  *
1640  * The records that represent all functions that can be traced need
1641  * to be updated when tracing has been enabled.
1642  */
1643 int ftrace_update_record(struct dyn_ftrace *rec, int enable)
1644 {
1645 	return ftrace_check_record(rec, enable, 1);
1646 }
1647 
1648 /**
1649  * ftrace_test_record, check if the record has been enabled or not
1650  * @rec: the record to test
1651  * @enable: set to 1 to check if enabled, 0 if it is disabled
1652  *
1653  * The arch code may need to test if a record is already set to
1654  * tracing to determine how to modify the function code that it
1655  * represents.
1656  */
1657 int ftrace_test_record(struct dyn_ftrace *rec, int enable)
1658 {
1659 	return ftrace_check_record(rec, enable, 0);
1660 }
1661 
1662 static int
1663 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1664 {
1665 	unsigned long ftrace_addr;
1666 	int ret;
1667 
1668 	ftrace_addr = (unsigned long)FTRACE_ADDR;
1669 
1670 	ret = ftrace_update_record(rec, enable);
1671 
1672 	switch (ret) {
1673 	case FTRACE_UPDATE_IGNORE:
1674 		return 0;
1675 
1676 	case FTRACE_UPDATE_MAKE_CALL:
1677 		return ftrace_make_call(rec, ftrace_addr);
1678 
1679 	case FTRACE_UPDATE_MAKE_NOP:
1680 		return ftrace_make_nop(NULL, rec, ftrace_addr);
1681 	}
1682 
1683 	return -1; /* unknow ftrace bug */
1684 }
1685 
1686 void __weak ftrace_replace_code(int enable)
1687 {
1688 	struct dyn_ftrace *rec;
1689 	struct ftrace_page *pg;
1690 	int failed;
1691 
1692 	if (unlikely(ftrace_disabled))
1693 		return;
1694 
1695 	do_for_each_ftrace_rec(pg, rec) {
1696 		failed = __ftrace_replace_code(rec, enable);
1697 		if (failed) {
1698 			ftrace_bug(failed, rec->ip);
1699 			/* Stop processing */
1700 			return;
1701 		}
1702 	} while_for_each_ftrace_rec();
1703 }
1704 
1705 struct ftrace_rec_iter {
1706 	struct ftrace_page	*pg;
1707 	int			index;
1708 };
1709 
1710 /**
1711  * ftrace_rec_iter_start, start up iterating over traced functions
1712  *
1713  * Returns an iterator handle that is used to iterate over all
1714  * the records that represent address locations where functions
1715  * are traced.
1716  *
1717  * May return NULL if no records are available.
1718  */
1719 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
1720 {
1721 	/*
1722 	 * We only use a single iterator.
1723 	 * Protected by the ftrace_lock mutex.
1724 	 */
1725 	static struct ftrace_rec_iter ftrace_rec_iter;
1726 	struct ftrace_rec_iter *iter = &ftrace_rec_iter;
1727 
1728 	iter->pg = ftrace_pages_start;
1729 	iter->index = 0;
1730 
1731 	/* Could have empty pages */
1732 	while (iter->pg && !iter->pg->index)
1733 		iter->pg = iter->pg->next;
1734 
1735 	if (!iter->pg)
1736 		return NULL;
1737 
1738 	return iter;
1739 }
1740 
1741 /**
1742  * ftrace_rec_iter_next, get the next record to process.
1743  * @iter: The handle to the iterator.
1744  *
1745  * Returns the next iterator after the given iterator @iter.
1746  */
1747 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
1748 {
1749 	iter->index++;
1750 
1751 	if (iter->index >= iter->pg->index) {
1752 		iter->pg = iter->pg->next;
1753 		iter->index = 0;
1754 
1755 		/* Could have empty pages */
1756 		while (iter->pg && !iter->pg->index)
1757 			iter->pg = iter->pg->next;
1758 	}
1759 
1760 	if (!iter->pg)
1761 		return NULL;
1762 
1763 	return iter;
1764 }
1765 
1766 /**
1767  * ftrace_rec_iter_record, get the record at the iterator location
1768  * @iter: The current iterator location
1769  *
1770  * Returns the record that the current @iter is at.
1771  */
1772 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
1773 {
1774 	return &iter->pg->records[iter->index];
1775 }
1776 
1777 static int
1778 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1779 {
1780 	unsigned long ip;
1781 	int ret;
1782 
1783 	ip = rec->ip;
1784 
1785 	if (unlikely(ftrace_disabled))
1786 		return 0;
1787 
1788 	ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1789 	if (ret) {
1790 		ftrace_bug(ret, ip);
1791 		return 0;
1792 	}
1793 	return 1;
1794 }
1795 
1796 /*
1797  * archs can override this function if they must do something
1798  * before the modifying code is performed.
1799  */
1800 int __weak ftrace_arch_code_modify_prepare(void)
1801 {
1802 	return 0;
1803 }
1804 
1805 /*
1806  * archs can override this function if they must do something
1807  * after the modifying code is performed.
1808  */
1809 int __weak ftrace_arch_code_modify_post_process(void)
1810 {
1811 	return 0;
1812 }
1813 
1814 void ftrace_modify_all_code(int command)
1815 {
1816 	if (command & FTRACE_UPDATE_CALLS)
1817 		ftrace_replace_code(1);
1818 	else if (command & FTRACE_DISABLE_CALLS)
1819 		ftrace_replace_code(0);
1820 
1821 	if (command & FTRACE_UPDATE_TRACE_FUNC)
1822 		ftrace_update_ftrace_func(ftrace_trace_function);
1823 
1824 	if (command & FTRACE_START_FUNC_RET)
1825 		ftrace_enable_ftrace_graph_caller();
1826 	else if (command & FTRACE_STOP_FUNC_RET)
1827 		ftrace_disable_ftrace_graph_caller();
1828 }
1829 
1830 static int __ftrace_modify_code(void *data)
1831 {
1832 	int *command = data;
1833 
1834 	ftrace_modify_all_code(*command);
1835 
1836 	return 0;
1837 }
1838 
1839 /**
1840  * ftrace_run_stop_machine, go back to the stop machine method
1841  * @command: The command to tell ftrace what to do
1842  *
1843  * If an arch needs to fall back to the stop machine method, the
1844  * it can call this function.
1845  */
1846 void ftrace_run_stop_machine(int command)
1847 {
1848 	stop_machine(__ftrace_modify_code, &command, NULL);
1849 }
1850 
1851 /**
1852  * arch_ftrace_update_code, modify the code to trace or not trace
1853  * @command: The command that needs to be done
1854  *
1855  * Archs can override this function if it does not need to
1856  * run stop_machine() to modify code.
1857  */
1858 void __weak arch_ftrace_update_code(int command)
1859 {
1860 	ftrace_run_stop_machine(command);
1861 }
1862 
1863 static void ftrace_run_update_code(int command)
1864 {
1865 	int ret;
1866 
1867 	ret = ftrace_arch_code_modify_prepare();
1868 	FTRACE_WARN_ON(ret);
1869 	if (ret)
1870 		return;
1871 	/*
1872 	 * Do not call function tracer while we update the code.
1873 	 * We are in stop machine.
1874 	 */
1875 	function_trace_stop++;
1876 
1877 	/*
1878 	 * By default we use stop_machine() to modify the code.
1879 	 * But archs can do what ever they want as long as it
1880 	 * is safe. The stop_machine() is the safest, but also
1881 	 * produces the most overhead.
1882 	 */
1883 	arch_ftrace_update_code(command);
1884 
1885 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
1886 	/*
1887 	 * For archs that call ftrace_test_stop_func(), we must
1888 	 * wait till after we update all the function callers
1889 	 * before we update the callback. This keeps different
1890 	 * ops that record different functions from corrupting
1891 	 * each other.
1892 	 */
1893 	__ftrace_trace_function = __ftrace_trace_function_delay;
1894 #endif
1895 	function_trace_stop--;
1896 
1897 	ret = ftrace_arch_code_modify_post_process();
1898 	FTRACE_WARN_ON(ret);
1899 }
1900 
1901 static ftrace_func_t saved_ftrace_func;
1902 static int ftrace_start_up;
1903 static int global_start_up;
1904 
1905 static void ftrace_startup_enable(int command)
1906 {
1907 	if (saved_ftrace_func != ftrace_trace_function) {
1908 		saved_ftrace_func = ftrace_trace_function;
1909 		command |= FTRACE_UPDATE_TRACE_FUNC;
1910 	}
1911 
1912 	if (!command || !ftrace_enabled)
1913 		return;
1914 
1915 	ftrace_run_update_code(command);
1916 }
1917 
1918 static int ftrace_startup(struct ftrace_ops *ops, int command)
1919 {
1920 	bool hash_enable = true;
1921 
1922 	if (unlikely(ftrace_disabled))
1923 		return -ENODEV;
1924 
1925 	ftrace_start_up++;
1926 	command |= FTRACE_UPDATE_CALLS;
1927 
1928 	/* ops marked global share the filter hashes */
1929 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1930 		ops = &global_ops;
1931 		/* Don't update hash if global is already set */
1932 		if (global_start_up)
1933 			hash_enable = false;
1934 		global_start_up++;
1935 	}
1936 
1937 	ops->flags |= FTRACE_OPS_FL_ENABLED;
1938 	if (hash_enable)
1939 		ftrace_hash_rec_enable(ops, 1);
1940 
1941 	ftrace_startup_enable(command);
1942 
1943 	return 0;
1944 }
1945 
1946 static void ftrace_shutdown(struct ftrace_ops *ops, int command)
1947 {
1948 	bool hash_disable = true;
1949 
1950 	if (unlikely(ftrace_disabled))
1951 		return;
1952 
1953 	ftrace_start_up--;
1954 	/*
1955 	 * Just warn in case of unbalance, no need to kill ftrace, it's not
1956 	 * critical but the ftrace_call callers may be never nopped again after
1957 	 * further ftrace uses.
1958 	 */
1959 	WARN_ON_ONCE(ftrace_start_up < 0);
1960 
1961 	if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1962 		ops = &global_ops;
1963 		global_start_up--;
1964 		WARN_ON_ONCE(global_start_up < 0);
1965 		/* Don't update hash if global still has users */
1966 		if (global_start_up) {
1967 			WARN_ON_ONCE(!ftrace_start_up);
1968 			hash_disable = false;
1969 		}
1970 	}
1971 
1972 	if (hash_disable)
1973 		ftrace_hash_rec_disable(ops, 1);
1974 
1975 	if (ops != &global_ops || !global_start_up)
1976 		ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1977 
1978 	command |= FTRACE_UPDATE_CALLS;
1979 
1980 	if (saved_ftrace_func != ftrace_trace_function) {
1981 		saved_ftrace_func = ftrace_trace_function;
1982 		command |= FTRACE_UPDATE_TRACE_FUNC;
1983 	}
1984 
1985 	if (!command || !ftrace_enabled)
1986 		return;
1987 
1988 	ftrace_run_update_code(command);
1989 }
1990 
1991 static void ftrace_startup_sysctl(void)
1992 {
1993 	if (unlikely(ftrace_disabled))
1994 		return;
1995 
1996 	/* Force update next time */
1997 	saved_ftrace_func = NULL;
1998 	/* ftrace_start_up is true if we want ftrace running */
1999 	if (ftrace_start_up)
2000 		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2001 }
2002 
2003 static void ftrace_shutdown_sysctl(void)
2004 {
2005 	if (unlikely(ftrace_disabled))
2006 		return;
2007 
2008 	/* ftrace_start_up is true if ftrace is running */
2009 	if (ftrace_start_up)
2010 		ftrace_run_update_code(FTRACE_DISABLE_CALLS);
2011 }
2012 
2013 static cycle_t		ftrace_update_time;
2014 static unsigned long	ftrace_update_cnt;
2015 unsigned long		ftrace_update_tot_cnt;
2016 
2017 static int ops_traces_mod(struct ftrace_ops *ops)
2018 {
2019 	struct ftrace_hash *hash;
2020 
2021 	hash = ops->filter_hash;
2022 	return ftrace_hash_empty(hash);
2023 }
2024 
2025 static int ftrace_update_code(struct module *mod)
2026 {
2027 	struct ftrace_page *pg;
2028 	struct dyn_ftrace *p;
2029 	cycle_t start, stop;
2030 	unsigned long ref = 0;
2031 	int i;
2032 
2033 	/*
2034 	 * When adding a module, we need to check if tracers are
2035 	 * currently enabled and if they are set to trace all functions.
2036 	 * If they are, we need to enable the module functions as well
2037 	 * as update the reference counts for those function records.
2038 	 */
2039 	if (mod) {
2040 		struct ftrace_ops *ops;
2041 
2042 		for (ops = ftrace_ops_list;
2043 		     ops != &ftrace_list_end; ops = ops->next) {
2044 			if (ops->flags & FTRACE_OPS_FL_ENABLED &&
2045 			    ops_traces_mod(ops))
2046 				ref++;
2047 		}
2048 	}
2049 
2050 	start = ftrace_now(raw_smp_processor_id());
2051 	ftrace_update_cnt = 0;
2052 
2053 	for (pg = ftrace_new_pgs; pg; pg = pg->next) {
2054 
2055 		for (i = 0; i < pg->index; i++) {
2056 			/* If something went wrong, bail without enabling anything */
2057 			if (unlikely(ftrace_disabled))
2058 				return -1;
2059 
2060 			p = &pg->records[i];
2061 			p->flags = ref;
2062 
2063 			/*
2064 			 * Do the initial record conversion from mcount jump
2065 			 * to the NOP instructions.
2066 			 */
2067 			if (!ftrace_code_disable(mod, p))
2068 				break;
2069 
2070 			ftrace_update_cnt++;
2071 
2072 			/*
2073 			 * If the tracing is enabled, go ahead and enable the record.
2074 			 *
2075 			 * The reason not to enable the record immediatelly is the
2076 			 * inherent check of ftrace_make_nop/ftrace_make_call for
2077 			 * correct previous instructions.  Making first the NOP
2078 			 * conversion puts the module to the correct state, thus
2079 			 * passing the ftrace_make_call check.
2080 			 */
2081 			if (ftrace_start_up && ref) {
2082 				int failed = __ftrace_replace_code(p, 1);
2083 				if (failed)
2084 					ftrace_bug(failed, p->ip);
2085 			}
2086 		}
2087 	}
2088 
2089 	ftrace_new_pgs = NULL;
2090 
2091 	stop = ftrace_now(raw_smp_processor_id());
2092 	ftrace_update_time = stop - start;
2093 	ftrace_update_tot_cnt += ftrace_update_cnt;
2094 
2095 	return 0;
2096 }
2097 
2098 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2099 {
2100 	int order;
2101 	int cnt;
2102 
2103 	if (WARN_ON(!count))
2104 		return -EINVAL;
2105 
2106 	order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2107 
2108 	/*
2109 	 * We want to fill as much as possible. No more than a page
2110 	 * may be empty.
2111 	 */
2112 	while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2113 		order--;
2114 
2115  again:
2116 	pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2117 
2118 	if (!pg->records) {
2119 		/* if we can't allocate this size, try something smaller */
2120 		if (!order)
2121 			return -ENOMEM;
2122 		order >>= 1;
2123 		goto again;
2124 	}
2125 
2126 	cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2127 	pg->size = cnt;
2128 
2129 	if (cnt > count)
2130 		cnt = count;
2131 
2132 	return cnt;
2133 }
2134 
2135 static struct ftrace_page *
2136 ftrace_allocate_pages(unsigned long num_to_init)
2137 {
2138 	struct ftrace_page *start_pg;
2139 	struct ftrace_page *pg;
2140 	int order;
2141 	int cnt;
2142 
2143 	if (!num_to_init)
2144 		return 0;
2145 
2146 	start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2147 	if (!pg)
2148 		return NULL;
2149 
2150 	/*
2151 	 * Try to allocate as much as possible in one continues
2152 	 * location that fills in all of the space. We want to
2153 	 * waste as little space as possible.
2154 	 */
2155 	for (;;) {
2156 		cnt = ftrace_allocate_records(pg, num_to_init);
2157 		if (cnt < 0)
2158 			goto free_pages;
2159 
2160 		num_to_init -= cnt;
2161 		if (!num_to_init)
2162 			break;
2163 
2164 		pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
2165 		if (!pg->next)
2166 			goto free_pages;
2167 
2168 		pg = pg->next;
2169 	}
2170 
2171 	return start_pg;
2172 
2173  free_pages:
2174 	while (start_pg) {
2175 		order = get_count_order(pg->size / ENTRIES_PER_PAGE);
2176 		free_pages((unsigned long)pg->records, order);
2177 		start_pg = pg->next;
2178 		kfree(pg);
2179 		pg = start_pg;
2180 	}
2181 	pr_info("ftrace: FAILED to allocate memory for functions\n");
2182 	return NULL;
2183 }
2184 
2185 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
2186 {
2187 	int cnt;
2188 
2189 	if (!num_to_init) {
2190 		pr_info("ftrace: No functions to be traced?\n");
2191 		return -1;
2192 	}
2193 
2194 	cnt = num_to_init / ENTRIES_PER_PAGE;
2195 	pr_info("ftrace: allocating %ld entries in %d pages\n",
2196 		num_to_init, cnt + 1);
2197 
2198 	return 0;
2199 }
2200 
2201 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2202 
2203 struct ftrace_iterator {
2204 	loff_t				pos;
2205 	loff_t				func_pos;
2206 	struct ftrace_page		*pg;
2207 	struct dyn_ftrace		*func;
2208 	struct ftrace_func_probe	*probe;
2209 	struct trace_parser		parser;
2210 	struct ftrace_hash		*hash;
2211 	struct ftrace_ops		*ops;
2212 	int				hidx;
2213 	int				idx;
2214 	unsigned			flags;
2215 };
2216 
2217 static void *
2218 t_hash_next(struct seq_file *m, loff_t *pos)
2219 {
2220 	struct ftrace_iterator *iter = m->private;
2221 	struct hlist_node *hnd = NULL;
2222 	struct hlist_head *hhd;
2223 
2224 	(*pos)++;
2225 	iter->pos = *pos;
2226 
2227 	if (iter->probe)
2228 		hnd = &iter->probe->node;
2229  retry:
2230 	if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
2231 		return NULL;
2232 
2233 	hhd = &ftrace_func_hash[iter->hidx];
2234 
2235 	if (hlist_empty(hhd)) {
2236 		iter->hidx++;
2237 		hnd = NULL;
2238 		goto retry;
2239 	}
2240 
2241 	if (!hnd)
2242 		hnd = hhd->first;
2243 	else {
2244 		hnd = hnd->next;
2245 		if (!hnd) {
2246 			iter->hidx++;
2247 			goto retry;
2248 		}
2249 	}
2250 
2251 	if (WARN_ON_ONCE(!hnd))
2252 		return NULL;
2253 
2254 	iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2255 
2256 	return iter;
2257 }
2258 
2259 static void *t_hash_start(struct seq_file *m, loff_t *pos)
2260 {
2261 	struct ftrace_iterator *iter = m->private;
2262 	void *p = NULL;
2263 	loff_t l;
2264 
2265 	if (!(iter->flags & FTRACE_ITER_DO_HASH))
2266 		return NULL;
2267 
2268 	if (iter->func_pos > *pos)
2269 		return NULL;
2270 
2271 	iter->hidx = 0;
2272 	for (l = 0; l <= (*pos - iter->func_pos); ) {
2273 		p = t_hash_next(m, &l);
2274 		if (!p)
2275 			break;
2276 	}
2277 	if (!p)
2278 		return NULL;
2279 
2280 	/* Only set this if we have an item */
2281 	iter->flags |= FTRACE_ITER_HASH;
2282 
2283 	return iter;
2284 }
2285 
2286 static int
2287 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2288 {
2289 	struct ftrace_func_probe *rec;
2290 
2291 	rec = iter->probe;
2292 	if (WARN_ON_ONCE(!rec))
2293 		return -EIO;
2294 
2295 	if (rec->ops->print)
2296 		return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2297 
2298 	seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
2299 
2300 	if (rec->data)
2301 		seq_printf(m, ":%p", rec->data);
2302 	seq_putc(m, '\n');
2303 
2304 	return 0;
2305 }
2306 
2307 static void *
2308 t_next(struct seq_file *m, void *v, loff_t *pos)
2309 {
2310 	struct ftrace_iterator *iter = m->private;
2311 	struct ftrace_ops *ops = iter->ops;
2312 	struct dyn_ftrace *rec = NULL;
2313 
2314 	if (unlikely(ftrace_disabled))
2315 		return NULL;
2316 
2317 	if (iter->flags & FTRACE_ITER_HASH)
2318 		return t_hash_next(m, pos);
2319 
2320 	(*pos)++;
2321 	iter->pos = iter->func_pos = *pos;
2322 
2323 	if (iter->flags & FTRACE_ITER_PRINTALL)
2324 		return t_hash_start(m, pos);
2325 
2326  retry:
2327 	if (iter->idx >= iter->pg->index) {
2328 		if (iter->pg->next) {
2329 			iter->pg = iter->pg->next;
2330 			iter->idx = 0;
2331 			goto retry;
2332 		}
2333 	} else {
2334 		rec = &iter->pg->records[iter->idx++];
2335 		if (((iter->flags & FTRACE_ITER_FILTER) &&
2336 		     !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2337 
2338 		    ((iter->flags & FTRACE_ITER_NOTRACE) &&
2339 		     !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2340 
2341 		    ((iter->flags & FTRACE_ITER_ENABLED) &&
2342 		     !(rec->flags & ~FTRACE_FL_MASK))) {
2343 
2344 			rec = NULL;
2345 			goto retry;
2346 		}
2347 	}
2348 
2349 	if (!rec)
2350 		return t_hash_start(m, pos);
2351 
2352 	iter->func = rec;
2353 
2354 	return iter;
2355 }
2356 
2357 static void reset_iter_read(struct ftrace_iterator *iter)
2358 {
2359 	iter->pos = 0;
2360 	iter->func_pos = 0;
2361 	iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
2362 }
2363 
2364 static void *t_start(struct seq_file *m, loff_t *pos)
2365 {
2366 	struct ftrace_iterator *iter = m->private;
2367 	struct ftrace_ops *ops = iter->ops;
2368 	void *p = NULL;
2369 	loff_t l;
2370 
2371 	mutex_lock(&ftrace_lock);
2372 
2373 	if (unlikely(ftrace_disabled))
2374 		return NULL;
2375 
2376 	/*
2377 	 * If an lseek was done, then reset and start from beginning.
2378 	 */
2379 	if (*pos < iter->pos)
2380 		reset_iter_read(iter);
2381 
2382 	/*
2383 	 * For set_ftrace_filter reading, if we have the filter
2384 	 * off, we can short cut and just print out that all
2385 	 * functions are enabled.
2386 	 */
2387 	if (iter->flags & FTRACE_ITER_FILTER &&
2388 	    ftrace_hash_empty(ops->filter_hash)) {
2389 		if (*pos > 0)
2390 			return t_hash_start(m, pos);
2391 		iter->flags |= FTRACE_ITER_PRINTALL;
2392 		/* reset in case of seek/pread */
2393 		iter->flags &= ~FTRACE_ITER_HASH;
2394 		return iter;
2395 	}
2396 
2397 	if (iter->flags & FTRACE_ITER_HASH)
2398 		return t_hash_start(m, pos);
2399 
2400 	/*
2401 	 * Unfortunately, we need to restart at ftrace_pages_start
2402 	 * every time we let go of the ftrace_mutex. This is because
2403 	 * those pointers can change without the lock.
2404 	 */
2405 	iter->pg = ftrace_pages_start;
2406 	iter->idx = 0;
2407 	for (l = 0; l <= *pos; ) {
2408 		p = t_next(m, p, &l);
2409 		if (!p)
2410 			break;
2411 	}
2412 
2413 	if (!p)
2414 		return t_hash_start(m, pos);
2415 
2416 	return iter;
2417 }
2418 
2419 static void t_stop(struct seq_file *m, void *p)
2420 {
2421 	mutex_unlock(&ftrace_lock);
2422 }
2423 
2424 static int t_show(struct seq_file *m, void *v)
2425 {
2426 	struct ftrace_iterator *iter = m->private;
2427 	struct dyn_ftrace *rec;
2428 
2429 	if (iter->flags & FTRACE_ITER_HASH)
2430 		return t_hash_show(m, iter);
2431 
2432 	if (iter->flags & FTRACE_ITER_PRINTALL) {
2433 		seq_printf(m, "#### all functions enabled ####\n");
2434 		return 0;
2435 	}
2436 
2437 	rec = iter->func;
2438 
2439 	if (!rec)
2440 		return 0;
2441 
2442 	seq_printf(m, "%ps", (void *)rec->ip);
2443 	if (iter->flags & FTRACE_ITER_ENABLED)
2444 		seq_printf(m, " (%ld)",
2445 			   rec->flags & ~FTRACE_FL_MASK);
2446 	seq_printf(m, "\n");
2447 
2448 	return 0;
2449 }
2450 
2451 static const struct seq_operations show_ftrace_seq_ops = {
2452 	.start = t_start,
2453 	.next = t_next,
2454 	.stop = t_stop,
2455 	.show = t_show,
2456 };
2457 
2458 static int
2459 ftrace_avail_open(struct inode *inode, struct file *file)
2460 {
2461 	struct ftrace_iterator *iter;
2462 
2463 	if (unlikely(ftrace_disabled))
2464 		return -ENODEV;
2465 
2466 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2467 	if (iter) {
2468 		iter->pg = ftrace_pages_start;
2469 		iter->ops = &global_ops;
2470 	}
2471 
2472 	return iter ? 0 : -ENOMEM;
2473 }
2474 
2475 static int
2476 ftrace_enabled_open(struct inode *inode, struct file *file)
2477 {
2478 	struct ftrace_iterator *iter;
2479 
2480 	if (unlikely(ftrace_disabled))
2481 		return -ENODEV;
2482 
2483 	iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2484 	if (iter) {
2485 		iter->pg = ftrace_pages_start;
2486 		iter->flags = FTRACE_ITER_ENABLED;
2487 		iter->ops = &global_ops;
2488 	}
2489 
2490 	return iter ? 0 : -ENOMEM;
2491 }
2492 
2493 static void ftrace_filter_reset(struct ftrace_hash *hash)
2494 {
2495 	mutex_lock(&ftrace_lock);
2496 	ftrace_hash_clear(hash);
2497 	mutex_unlock(&ftrace_lock);
2498 }
2499 
2500 /**
2501  * ftrace_regex_open - initialize function tracer filter files
2502  * @ops: The ftrace_ops that hold the hash filters
2503  * @flag: The type of filter to process
2504  * @inode: The inode, usually passed in to your open routine
2505  * @file: The file, usually passed in to your open routine
2506  *
2507  * ftrace_regex_open() initializes the filter files for the
2508  * @ops. Depending on @flag it may process the filter hash or
2509  * the notrace hash of @ops. With this called from the open
2510  * routine, you can use ftrace_filter_write() for the write
2511  * routine if @flag has FTRACE_ITER_FILTER set, or
2512  * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
2513  * ftrace_regex_lseek() should be used as the lseek routine, and
2514  * release must call ftrace_regex_release().
2515  */
2516 int
2517 ftrace_regex_open(struct ftrace_ops *ops, int flag,
2518 		  struct inode *inode, struct file *file)
2519 {
2520 	struct ftrace_iterator *iter;
2521 	struct ftrace_hash *hash;
2522 	int ret = 0;
2523 
2524 	if (unlikely(ftrace_disabled))
2525 		return -ENODEV;
2526 
2527 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2528 	if (!iter)
2529 		return -ENOMEM;
2530 
2531 	if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2532 		kfree(iter);
2533 		return -ENOMEM;
2534 	}
2535 
2536 	if (flag & FTRACE_ITER_NOTRACE)
2537 		hash = ops->notrace_hash;
2538 	else
2539 		hash = ops->filter_hash;
2540 
2541 	iter->ops = ops;
2542 	iter->flags = flag;
2543 
2544 	if (file->f_mode & FMODE_WRITE) {
2545 		mutex_lock(&ftrace_lock);
2546 		iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2547 		mutex_unlock(&ftrace_lock);
2548 
2549 		if (!iter->hash) {
2550 			trace_parser_put(&iter->parser);
2551 			kfree(iter);
2552 			return -ENOMEM;
2553 		}
2554 	}
2555 
2556 	mutex_lock(&ftrace_regex_lock);
2557 
2558 	if ((file->f_mode & FMODE_WRITE) &&
2559 	    (file->f_flags & O_TRUNC))
2560 		ftrace_filter_reset(iter->hash);
2561 
2562 	if (file->f_mode & FMODE_READ) {
2563 		iter->pg = ftrace_pages_start;
2564 
2565 		ret = seq_open(file, &show_ftrace_seq_ops);
2566 		if (!ret) {
2567 			struct seq_file *m = file->private_data;
2568 			m->private = iter;
2569 		} else {
2570 			/* Failed */
2571 			free_ftrace_hash(iter->hash);
2572 			trace_parser_put(&iter->parser);
2573 			kfree(iter);
2574 		}
2575 	} else
2576 		file->private_data = iter;
2577 	mutex_unlock(&ftrace_regex_lock);
2578 
2579 	return ret;
2580 }
2581 
2582 static int
2583 ftrace_filter_open(struct inode *inode, struct file *file)
2584 {
2585 	return ftrace_regex_open(&global_ops,
2586 			FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
2587 			inode, file);
2588 }
2589 
2590 static int
2591 ftrace_notrace_open(struct inode *inode, struct file *file)
2592 {
2593 	return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2594 				 inode, file);
2595 }
2596 
2597 loff_t
2598 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
2599 {
2600 	loff_t ret;
2601 
2602 	if (file->f_mode & FMODE_READ)
2603 		ret = seq_lseek(file, offset, origin);
2604 	else
2605 		file->f_pos = ret = 1;
2606 
2607 	return ret;
2608 }
2609 
2610 static int ftrace_match(char *str, char *regex, int len, int type)
2611 {
2612 	int matched = 0;
2613 	int slen;
2614 
2615 	switch (type) {
2616 	case MATCH_FULL:
2617 		if (strcmp(str, regex) == 0)
2618 			matched = 1;
2619 		break;
2620 	case MATCH_FRONT_ONLY:
2621 		if (strncmp(str, regex, len) == 0)
2622 			matched = 1;
2623 		break;
2624 	case MATCH_MIDDLE_ONLY:
2625 		if (strstr(str, regex))
2626 			matched = 1;
2627 		break;
2628 	case MATCH_END_ONLY:
2629 		slen = strlen(str);
2630 		if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2631 			matched = 1;
2632 		break;
2633 	}
2634 
2635 	return matched;
2636 }
2637 
2638 static int
2639 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2640 {
2641 	struct ftrace_func_entry *entry;
2642 	int ret = 0;
2643 
2644 	entry = ftrace_lookup_ip(hash, rec->ip);
2645 	if (not) {
2646 		/* Do nothing if it doesn't exist */
2647 		if (!entry)
2648 			return 0;
2649 
2650 		free_hash_entry(hash, entry);
2651 	} else {
2652 		/* Do nothing if it exists */
2653 		if (entry)
2654 			return 0;
2655 
2656 		ret = add_hash_entry(hash, rec->ip);
2657 	}
2658 	return ret;
2659 }
2660 
2661 static int
2662 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2663 		    char *regex, int len, int type)
2664 {
2665 	char str[KSYM_SYMBOL_LEN];
2666 	char *modname;
2667 
2668 	kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2669 
2670 	if (mod) {
2671 		/* module lookup requires matching the module */
2672 		if (!modname || strcmp(modname, mod))
2673 			return 0;
2674 
2675 		/* blank search means to match all funcs in the mod */
2676 		if (!len)
2677 			return 1;
2678 	}
2679 
2680 	return ftrace_match(str, regex, len, type);
2681 }
2682 
2683 static int
2684 match_records(struct ftrace_hash *hash, char *buff,
2685 	      int len, char *mod, int not)
2686 {
2687 	unsigned search_len = 0;
2688 	struct ftrace_page *pg;
2689 	struct dyn_ftrace *rec;
2690 	int type = MATCH_FULL;
2691 	char *search = buff;
2692 	int found = 0;
2693 	int ret;
2694 
2695 	if (len) {
2696 		type = filter_parse_regex(buff, len, &search, &not);
2697 		search_len = strlen(search);
2698 	}
2699 
2700 	mutex_lock(&ftrace_lock);
2701 
2702 	if (unlikely(ftrace_disabled))
2703 		goto out_unlock;
2704 
2705 	do_for_each_ftrace_rec(pg, rec) {
2706 		if (ftrace_match_record(rec, mod, search, search_len, type)) {
2707 			ret = enter_record(hash, rec, not);
2708 			if (ret < 0) {
2709 				found = ret;
2710 				goto out_unlock;
2711 			}
2712 			found = 1;
2713 		}
2714 	} while_for_each_ftrace_rec();
2715  out_unlock:
2716 	mutex_unlock(&ftrace_lock);
2717 
2718 	return found;
2719 }
2720 
2721 static int
2722 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2723 {
2724 	return match_records(hash, buff, len, NULL, 0);
2725 }
2726 
2727 static int
2728 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2729 {
2730 	int not = 0;
2731 
2732 	/* blank or '*' mean the same */
2733 	if (strcmp(buff, "*") == 0)
2734 		buff[0] = 0;
2735 
2736 	/* handle the case of 'dont filter this module' */
2737 	if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2738 		buff[0] = 0;
2739 		not = 1;
2740 	}
2741 
2742 	return match_records(hash, buff, strlen(buff), mod, not);
2743 }
2744 
2745 /*
2746  * We register the module command as a template to show others how
2747  * to register the a command as well.
2748  */
2749 
2750 static int
2751 ftrace_mod_callback(struct ftrace_hash *hash,
2752 		    char *func, char *cmd, char *param, int enable)
2753 {
2754 	char *mod;
2755 	int ret = -EINVAL;
2756 
2757 	/*
2758 	 * cmd == 'mod' because we only registered this func
2759 	 * for the 'mod' ftrace_func_command.
2760 	 * But if you register one func with multiple commands,
2761 	 * you can tell which command was used by the cmd
2762 	 * parameter.
2763 	 */
2764 
2765 	/* we must have a module name */
2766 	if (!param)
2767 		return ret;
2768 
2769 	mod = strsep(&param, ":");
2770 	if (!strlen(mod))
2771 		return ret;
2772 
2773 	ret = ftrace_match_module_records(hash, func, mod);
2774 	if (!ret)
2775 		ret = -EINVAL;
2776 	if (ret < 0)
2777 		return ret;
2778 
2779 	return 0;
2780 }
2781 
2782 static struct ftrace_func_command ftrace_mod_cmd = {
2783 	.name			= "mod",
2784 	.func			= ftrace_mod_callback,
2785 };
2786 
2787 static int __init ftrace_mod_cmd_init(void)
2788 {
2789 	return register_ftrace_command(&ftrace_mod_cmd);
2790 }
2791 device_initcall(ftrace_mod_cmd_init);
2792 
2793 static void
2794 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
2795 {
2796 	struct ftrace_func_probe *entry;
2797 	struct hlist_head *hhd;
2798 	struct hlist_node *n;
2799 	unsigned long key;
2800 
2801 	key = hash_long(ip, FTRACE_HASH_BITS);
2802 
2803 	hhd = &ftrace_func_hash[key];
2804 
2805 	if (hlist_empty(hhd))
2806 		return;
2807 
2808 	/*
2809 	 * Disable preemption for these calls to prevent a RCU grace
2810 	 * period. This syncs the hash iteration and freeing of items
2811 	 * on the hash. rcu_read_lock is too dangerous here.
2812 	 */
2813 	preempt_disable_notrace();
2814 	hlist_for_each_entry_rcu(entry, n, hhd, node) {
2815 		if (entry->ip == ip)
2816 			entry->ops->func(ip, parent_ip, &entry->data);
2817 	}
2818 	preempt_enable_notrace();
2819 }
2820 
2821 static struct ftrace_ops trace_probe_ops __read_mostly =
2822 {
2823 	.func		= function_trace_probe_call,
2824 };
2825 
2826 static int ftrace_probe_registered;
2827 
2828 static void __enable_ftrace_function_probe(void)
2829 {
2830 	int ret;
2831 	int i;
2832 
2833 	if (ftrace_probe_registered)
2834 		return;
2835 
2836 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2837 		struct hlist_head *hhd = &ftrace_func_hash[i];
2838 		if (hhd->first)
2839 			break;
2840 	}
2841 	/* Nothing registered? */
2842 	if (i == FTRACE_FUNC_HASHSIZE)
2843 		return;
2844 
2845 	ret = __register_ftrace_function(&trace_probe_ops);
2846 	if (!ret)
2847 		ret = ftrace_startup(&trace_probe_ops, 0);
2848 
2849 	ftrace_probe_registered = 1;
2850 }
2851 
2852 static void __disable_ftrace_function_probe(void)
2853 {
2854 	int ret;
2855 	int i;
2856 
2857 	if (!ftrace_probe_registered)
2858 		return;
2859 
2860 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2861 		struct hlist_head *hhd = &ftrace_func_hash[i];
2862 		if (hhd->first)
2863 			return;
2864 	}
2865 
2866 	/* no more funcs left */
2867 	ret = __unregister_ftrace_function(&trace_probe_ops);
2868 	if (!ret)
2869 		ftrace_shutdown(&trace_probe_ops, 0);
2870 
2871 	ftrace_probe_registered = 0;
2872 }
2873 
2874 
2875 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2876 {
2877 	struct ftrace_func_probe *entry =
2878 		container_of(rhp, struct ftrace_func_probe, rcu);
2879 
2880 	if (entry->ops->free)
2881 		entry->ops->free(&entry->data);
2882 	kfree(entry);
2883 }
2884 
2885 
2886 int
2887 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2888 			      void *data)
2889 {
2890 	struct ftrace_func_probe *entry;
2891 	struct ftrace_page *pg;
2892 	struct dyn_ftrace *rec;
2893 	int type, len, not;
2894 	unsigned long key;
2895 	int count = 0;
2896 	char *search;
2897 
2898 	type = filter_parse_regex(glob, strlen(glob), &search, &not);
2899 	len = strlen(search);
2900 
2901 	/* we do not support '!' for function probes */
2902 	if (WARN_ON(not))
2903 		return -EINVAL;
2904 
2905 	mutex_lock(&ftrace_lock);
2906 
2907 	if (unlikely(ftrace_disabled))
2908 		goto out_unlock;
2909 
2910 	do_for_each_ftrace_rec(pg, rec) {
2911 
2912 		if (!ftrace_match_record(rec, NULL, search, len, type))
2913 			continue;
2914 
2915 		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2916 		if (!entry) {
2917 			/* If we did not process any, then return error */
2918 			if (!count)
2919 				count = -ENOMEM;
2920 			goto out_unlock;
2921 		}
2922 
2923 		count++;
2924 
2925 		entry->data = data;
2926 
2927 		/*
2928 		 * The caller might want to do something special
2929 		 * for each function we find. We call the callback
2930 		 * to give the caller an opportunity to do so.
2931 		 */
2932 		if (ops->callback) {
2933 			if (ops->callback(rec->ip, &entry->data) < 0) {
2934 				/* caller does not like this func */
2935 				kfree(entry);
2936 				continue;
2937 			}
2938 		}
2939 
2940 		entry->ops = ops;
2941 		entry->ip = rec->ip;
2942 
2943 		key = hash_long(entry->ip, FTRACE_HASH_BITS);
2944 		hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2945 
2946 	} while_for_each_ftrace_rec();
2947 	__enable_ftrace_function_probe();
2948 
2949  out_unlock:
2950 	mutex_unlock(&ftrace_lock);
2951 
2952 	return count;
2953 }
2954 
2955 enum {
2956 	PROBE_TEST_FUNC		= 1,
2957 	PROBE_TEST_DATA		= 2
2958 };
2959 
2960 static void
2961 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2962 				  void *data, int flags)
2963 {
2964 	struct ftrace_func_probe *entry;
2965 	struct hlist_node *n, *tmp;
2966 	char str[KSYM_SYMBOL_LEN];
2967 	int type = MATCH_FULL;
2968 	int i, len = 0;
2969 	char *search;
2970 
2971 	if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2972 		glob = NULL;
2973 	else if (glob) {
2974 		int not;
2975 
2976 		type = filter_parse_regex(glob, strlen(glob), &search, &not);
2977 		len = strlen(search);
2978 
2979 		/* we do not support '!' for function probes */
2980 		if (WARN_ON(not))
2981 			return;
2982 	}
2983 
2984 	mutex_lock(&ftrace_lock);
2985 	for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2986 		struct hlist_head *hhd = &ftrace_func_hash[i];
2987 
2988 		hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2989 
2990 			/* break up if statements for readability */
2991 			if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2992 				continue;
2993 
2994 			if ((flags & PROBE_TEST_DATA) && entry->data != data)
2995 				continue;
2996 
2997 			/* do this last, since it is the most expensive */
2998 			if (glob) {
2999 				kallsyms_lookup(entry->ip, NULL, NULL,
3000 						NULL, str);
3001 				if (!ftrace_match(str, glob, len, type))
3002 					continue;
3003 			}
3004 
3005 			hlist_del(&entry->node);
3006 			call_rcu(&entry->rcu, ftrace_free_entry_rcu);
3007 		}
3008 	}
3009 	__disable_ftrace_function_probe();
3010 	mutex_unlock(&ftrace_lock);
3011 }
3012 
3013 void
3014 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3015 				void *data)
3016 {
3017 	__unregister_ftrace_function_probe(glob, ops, data,
3018 					  PROBE_TEST_FUNC | PROBE_TEST_DATA);
3019 }
3020 
3021 void
3022 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3023 {
3024 	__unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
3025 }
3026 
3027 void unregister_ftrace_function_probe_all(char *glob)
3028 {
3029 	__unregister_ftrace_function_probe(glob, NULL, NULL, 0);
3030 }
3031 
3032 static LIST_HEAD(ftrace_commands);
3033 static DEFINE_MUTEX(ftrace_cmd_mutex);
3034 
3035 int register_ftrace_command(struct ftrace_func_command *cmd)
3036 {
3037 	struct ftrace_func_command *p;
3038 	int ret = 0;
3039 
3040 	mutex_lock(&ftrace_cmd_mutex);
3041 	list_for_each_entry(p, &ftrace_commands, list) {
3042 		if (strcmp(cmd->name, p->name) == 0) {
3043 			ret = -EBUSY;
3044 			goto out_unlock;
3045 		}
3046 	}
3047 	list_add(&cmd->list, &ftrace_commands);
3048  out_unlock:
3049 	mutex_unlock(&ftrace_cmd_mutex);
3050 
3051 	return ret;
3052 }
3053 
3054 int unregister_ftrace_command(struct ftrace_func_command *cmd)
3055 {
3056 	struct ftrace_func_command *p, *n;
3057 	int ret = -ENODEV;
3058 
3059 	mutex_lock(&ftrace_cmd_mutex);
3060 	list_for_each_entry_safe(p, n, &ftrace_commands, list) {
3061 		if (strcmp(cmd->name, p->name) == 0) {
3062 			ret = 0;
3063 			list_del_init(&p->list);
3064 			goto out_unlock;
3065 		}
3066 	}
3067  out_unlock:
3068 	mutex_unlock(&ftrace_cmd_mutex);
3069 
3070 	return ret;
3071 }
3072 
3073 static int ftrace_process_regex(struct ftrace_hash *hash,
3074 				char *buff, int len, int enable)
3075 {
3076 	char *func, *command, *next = buff;
3077 	struct ftrace_func_command *p;
3078 	int ret = -EINVAL;
3079 
3080 	func = strsep(&next, ":");
3081 
3082 	if (!next) {
3083 		ret = ftrace_match_records(hash, func, len);
3084 		if (!ret)
3085 			ret = -EINVAL;
3086 		if (ret < 0)
3087 			return ret;
3088 		return 0;
3089 	}
3090 
3091 	/* command found */
3092 
3093 	command = strsep(&next, ":");
3094 
3095 	mutex_lock(&ftrace_cmd_mutex);
3096 	list_for_each_entry(p, &ftrace_commands, list) {
3097 		if (strcmp(p->name, command) == 0) {
3098 			ret = p->func(hash, func, command, next, enable);
3099 			goto out_unlock;
3100 		}
3101 	}
3102  out_unlock:
3103 	mutex_unlock(&ftrace_cmd_mutex);
3104 
3105 	return ret;
3106 }
3107 
3108 static ssize_t
3109 ftrace_regex_write(struct file *file, const char __user *ubuf,
3110 		   size_t cnt, loff_t *ppos, int enable)
3111 {
3112 	struct ftrace_iterator *iter;
3113 	struct trace_parser *parser;
3114 	ssize_t ret, read;
3115 
3116 	if (!cnt)
3117 		return 0;
3118 
3119 	mutex_lock(&ftrace_regex_lock);
3120 
3121 	ret = -ENODEV;
3122 	if (unlikely(ftrace_disabled))
3123 		goto out_unlock;
3124 
3125 	if (file->f_mode & FMODE_READ) {
3126 		struct seq_file *m = file->private_data;
3127 		iter = m->private;
3128 	} else
3129 		iter = file->private_data;
3130 
3131 	parser = &iter->parser;
3132 	read = trace_get_user(parser, ubuf, cnt, ppos);
3133 
3134 	if (read >= 0 && trace_parser_loaded(parser) &&
3135 	    !trace_parser_cont(parser)) {
3136 		ret = ftrace_process_regex(iter->hash, parser->buffer,
3137 					   parser->idx, enable);
3138 		trace_parser_clear(parser);
3139 		if (ret)
3140 			goto out_unlock;
3141 	}
3142 
3143 	ret = read;
3144 out_unlock:
3145 	mutex_unlock(&ftrace_regex_lock);
3146 
3147 	return ret;
3148 }
3149 
3150 ssize_t
3151 ftrace_filter_write(struct file *file, const char __user *ubuf,
3152 		    size_t cnt, loff_t *ppos)
3153 {
3154 	return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
3155 }
3156 
3157 ssize_t
3158 ftrace_notrace_write(struct file *file, const char __user *ubuf,
3159 		     size_t cnt, loff_t *ppos)
3160 {
3161 	return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
3162 }
3163 
3164 static int
3165 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
3166 		 int reset, int enable)
3167 {
3168 	struct ftrace_hash **orig_hash;
3169 	struct ftrace_hash *hash;
3170 	int ret;
3171 
3172 	/* All global ops uses the global ops filters */
3173 	if (ops->flags & FTRACE_OPS_FL_GLOBAL)
3174 		ops = &global_ops;
3175 
3176 	if (unlikely(ftrace_disabled))
3177 		return -ENODEV;
3178 
3179 	if (enable)
3180 		orig_hash = &ops->filter_hash;
3181 	else
3182 		orig_hash = &ops->notrace_hash;
3183 
3184 	hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3185 	if (!hash)
3186 		return -ENOMEM;
3187 
3188 	mutex_lock(&ftrace_regex_lock);
3189 	if (reset)
3190 		ftrace_filter_reset(hash);
3191 	if (buf && !ftrace_match_records(hash, buf, len)) {
3192 		ret = -EINVAL;
3193 		goto out_regex_unlock;
3194 	}
3195 
3196 	mutex_lock(&ftrace_lock);
3197 	ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3198 	if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
3199 	    && ftrace_enabled)
3200 		ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3201 
3202 	mutex_unlock(&ftrace_lock);
3203 
3204  out_regex_unlock:
3205 	mutex_unlock(&ftrace_regex_lock);
3206 
3207 	free_ftrace_hash(hash);
3208 	return ret;
3209 }
3210 
3211 /**
3212  * ftrace_set_filter - set a function to filter on in ftrace
3213  * @ops - the ops to set the filter with
3214  * @buf - the string that holds the function filter text.
3215  * @len - the length of the string.
3216  * @reset - non zero to reset all filters before applying this filter.
3217  *
3218  * Filters denote which functions should be enabled when tracing is enabled.
3219  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3220  */
3221 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
3222 		       int len, int reset)
3223 {
3224 	return ftrace_set_regex(ops, buf, len, reset, 1);
3225 }
3226 EXPORT_SYMBOL_GPL(ftrace_set_filter);
3227 
3228 /**
3229  * ftrace_set_notrace - set a function to not trace in ftrace
3230  * @ops - the ops to set the notrace filter with
3231  * @buf - the string that holds the function notrace text.
3232  * @len - the length of the string.
3233  * @reset - non zero to reset all filters before applying this filter.
3234  *
3235  * Notrace Filters denote which functions should not be enabled when tracing
3236  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3237  * for tracing.
3238  */
3239 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3240 			int len, int reset)
3241 {
3242 	return ftrace_set_regex(ops, buf, len, reset, 0);
3243 }
3244 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
3245 /**
3246  * ftrace_set_filter - set a function to filter on in ftrace
3247  * @ops - the ops to set the filter with
3248  * @buf - the string that holds the function filter text.
3249  * @len - the length of the string.
3250  * @reset - non zero to reset all filters before applying this filter.
3251  *
3252  * Filters denote which functions should be enabled when tracing is enabled.
3253  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3254  */
3255 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3256 {
3257 	ftrace_set_regex(&global_ops, buf, len, reset, 1);
3258 }
3259 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3260 
3261 /**
3262  * ftrace_set_notrace - set a function to not trace in ftrace
3263  * @ops - the ops to set the notrace filter with
3264  * @buf - the string that holds the function notrace text.
3265  * @len - the length of the string.
3266  * @reset - non zero to reset all filters before applying this filter.
3267  *
3268  * Notrace Filters denote which functions should not be enabled when tracing
3269  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3270  * for tracing.
3271  */
3272 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3273 {
3274 	ftrace_set_regex(&global_ops, buf, len, reset, 0);
3275 }
3276 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3277 
3278 /*
3279  * command line interface to allow users to set filters on boot up.
3280  */
3281 #define FTRACE_FILTER_SIZE		COMMAND_LINE_SIZE
3282 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3283 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3284 
3285 static int __init set_ftrace_notrace(char *str)
3286 {
3287 	strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3288 	return 1;
3289 }
3290 __setup("ftrace_notrace=", set_ftrace_notrace);
3291 
3292 static int __init set_ftrace_filter(char *str)
3293 {
3294 	strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3295 	return 1;
3296 }
3297 __setup("ftrace_filter=", set_ftrace_filter);
3298 
3299 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3300 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3301 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
3302 
3303 static int __init set_graph_function(char *str)
3304 {
3305 	strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3306 	return 1;
3307 }
3308 __setup("ftrace_graph_filter=", set_graph_function);
3309 
3310 static void __init set_ftrace_early_graph(char *buf)
3311 {
3312 	int ret;
3313 	char *func;
3314 
3315 	while (buf) {
3316 		func = strsep(&buf, ",");
3317 		/* we allow only one expression at a time */
3318 		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3319 				      func);
3320 		if (ret)
3321 			printk(KERN_DEBUG "ftrace: function %s not "
3322 					  "traceable\n", func);
3323 	}
3324 }
3325 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3326 
3327 void __init
3328 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3329 {
3330 	char *func;
3331 
3332 	while (buf) {
3333 		func = strsep(&buf, ",");
3334 		ftrace_set_regex(ops, func, strlen(func), 0, enable);
3335 	}
3336 }
3337 
3338 static void __init set_ftrace_early_filters(void)
3339 {
3340 	if (ftrace_filter_buf[0])
3341 		ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
3342 	if (ftrace_notrace_buf[0])
3343 		ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
3344 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3345 	if (ftrace_graph_buf[0])
3346 		set_ftrace_early_graph(ftrace_graph_buf);
3347 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3348 }
3349 
3350 int ftrace_regex_release(struct inode *inode, struct file *file)
3351 {
3352 	struct seq_file *m = (struct seq_file *)file->private_data;
3353 	struct ftrace_iterator *iter;
3354 	struct ftrace_hash **orig_hash;
3355 	struct trace_parser *parser;
3356 	int filter_hash;
3357 	int ret;
3358 
3359 	mutex_lock(&ftrace_regex_lock);
3360 	if (file->f_mode & FMODE_READ) {
3361 		iter = m->private;
3362 
3363 		seq_release(inode, file);
3364 	} else
3365 		iter = file->private_data;
3366 
3367 	parser = &iter->parser;
3368 	if (trace_parser_loaded(parser)) {
3369 		parser->buffer[parser->idx] = 0;
3370 		ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3371 	}
3372 
3373 	trace_parser_put(parser);
3374 
3375 	if (file->f_mode & FMODE_WRITE) {
3376 		filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3377 
3378 		if (filter_hash)
3379 			orig_hash = &iter->ops->filter_hash;
3380 		else
3381 			orig_hash = &iter->ops->notrace_hash;
3382 
3383 		mutex_lock(&ftrace_lock);
3384 		ret = ftrace_hash_move(iter->ops, filter_hash,
3385 				       orig_hash, iter->hash);
3386 		if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
3387 		    && ftrace_enabled)
3388 			ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3389 
3390 		mutex_unlock(&ftrace_lock);
3391 	}
3392 	free_ftrace_hash(iter->hash);
3393 	kfree(iter);
3394 
3395 	mutex_unlock(&ftrace_regex_lock);
3396 	return 0;
3397 }
3398 
3399 static const struct file_operations ftrace_avail_fops = {
3400 	.open = ftrace_avail_open,
3401 	.read = seq_read,
3402 	.llseek = seq_lseek,
3403 	.release = seq_release_private,
3404 };
3405 
3406 static const struct file_operations ftrace_enabled_fops = {
3407 	.open = ftrace_enabled_open,
3408 	.read = seq_read,
3409 	.llseek = seq_lseek,
3410 	.release = seq_release_private,
3411 };
3412 
3413 static const struct file_operations ftrace_filter_fops = {
3414 	.open = ftrace_filter_open,
3415 	.read = seq_read,
3416 	.write = ftrace_filter_write,
3417 	.llseek = ftrace_regex_lseek,
3418 	.release = ftrace_regex_release,
3419 };
3420 
3421 static const struct file_operations ftrace_notrace_fops = {
3422 	.open = ftrace_notrace_open,
3423 	.read = seq_read,
3424 	.write = ftrace_notrace_write,
3425 	.llseek = ftrace_regex_lseek,
3426 	.release = ftrace_regex_release,
3427 };
3428 
3429 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3430 
3431 static DEFINE_MUTEX(graph_lock);
3432 
3433 int ftrace_graph_count;
3434 int ftrace_graph_filter_enabled;
3435 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3436 
3437 static void *
3438 __g_next(struct seq_file *m, loff_t *pos)
3439 {
3440 	if (*pos >= ftrace_graph_count)
3441 		return NULL;
3442 	return &ftrace_graph_funcs[*pos];
3443 }
3444 
3445 static void *
3446 g_next(struct seq_file *m, void *v, loff_t *pos)
3447 {
3448 	(*pos)++;
3449 	return __g_next(m, pos);
3450 }
3451 
3452 static void *g_start(struct seq_file *m, loff_t *pos)
3453 {
3454 	mutex_lock(&graph_lock);
3455 
3456 	/* Nothing, tell g_show to print all functions are enabled */
3457 	if (!ftrace_graph_filter_enabled && !*pos)
3458 		return (void *)1;
3459 
3460 	return __g_next(m, pos);
3461 }
3462 
3463 static void g_stop(struct seq_file *m, void *p)
3464 {
3465 	mutex_unlock(&graph_lock);
3466 }
3467 
3468 static int g_show(struct seq_file *m, void *v)
3469 {
3470 	unsigned long *ptr = v;
3471 
3472 	if (!ptr)
3473 		return 0;
3474 
3475 	if (ptr == (unsigned long *)1) {
3476 		seq_printf(m, "#### all functions enabled ####\n");
3477 		return 0;
3478 	}
3479 
3480 	seq_printf(m, "%ps\n", (void *)*ptr);
3481 
3482 	return 0;
3483 }
3484 
3485 static const struct seq_operations ftrace_graph_seq_ops = {
3486 	.start = g_start,
3487 	.next = g_next,
3488 	.stop = g_stop,
3489 	.show = g_show,
3490 };
3491 
3492 static int
3493 ftrace_graph_open(struct inode *inode, struct file *file)
3494 {
3495 	int ret = 0;
3496 
3497 	if (unlikely(ftrace_disabled))
3498 		return -ENODEV;
3499 
3500 	mutex_lock(&graph_lock);
3501 	if ((file->f_mode & FMODE_WRITE) &&
3502 	    (file->f_flags & O_TRUNC)) {
3503 		ftrace_graph_filter_enabled = 0;
3504 		ftrace_graph_count = 0;
3505 		memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3506 	}
3507 	mutex_unlock(&graph_lock);
3508 
3509 	if (file->f_mode & FMODE_READ)
3510 		ret = seq_open(file, &ftrace_graph_seq_ops);
3511 
3512 	return ret;
3513 }
3514 
3515 static int
3516 ftrace_graph_release(struct inode *inode, struct file *file)
3517 {
3518 	if (file->f_mode & FMODE_READ)
3519 		seq_release(inode, file);
3520 	return 0;
3521 }
3522 
3523 static int
3524 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3525 {
3526 	struct dyn_ftrace *rec;
3527 	struct ftrace_page *pg;
3528 	int search_len;
3529 	int fail = 1;
3530 	int type, not;
3531 	char *search;
3532 	bool exists;
3533 	int i;
3534 
3535 	/* decode regex */
3536 	type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3537 	if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3538 		return -EBUSY;
3539 
3540 	search_len = strlen(search);
3541 
3542 	mutex_lock(&ftrace_lock);
3543 
3544 	if (unlikely(ftrace_disabled)) {
3545 		mutex_unlock(&ftrace_lock);
3546 		return -ENODEV;
3547 	}
3548 
3549 	do_for_each_ftrace_rec(pg, rec) {
3550 
3551 		if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3552 			/* if it is in the array */
3553 			exists = false;
3554 			for (i = 0; i < *idx; i++) {
3555 				if (array[i] == rec->ip) {
3556 					exists = true;
3557 					break;
3558 				}
3559 			}
3560 
3561 			if (!not) {
3562 				fail = 0;
3563 				if (!exists) {
3564 					array[(*idx)++] = rec->ip;
3565 					if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3566 						goto out;
3567 				}
3568 			} else {
3569 				if (exists) {
3570 					array[i] = array[--(*idx)];
3571 					array[*idx] = 0;
3572 					fail = 0;
3573 				}
3574 			}
3575 		}
3576 	} while_for_each_ftrace_rec();
3577 out:
3578 	mutex_unlock(&ftrace_lock);
3579 
3580 	if (fail)
3581 		return -EINVAL;
3582 
3583 	ftrace_graph_filter_enabled = 1;
3584 	return 0;
3585 }
3586 
3587 static ssize_t
3588 ftrace_graph_write(struct file *file, const char __user *ubuf,
3589 		   size_t cnt, loff_t *ppos)
3590 {
3591 	struct trace_parser parser;
3592 	ssize_t read, ret;
3593 
3594 	if (!cnt)
3595 		return 0;
3596 
3597 	mutex_lock(&graph_lock);
3598 
3599 	if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3600 		ret = -ENOMEM;
3601 		goto out_unlock;
3602 	}
3603 
3604 	read = trace_get_user(&parser, ubuf, cnt, ppos);
3605 
3606 	if (read >= 0 && trace_parser_loaded((&parser))) {
3607 		parser.buffer[parser.idx] = 0;
3608 
3609 		/* we allow only one expression at a time */
3610 		ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3611 					parser.buffer);
3612 		if (ret)
3613 			goto out_free;
3614 	}
3615 
3616 	ret = read;
3617 
3618 out_free:
3619 	trace_parser_put(&parser);
3620 out_unlock:
3621 	mutex_unlock(&graph_lock);
3622 
3623 	return ret;
3624 }
3625 
3626 static const struct file_operations ftrace_graph_fops = {
3627 	.open		= ftrace_graph_open,
3628 	.read		= seq_read,
3629 	.write		= ftrace_graph_write,
3630 	.release	= ftrace_graph_release,
3631 	.llseek		= seq_lseek,
3632 };
3633 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3634 
3635 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3636 {
3637 
3638 	trace_create_file("available_filter_functions", 0444,
3639 			d_tracer, NULL, &ftrace_avail_fops);
3640 
3641 	trace_create_file("enabled_functions", 0444,
3642 			d_tracer, NULL, &ftrace_enabled_fops);
3643 
3644 	trace_create_file("set_ftrace_filter", 0644, d_tracer,
3645 			NULL, &ftrace_filter_fops);
3646 
3647 	trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3648 				    NULL, &ftrace_notrace_fops);
3649 
3650 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3651 	trace_create_file("set_graph_function", 0444, d_tracer,
3652 				    NULL,
3653 				    &ftrace_graph_fops);
3654 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3655 
3656 	return 0;
3657 }
3658 
3659 static int ftrace_cmp_ips(const void *a, const void *b)
3660 {
3661 	const unsigned long *ipa = a;
3662 	const unsigned long *ipb = b;
3663 
3664 	if (*ipa > *ipb)
3665 		return 1;
3666 	if (*ipa < *ipb)
3667 		return -1;
3668 	return 0;
3669 }
3670 
3671 static void ftrace_swap_ips(void *a, void *b, int size)
3672 {
3673 	unsigned long *ipa = a;
3674 	unsigned long *ipb = b;
3675 	unsigned long t;
3676 
3677 	t = *ipa;
3678 	*ipa = *ipb;
3679 	*ipb = t;
3680 }
3681 
3682 static int ftrace_process_locs(struct module *mod,
3683 			       unsigned long *start,
3684 			       unsigned long *end)
3685 {
3686 	struct ftrace_page *start_pg;
3687 	struct ftrace_page *pg;
3688 	struct dyn_ftrace *rec;
3689 	unsigned long count;
3690 	unsigned long *p;
3691 	unsigned long addr;
3692 	unsigned long flags = 0; /* Shut up gcc */
3693 	int ret = -ENOMEM;
3694 
3695 	count = end - start;
3696 
3697 	if (!count)
3698 		return 0;
3699 
3700 	sort(start, count, sizeof(*start),
3701 	     ftrace_cmp_ips, ftrace_swap_ips);
3702 
3703 	start_pg = ftrace_allocate_pages(count);
3704 	if (!start_pg)
3705 		return -ENOMEM;
3706 
3707 	mutex_lock(&ftrace_lock);
3708 
3709 	/*
3710 	 * Core and each module needs their own pages, as
3711 	 * modules will free them when they are removed.
3712 	 * Force a new page to be allocated for modules.
3713 	 */
3714 	if (!mod) {
3715 		WARN_ON(ftrace_pages || ftrace_pages_start);
3716 		/* First initialization */
3717 		ftrace_pages = ftrace_pages_start = start_pg;
3718 	} else {
3719 		if (!ftrace_pages)
3720 			goto out;
3721 
3722 		if (WARN_ON(ftrace_pages->next)) {
3723 			/* Hmm, we have free pages? */
3724 			while (ftrace_pages->next)
3725 				ftrace_pages = ftrace_pages->next;
3726 		}
3727 
3728 		ftrace_pages->next = start_pg;
3729 	}
3730 
3731 	p = start;
3732 	pg = start_pg;
3733 	while (p < end) {
3734 		addr = ftrace_call_adjust(*p++);
3735 		/*
3736 		 * Some architecture linkers will pad between
3737 		 * the different mcount_loc sections of different
3738 		 * object files to satisfy alignments.
3739 		 * Skip any NULL pointers.
3740 		 */
3741 		if (!addr)
3742 			continue;
3743 
3744 		if (pg->index == pg->size) {
3745 			/* We should have allocated enough */
3746 			if (WARN_ON(!pg->next))
3747 				break;
3748 			pg = pg->next;
3749 		}
3750 
3751 		rec = &pg->records[pg->index++];
3752 		rec->ip = addr;
3753 	}
3754 
3755 	/* We should have used all pages */
3756 	WARN_ON(pg->next);
3757 
3758 	/* Assign the last page to ftrace_pages */
3759 	ftrace_pages = pg;
3760 
3761 	/* These new locations need to be initialized */
3762 	ftrace_new_pgs = start_pg;
3763 
3764 	/*
3765 	 * We only need to disable interrupts on start up
3766 	 * because we are modifying code that an interrupt
3767 	 * may execute, and the modification is not atomic.
3768 	 * But for modules, nothing runs the code we modify
3769 	 * until we are finished with it, and there's no
3770 	 * reason to cause large interrupt latencies while we do it.
3771 	 */
3772 	if (!mod)
3773 		local_irq_save(flags);
3774 	ftrace_update_code(mod);
3775 	if (!mod)
3776 		local_irq_restore(flags);
3777 	ret = 0;
3778  out:
3779 	mutex_unlock(&ftrace_lock);
3780 
3781 	return ret;
3782 }
3783 
3784 #ifdef CONFIG_MODULES
3785 
3786 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
3787 
3788 void ftrace_release_mod(struct module *mod)
3789 {
3790 	struct dyn_ftrace *rec;
3791 	struct ftrace_page **last_pg;
3792 	struct ftrace_page *pg;
3793 	int order;
3794 
3795 	mutex_lock(&ftrace_lock);
3796 
3797 	if (ftrace_disabled)
3798 		goto out_unlock;
3799 
3800 	/*
3801 	 * Each module has its own ftrace_pages, remove
3802 	 * them from the list.
3803 	 */
3804 	last_pg = &ftrace_pages_start;
3805 	for (pg = ftrace_pages_start; pg; pg = *last_pg) {
3806 		rec = &pg->records[0];
3807 		if (within_module_core(rec->ip, mod)) {
3808 			/*
3809 			 * As core pages are first, the first
3810 			 * page should never be a module page.
3811 			 */
3812 			if (WARN_ON(pg == ftrace_pages_start))
3813 				goto out_unlock;
3814 
3815 			/* Check if we are deleting the last page */
3816 			if (pg == ftrace_pages)
3817 				ftrace_pages = next_to_ftrace_page(last_pg);
3818 
3819 			*last_pg = pg->next;
3820 			order = get_count_order(pg->size / ENTRIES_PER_PAGE);
3821 			free_pages((unsigned long)pg->records, order);
3822 			kfree(pg);
3823 		} else
3824 			last_pg = &pg->next;
3825 	}
3826  out_unlock:
3827 	mutex_unlock(&ftrace_lock);
3828 }
3829 
3830 static void ftrace_init_module(struct module *mod,
3831 			       unsigned long *start, unsigned long *end)
3832 {
3833 	if (ftrace_disabled || start == end)
3834 		return;
3835 	ftrace_process_locs(mod, start, end);
3836 }
3837 
3838 static int ftrace_module_notify(struct notifier_block *self,
3839 				unsigned long val, void *data)
3840 {
3841 	struct module *mod = data;
3842 
3843 	switch (val) {
3844 	case MODULE_STATE_COMING:
3845 		ftrace_init_module(mod, mod->ftrace_callsites,
3846 				   mod->ftrace_callsites +
3847 				   mod->num_ftrace_callsites);
3848 		break;
3849 	case MODULE_STATE_GOING:
3850 		ftrace_release_mod(mod);
3851 		break;
3852 	}
3853 
3854 	return 0;
3855 }
3856 #else
3857 static int ftrace_module_notify(struct notifier_block *self,
3858 				unsigned long val, void *data)
3859 {
3860 	return 0;
3861 }
3862 #endif /* CONFIG_MODULES */
3863 
3864 struct notifier_block ftrace_module_nb = {
3865 	.notifier_call = ftrace_module_notify,
3866 	.priority = 0,
3867 };
3868 
3869 extern unsigned long __start_mcount_loc[];
3870 extern unsigned long __stop_mcount_loc[];
3871 
3872 void __init ftrace_init(void)
3873 {
3874 	unsigned long count, addr, flags;
3875 	int ret;
3876 
3877 	/* Keep the ftrace pointer to the stub */
3878 	addr = (unsigned long)ftrace_stub;
3879 
3880 	local_irq_save(flags);
3881 	ftrace_dyn_arch_init(&addr);
3882 	local_irq_restore(flags);
3883 
3884 	/* ftrace_dyn_arch_init places the return code in addr */
3885 	if (addr)
3886 		goto failed;
3887 
3888 	count = __stop_mcount_loc - __start_mcount_loc;
3889 
3890 	ret = ftrace_dyn_table_alloc(count);
3891 	if (ret)
3892 		goto failed;
3893 
3894 	last_ftrace_enabled = ftrace_enabled = 1;
3895 
3896 	ret = ftrace_process_locs(NULL,
3897 				  __start_mcount_loc,
3898 				  __stop_mcount_loc);
3899 
3900 	ret = register_module_notifier(&ftrace_module_nb);
3901 	if (ret)
3902 		pr_warning("Failed to register trace ftrace module notifier\n");
3903 
3904 	set_ftrace_early_filters();
3905 
3906 	return;
3907  failed:
3908 	ftrace_disabled = 1;
3909 }
3910 
3911 #else
3912 
3913 static struct ftrace_ops global_ops = {
3914 	.func			= ftrace_stub,
3915 };
3916 
3917 static int __init ftrace_nodyn_init(void)
3918 {
3919 	ftrace_enabled = 1;
3920 	return 0;
3921 }
3922 device_initcall(ftrace_nodyn_init);
3923 
3924 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
3925 static inline void ftrace_startup_enable(int command) { }
3926 /* Keep as macros so we do not need to define the commands */
3927 # define ftrace_startup(ops, command)			\
3928 	({						\
3929 		(ops)->flags |= FTRACE_OPS_FL_ENABLED;	\
3930 		0;					\
3931 	})
3932 # define ftrace_shutdown(ops, command)	do { } while (0)
3933 # define ftrace_startup_sysctl()	do { } while (0)
3934 # define ftrace_shutdown_sysctl()	do { } while (0)
3935 
3936 static inline int
3937 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3938 {
3939 	return 1;
3940 }
3941 
3942 #endif /* CONFIG_DYNAMIC_FTRACE */
3943 
3944 static void
3945 ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip)
3946 {
3947 	struct ftrace_ops *op;
3948 
3949 	if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
3950 		return;
3951 
3952 	/*
3953 	 * Some of the ops may be dynamically allocated,
3954 	 * they must be freed after a synchronize_sched().
3955 	 */
3956 	preempt_disable_notrace();
3957 	trace_recursion_set(TRACE_CONTROL_BIT);
3958 	op = rcu_dereference_raw(ftrace_control_list);
3959 	while (op != &ftrace_list_end) {
3960 		if (!ftrace_function_local_disabled(op) &&
3961 		    ftrace_ops_test(op, ip))
3962 			op->func(ip, parent_ip);
3963 
3964 		op = rcu_dereference_raw(op->next);
3965 	};
3966 	trace_recursion_clear(TRACE_CONTROL_BIT);
3967 	preempt_enable_notrace();
3968 }
3969 
3970 static struct ftrace_ops control_ops = {
3971 	.func = ftrace_ops_control_func,
3972 };
3973 
3974 static void
3975 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3976 {
3977 	struct ftrace_ops *op;
3978 
3979 	if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
3980 		return;
3981 
3982 	trace_recursion_set(TRACE_INTERNAL_BIT);
3983 	/*
3984 	 * Some of the ops may be dynamically allocated,
3985 	 * they must be freed after a synchronize_sched().
3986 	 */
3987 	preempt_disable_notrace();
3988 	op = rcu_dereference_raw(ftrace_ops_list);
3989 	while (op != &ftrace_list_end) {
3990 		if (ftrace_ops_test(op, ip))
3991 			op->func(ip, parent_ip);
3992 		op = rcu_dereference_raw(op->next);
3993 	};
3994 	preempt_enable_notrace();
3995 	trace_recursion_clear(TRACE_INTERNAL_BIT);
3996 }
3997 
3998 static void clear_ftrace_swapper(void)
3999 {
4000 	struct task_struct *p;
4001 	int cpu;
4002 
4003 	get_online_cpus();
4004 	for_each_online_cpu(cpu) {
4005 		p = idle_task(cpu);
4006 		clear_tsk_trace_trace(p);
4007 	}
4008 	put_online_cpus();
4009 }
4010 
4011 static void set_ftrace_swapper(void)
4012 {
4013 	struct task_struct *p;
4014 	int cpu;
4015 
4016 	get_online_cpus();
4017 	for_each_online_cpu(cpu) {
4018 		p = idle_task(cpu);
4019 		set_tsk_trace_trace(p);
4020 	}
4021 	put_online_cpus();
4022 }
4023 
4024 static void clear_ftrace_pid(struct pid *pid)
4025 {
4026 	struct task_struct *p;
4027 
4028 	rcu_read_lock();
4029 	do_each_pid_task(pid, PIDTYPE_PID, p) {
4030 		clear_tsk_trace_trace(p);
4031 	} while_each_pid_task(pid, PIDTYPE_PID, p);
4032 	rcu_read_unlock();
4033 
4034 	put_pid(pid);
4035 }
4036 
4037 static void set_ftrace_pid(struct pid *pid)
4038 {
4039 	struct task_struct *p;
4040 
4041 	rcu_read_lock();
4042 	do_each_pid_task(pid, PIDTYPE_PID, p) {
4043 		set_tsk_trace_trace(p);
4044 	} while_each_pid_task(pid, PIDTYPE_PID, p);
4045 	rcu_read_unlock();
4046 }
4047 
4048 static void clear_ftrace_pid_task(struct pid *pid)
4049 {
4050 	if (pid == ftrace_swapper_pid)
4051 		clear_ftrace_swapper();
4052 	else
4053 		clear_ftrace_pid(pid);
4054 }
4055 
4056 static void set_ftrace_pid_task(struct pid *pid)
4057 {
4058 	if (pid == ftrace_swapper_pid)
4059 		set_ftrace_swapper();
4060 	else
4061 		set_ftrace_pid(pid);
4062 }
4063 
4064 static int ftrace_pid_add(int p)
4065 {
4066 	struct pid *pid;
4067 	struct ftrace_pid *fpid;
4068 	int ret = -EINVAL;
4069 
4070 	mutex_lock(&ftrace_lock);
4071 
4072 	if (!p)
4073 		pid = ftrace_swapper_pid;
4074 	else
4075 		pid = find_get_pid(p);
4076 
4077 	if (!pid)
4078 		goto out;
4079 
4080 	ret = 0;
4081 
4082 	list_for_each_entry(fpid, &ftrace_pids, list)
4083 		if (fpid->pid == pid)
4084 			goto out_put;
4085 
4086 	ret = -ENOMEM;
4087 
4088 	fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
4089 	if (!fpid)
4090 		goto out_put;
4091 
4092 	list_add(&fpid->list, &ftrace_pids);
4093 	fpid->pid = pid;
4094 
4095 	set_ftrace_pid_task(pid);
4096 
4097 	ftrace_update_pid_func();
4098 	ftrace_startup_enable(0);
4099 
4100 	mutex_unlock(&ftrace_lock);
4101 	return 0;
4102 
4103 out_put:
4104 	if (pid != ftrace_swapper_pid)
4105 		put_pid(pid);
4106 
4107 out:
4108 	mutex_unlock(&ftrace_lock);
4109 	return ret;
4110 }
4111 
4112 static void ftrace_pid_reset(void)
4113 {
4114 	struct ftrace_pid *fpid, *safe;
4115 
4116 	mutex_lock(&ftrace_lock);
4117 	list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
4118 		struct pid *pid = fpid->pid;
4119 
4120 		clear_ftrace_pid_task(pid);
4121 
4122 		list_del(&fpid->list);
4123 		kfree(fpid);
4124 	}
4125 
4126 	ftrace_update_pid_func();
4127 	ftrace_startup_enable(0);
4128 
4129 	mutex_unlock(&ftrace_lock);
4130 }
4131 
4132 static void *fpid_start(struct seq_file *m, loff_t *pos)
4133 {
4134 	mutex_lock(&ftrace_lock);
4135 
4136 	if (list_empty(&ftrace_pids) && (!*pos))
4137 		return (void *) 1;
4138 
4139 	return seq_list_start(&ftrace_pids, *pos);
4140 }
4141 
4142 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
4143 {
4144 	if (v == (void *)1)
4145 		return NULL;
4146 
4147 	return seq_list_next(v, &ftrace_pids, pos);
4148 }
4149 
4150 static void fpid_stop(struct seq_file *m, void *p)
4151 {
4152 	mutex_unlock(&ftrace_lock);
4153 }
4154 
4155 static int fpid_show(struct seq_file *m, void *v)
4156 {
4157 	const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
4158 
4159 	if (v == (void *)1) {
4160 		seq_printf(m, "no pid\n");
4161 		return 0;
4162 	}
4163 
4164 	if (fpid->pid == ftrace_swapper_pid)
4165 		seq_printf(m, "swapper tasks\n");
4166 	else
4167 		seq_printf(m, "%u\n", pid_vnr(fpid->pid));
4168 
4169 	return 0;
4170 }
4171 
4172 static const struct seq_operations ftrace_pid_sops = {
4173 	.start = fpid_start,
4174 	.next = fpid_next,
4175 	.stop = fpid_stop,
4176 	.show = fpid_show,
4177 };
4178 
4179 static int
4180 ftrace_pid_open(struct inode *inode, struct file *file)
4181 {
4182 	int ret = 0;
4183 
4184 	if ((file->f_mode & FMODE_WRITE) &&
4185 	    (file->f_flags & O_TRUNC))
4186 		ftrace_pid_reset();
4187 
4188 	if (file->f_mode & FMODE_READ)
4189 		ret = seq_open(file, &ftrace_pid_sops);
4190 
4191 	return ret;
4192 }
4193 
4194 static ssize_t
4195 ftrace_pid_write(struct file *filp, const char __user *ubuf,
4196 		   size_t cnt, loff_t *ppos)
4197 {
4198 	char buf[64], *tmp;
4199 	long val;
4200 	int ret;
4201 
4202 	if (cnt >= sizeof(buf))
4203 		return -EINVAL;
4204 
4205 	if (copy_from_user(&buf, ubuf, cnt))
4206 		return -EFAULT;
4207 
4208 	buf[cnt] = 0;
4209 
4210 	/*
4211 	 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
4212 	 * to clean the filter quietly.
4213 	 */
4214 	tmp = strstrip(buf);
4215 	if (strlen(tmp) == 0)
4216 		return 1;
4217 
4218 	ret = strict_strtol(tmp, 10, &val);
4219 	if (ret < 0)
4220 		return ret;
4221 
4222 	ret = ftrace_pid_add(val);
4223 
4224 	return ret ? ret : cnt;
4225 }
4226 
4227 static int
4228 ftrace_pid_release(struct inode *inode, struct file *file)
4229 {
4230 	if (file->f_mode & FMODE_READ)
4231 		seq_release(inode, file);
4232 
4233 	return 0;
4234 }
4235 
4236 static const struct file_operations ftrace_pid_fops = {
4237 	.open		= ftrace_pid_open,
4238 	.write		= ftrace_pid_write,
4239 	.read		= seq_read,
4240 	.llseek		= seq_lseek,
4241 	.release	= ftrace_pid_release,
4242 };
4243 
4244 static __init int ftrace_init_debugfs(void)
4245 {
4246 	struct dentry *d_tracer;
4247 
4248 	d_tracer = tracing_init_dentry();
4249 	if (!d_tracer)
4250 		return 0;
4251 
4252 	ftrace_init_dyn_debugfs(d_tracer);
4253 
4254 	trace_create_file("set_ftrace_pid", 0644, d_tracer,
4255 			    NULL, &ftrace_pid_fops);
4256 
4257 	ftrace_profile_debugfs(d_tracer);
4258 
4259 	return 0;
4260 }
4261 fs_initcall(ftrace_init_debugfs);
4262 
4263 /**
4264  * ftrace_kill - kill ftrace
4265  *
4266  * This function should be used by panic code. It stops ftrace
4267  * but in a not so nice way. If you need to simply kill ftrace
4268  * from a non-atomic section, use ftrace_kill.
4269  */
4270 void ftrace_kill(void)
4271 {
4272 	ftrace_disabled = 1;
4273 	ftrace_enabled = 0;
4274 	clear_ftrace_function();
4275 }
4276 
4277 /**
4278  * Test if ftrace is dead or not.
4279  */
4280 int ftrace_is_dead(void)
4281 {
4282 	return ftrace_disabled;
4283 }
4284 
4285 /**
4286  * register_ftrace_function - register a function for profiling
4287  * @ops - ops structure that holds the function for profiling.
4288  *
4289  * Register a function to be called by all functions in the
4290  * kernel.
4291  *
4292  * Note: @ops->func and all the functions it calls must be labeled
4293  *       with "notrace", otherwise it will go into a
4294  *       recursive loop.
4295  */
4296 int register_ftrace_function(struct ftrace_ops *ops)
4297 {
4298 	int ret = -1;
4299 
4300 	mutex_lock(&ftrace_lock);
4301 
4302 	if (unlikely(ftrace_disabled))
4303 		goto out_unlock;
4304 
4305 	ret = __register_ftrace_function(ops);
4306 	if (!ret)
4307 		ret = ftrace_startup(ops, 0);
4308 
4309 
4310  out_unlock:
4311 	mutex_unlock(&ftrace_lock);
4312 	return ret;
4313 }
4314 EXPORT_SYMBOL_GPL(register_ftrace_function);
4315 
4316 /**
4317  * unregister_ftrace_function - unregister a function for profiling.
4318  * @ops - ops structure that holds the function to unregister
4319  *
4320  * Unregister a function that was added to be called by ftrace profiling.
4321  */
4322 int unregister_ftrace_function(struct ftrace_ops *ops)
4323 {
4324 	int ret;
4325 
4326 	mutex_lock(&ftrace_lock);
4327 	ret = __unregister_ftrace_function(ops);
4328 	if (!ret)
4329 		ftrace_shutdown(ops, 0);
4330 	mutex_unlock(&ftrace_lock);
4331 
4332 	return ret;
4333 }
4334 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
4335 
4336 int
4337 ftrace_enable_sysctl(struct ctl_table *table, int write,
4338 		     void __user *buffer, size_t *lenp,
4339 		     loff_t *ppos)
4340 {
4341 	int ret = -ENODEV;
4342 
4343 	mutex_lock(&ftrace_lock);
4344 
4345 	if (unlikely(ftrace_disabled))
4346 		goto out;
4347 
4348 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
4349 
4350 	if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4351 		goto out;
4352 
4353 	last_ftrace_enabled = !!ftrace_enabled;
4354 
4355 	if (ftrace_enabled) {
4356 
4357 		ftrace_startup_sysctl();
4358 
4359 		/* we are starting ftrace again */
4360 		if (ftrace_ops_list != &ftrace_list_end) {
4361 			if (ftrace_ops_list->next == &ftrace_list_end)
4362 				ftrace_trace_function = ftrace_ops_list->func;
4363 			else
4364 				ftrace_trace_function = ftrace_ops_list_func;
4365 		}
4366 
4367 	} else {
4368 		/* stopping ftrace calls (just send to ftrace_stub) */
4369 		ftrace_trace_function = ftrace_stub;
4370 
4371 		ftrace_shutdown_sysctl();
4372 	}
4373 
4374  out:
4375 	mutex_unlock(&ftrace_lock);
4376 	return ret;
4377 }
4378 
4379 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4380 
4381 static int ftrace_graph_active;
4382 static struct notifier_block ftrace_suspend_notifier;
4383 
4384 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4385 {
4386 	return 0;
4387 }
4388 
4389 /* The callbacks that hook a function */
4390 trace_func_graph_ret_t ftrace_graph_return =
4391 			(trace_func_graph_ret_t)ftrace_stub;
4392 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4393 
4394 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4395 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
4396 {
4397 	int i;
4398 	int ret = 0;
4399 	unsigned long flags;
4400 	int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4401 	struct task_struct *g, *t;
4402 
4403 	for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4404 		ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4405 					* sizeof(struct ftrace_ret_stack),
4406 					GFP_KERNEL);
4407 		if (!ret_stack_list[i]) {
4408 			start = 0;
4409 			end = i;
4410 			ret = -ENOMEM;
4411 			goto free;
4412 		}
4413 	}
4414 
4415 	read_lock_irqsave(&tasklist_lock, flags);
4416 	do_each_thread(g, t) {
4417 		if (start == end) {
4418 			ret = -EAGAIN;
4419 			goto unlock;
4420 		}
4421 
4422 		if (t->ret_stack == NULL) {
4423 			atomic_set(&t->tracing_graph_pause, 0);
4424 			atomic_set(&t->trace_overrun, 0);
4425 			t->curr_ret_stack = -1;
4426 			/* Make sure the tasks see the -1 first: */
4427 			smp_wmb();
4428 			t->ret_stack = ret_stack_list[start++];
4429 		}
4430 	} while_each_thread(g, t);
4431 
4432 unlock:
4433 	read_unlock_irqrestore(&tasklist_lock, flags);
4434 free:
4435 	for (i = start; i < end; i++)
4436 		kfree(ret_stack_list[i]);
4437 	return ret;
4438 }
4439 
4440 static void
4441 ftrace_graph_probe_sched_switch(void *ignore,
4442 			struct task_struct *prev, struct task_struct *next)
4443 {
4444 	unsigned long long timestamp;
4445 	int index;
4446 
4447 	/*
4448 	 * Does the user want to count the time a function was asleep.
4449 	 * If so, do not update the time stamps.
4450 	 */
4451 	if (trace_flags & TRACE_ITER_SLEEP_TIME)
4452 		return;
4453 
4454 	timestamp = trace_clock_local();
4455 
4456 	prev->ftrace_timestamp = timestamp;
4457 
4458 	/* only process tasks that we timestamped */
4459 	if (!next->ftrace_timestamp)
4460 		return;
4461 
4462 	/*
4463 	 * Update all the counters in next to make up for the
4464 	 * time next was sleeping.
4465 	 */
4466 	timestamp -= next->ftrace_timestamp;
4467 
4468 	for (index = next->curr_ret_stack; index >= 0; index--)
4469 		next->ret_stack[index].calltime += timestamp;
4470 }
4471 
4472 /* Allocate a return stack for each task */
4473 static int start_graph_tracing(void)
4474 {
4475 	struct ftrace_ret_stack **ret_stack_list;
4476 	int ret, cpu;
4477 
4478 	ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4479 				sizeof(struct ftrace_ret_stack *),
4480 				GFP_KERNEL);
4481 
4482 	if (!ret_stack_list)
4483 		return -ENOMEM;
4484 
4485 	/* The cpu_boot init_task->ret_stack will never be freed */
4486 	for_each_online_cpu(cpu) {
4487 		if (!idle_task(cpu)->ret_stack)
4488 			ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4489 	}
4490 
4491 	do {
4492 		ret = alloc_retstack_tasklist(ret_stack_list);
4493 	} while (ret == -EAGAIN);
4494 
4495 	if (!ret) {
4496 		ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4497 		if (ret)
4498 			pr_info("ftrace_graph: Couldn't activate tracepoint"
4499 				" probe to kernel_sched_switch\n");
4500 	}
4501 
4502 	kfree(ret_stack_list);
4503 	return ret;
4504 }
4505 
4506 /*
4507  * Hibernation protection.
4508  * The state of the current task is too much unstable during
4509  * suspend/restore to disk. We want to protect against that.
4510  */
4511 static int
4512 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4513 							void *unused)
4514 {
4515 	switch (state) {
4516 	case PM_HIBERNATION_PREPARE:
4517 		pause_graph_tracing();
4518 		break;
4519 
4520 	case PM_POST_HIBERNATION:
4521 		unpause_graph_tracing();
4522 		break;
4523 	}
4524 	return NOTIFY_DONE;
4525 }
4526 
4527 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4528 			trace_func_graph_ent_t entryfunc)
4529 {
4530 	int ret = 0;
4531 
4532 	mutex_lock(&ftrace_lock);
4533 
4534 	/* we currently allow only one tracer registered at a time */
4535 	if (ftrace_graph_active) {
4536 		ret = -EBUSY;
4537 		goto out;
4538 	}
4539 
4540 	ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4541 	register_pm_notifier(&ftrace_suspend_notifier);
4542 
4543 	ftrace_graph_active++;
4544 	ret = start_graph_tracing();
4545 	if (ret) {
4546 		ftrace_graph_active--;
4547 		goto out;
4548 	}
4549 
4550 	ftrace_graph_return = retfunc;
4551 	ftrace_graph_entry = entryfunc;
4552 
4553 	ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
4554 
4555 out:
4556 	mutex_unlock(&ftrace_lock);
4557 	return ret;
4558 }
4559 
4560 void unregister_ftrace_graph(void)
4561 {
4562 	mutex_lock(&ftrace_lock);
4563 
4564 	if (unlikely(!ftrace_graph_active))
4565 		goto out;
4566 
4567 	ftrace_graph_active--;
4568 	ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4569 	ftrace_graph_entry = ftrace_graph_entry_stub;
4570 	ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
4571 	unregister_pm_notifier(&ftrace_suspend_notifier);
4572 	unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4573 
4574  out:
4575 	mutex_unlock(&ftrace_lock);
4576 }
4577 
4578 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4579 
4580 static void
4581 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4582 {
4583 	atomic_set(&t->tracing_graph_pause, 0);
4584 	atomic_set(&t->trace_overrun, 0);
4585 	t->ftrace_timestamp = 0;
4586 	/* make curr_ret_stack visible before we add the ret_stack */
4587 	smp_wmb();
4588 	t->ret_stack = ret_stack;
4589 }
4590 
4591 /*
4592  * Allocate a return stack for the idle task. May be the first
4593  * time through, or it may be done by CPU hotplug online.
4594  */
4595 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4596 {
4597 	t->curr_ret_stack = -1;
4598 	/*
4599 	 * The idle task has no parent, it either has its own
4600 	 * stack or no stack at all.
4601 	 */
4602 	if (t->ret_stack)
4603 		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4604 
4605 	if (ftrace_graph_active) {
4606 		struct ftrace_ret_stack *ret_stack;
4607 
4608 		ret_stack = per_cpu(idle_ret_stack, cpu);
4609 		if (!ret_stack) {
4610 			ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4611 					    * sizeof(struct ftrace_ret_stack),
4612 					    GFP_KERNEL);
4613 			if (!ret_stack)
4614 				return;
4615 			per_cpu(idle_ret_stack, cpu) = ret_stack;
4616 		}
4617 		graph_init_task(t, ret_stack);
4618 	}
4619 }
4620 
4621 /* Allocate a return stack for newly created task */
4622 void ftrace_graph_init_task(struct task_struct *t)
4623 {
4624 	/* Make sure we do not use the parent ret_stack */
4625 	t->ret_stack = NULL;
4626 	t->curr_ret_stack = -1;
4627 
4628 	if (ftrace_graph_active) {
4629 		struct ftrace_ret_stack *ret_stack;
4630 
4631 		ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4632 				* sizeof(struct ftrace_ret_stack),
4633 				GFP_KERNEL);
4634 		if (!ret_stack)
4635 			return;
4636 		graph_init_task(t, ret_stack);
4637 	}
4638 }
4639 
4640 void ftrace_graph_exit_task(struct task_struct *t)
4641 {
4642 	struct ftrace_ret_stack	*ret_stack = t->ret_stack;
4643 
4644 	t->ret_stack = NULL;
4645 	/* NULL must become visible to IRQs before we free it: */
4646 	barrier();
4647 
4648 	kfree(ret_stack);
4649 }
4650 
4651 void ftrace_graph_stop(void)
4652 {
4653 	ftrace_stop();
4654 }
4655 #endif
4656