Lines Matching +full:ftrace +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0
3 * Infrastructure for profiling code inserted by 'gcc -pg'.
5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
8 * Originally ported from the -rt patch by:
13 * Copyright (C) 2004-2006 Ingo Molnar
29 #include <linux/ftrace.h>
94 /* ftrace_enabled is a method to turn ftrace on or off */
107 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) in ftrace_pids_enabled()
110 tr = ops->private; in ftrace_pids_enabled()
112 return tr->function_pids != NULL || tr->function_no_pids != NULL; in ftrace_pids_enabled()
163 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { in ftrace_ops_init()
164 mutex_init(&ops->local_hash.regex_lock); in ftrace_ops_init()
165 INIT_LIST_HEAD(&ops->subop_list); in ftrace_ops_init()
166 ops->func_hash = &ops->local_hash; in ftrace_ops_init()
167 ops->flags |= FTRACE_OPS_FL_INITIALIZED; in ftrace_ops_init()
176 struct trace_array *tr = op->private; in ftrace_pid_func()
180 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid); in ftrace_pid_func()
184 pid != current->pid) in ftrace_pid_func()
188 op->saved_func(ip, parent_ip, op, fregs); in ftrace_pid_func()
203 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) || in ftrace_ops_get_list_func()
231 } else if (rcu_dereference_protected(ftrace_ops_list->next, in update_ftrace_function()
290 rcu_assign_pointer(ops->next, *list); in add_ftrace_ops()
295 * the ops->next pointer is valid before another CPU sees in add_ftrace_ops()
312 rcu_dereference_protected(ops->next, in remove_ftrace_ops()
318 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) in remove_ftrace_ops()
323 return -1; in remove_ftrace_ops()
325 *p = (*p)->next; in remove_ftrace_ops()
333 if (ops->flags & FTRACE_OPS_FL_DELETED) in __register_ftrace_function()
334 return -EINVAL; in __register_ftrace_function()
336 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) in __register_ftrace_function()
337 return -EBUSY; in __register_ftrace_function()
345 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && in __register_ftrace_function()
346 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) in __register_ftrace_function()
347 return -EINVAL; in __register_ftrace_function()
349 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) in __register_ftrace_function()
350 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; in __register_ftrace_function()
352 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT)) in __register_ftrace_function()
353 return -EBUSY; in __register_ftrace_function()
356 ops->flags |= FTRACE_OPS_FL_DYNAMIC; in __register_ftrace_function()
361 ops->saved_func = ops->func; in __register_ftrace_function()
364 ops->func = ftrace_pid_func; in __register_ftrace_function()
378 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) in __unregister_ftrace_function()
379 return -EBUSY; in __unregister_ftrace_function()
389 ops->func = ops->saved_func; in __unregister_ftrace_function()
403 if (op->flags & FTRACE_OPS_FL_PID) { in ftrace_update_pid_func()
404 op->func = ftrace_pids_enabled(op) ? in ftrace_update_pid_func()
405 ftrace_pid_func : op->saved_func; in ftrace_update_pid_func()
441 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
448 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
468 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
469 pg = pg->next; in function_stat_next()
472 rec = &pg->records[0]; in function_stat_next()
473 if (!rec->counter) in function_stat_next()
485 if (!stat || !stat->start) in function_stat_start()
488 return function_stat_next(&stat->start->records[0], 0); in function_stat_start()
498 if (a->time < b->time) in function_stat_cmp()
499 return -1; in function_stat_cmp()
500 if (a->time > b->time) in function_stat_cmp()
512 if (a->counter < b->counter) in function_stat_cmp()
513 return -1; in function_stat_cmp()
514 if (a->counter > b->counter) in function_stat_cmp()
526 " -------- " in function_stat_headers()
527 "--- ---- --- ---\n"); in function_stat_headers()
530 " -------- ---\n"); in function_stat_headers()
548 if (unlikely(rec->counter == 0)) { in function_stat_show()
549 ret = -EBUSY; in function_stat_show()
554 avg = div64_ul(rec->time, rec->counter); in function_stat_show()
559 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); in function_stat_show()
560 seq_printf(m, " %-30.30s %10lu", str, rec->counter); in function_stat_show()
566 if (rec->counter <= 1) in function_stat_show()
571 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) in function_stat_show()
573 stddev = rec->counter * rec->time_squared - in function_stat_show()
574 rec->time * rec->time; in function_stat_show()
577 * Divide only 1000 for ns^2 -> us^2 conversion. in function_stat_show()
581 rec->counter * (rec->counter - 1) * 1000); in function_stat_show()
585 trace_print_graph_duration(rec->time, &s); in function_stat_show()
603 pg = stat->pages = stat->start; in ftrace_profile_reset()
606 memset(pg->records, 0, PROFILE_RECORDS_SIZE); in ftrace_profile_reset()
607 pg->index = 0; in ftrace_profile_reset()
608 pg = pg->next; in ftrace_profile_reset()
611 memset(stat->hash, 0, in ftrace_profile_reset()
623 if (stat->pages) in ftrace_profile_pages_init()
626 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
627 if (!stat->pages) in ftrace_profile_pages_init()
628 return -ENOMEM; in ftrace_profile_pages_init()
643 pg = stat->start = stat->pages; in ftrace_profile_pages_init()
648 pg->next = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
649 if (!pg->next) in ftrace_profile_pages_init()
651 pg = pg->next; in ftrace_profile_pages_init()
657 pg = stat->start; in ftrace_profile_pages_init()
661 pg = pg->next; in ftrace_profile_pages_init()
665 stat->pages = NULL; in ftrace_profile_pages_init()
666 stat->start = NULL; in ftrace_profile_pages_init()
668 return -ENOMEM; in ftrace_profile_pages_init()
674 int size; in ftrace_profile_init_cpu() local
678 if (stat->hash) { in ftrace_profile_init_cpu()
688 size = FTRACE_PROFILE_HASH_SIZE; in ftrace_profile_init_cpu()
690 stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL); in ftrace_profile_init_cpu()
692 if (!stat->hash) in ftrace_profile_init_cpu()
693 return -ENOMEM; in ftrace_profile_init_cpu()
697 kfree(stat->hash); in ftrace_profile_init_cpu()
698 stat->hash = NULL; in ftrace_profile_init_cpu()
699 return -ENOMEM; in ftrace_profile_init_cpu()
728 hhd = &stat->hash[key]; in ftrace_find_profiled_func()
734 if (rec->ip == ip) in ftrace_find_profiled_func()
746 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); in ftrace_add_profile()
747 hlist_add_head_rcu(&rec->node, &stat->hash[key]); in ftrace_add_profile()
759 if (atomic_inc_return(&stat->disabled) != 1) in ftrace_profile_alloc()
770 if (stat->pages->index == PROFILES_PER_PAGE) { in ftrace_profile_alloc()
771 if (!stat->pages->next) in ftrace_profile_alloc()
773 stat->pages = stat->pages->next; in ftrace_profile_alloc()
776 rec = &stat->pages->records[stat->pages->index++]; in ftrace_profile_alloc()
777 rec->ip = ip; in ftrace_profile_alloc()
781 atomic_dec(&stat->disabled); in ftrace_profile_alloc()
800 if (!stat->hash || !ftrace_profile_enabled) in function_profile_call()
810 rec->counter++; in function_profile_call()
834 function_profile_call(trace->func, 0, NULL, NULL); in profile_graph_entry()
837 if (!current->ret_stack) in profile_graph_entry()
840 profile_data = fgraph_reserve_data(gops->idx, sizeof(*profile_data)); in profile_graph_entry()
844 profile_data->subtime = 0; in profile_graph_entry()
845 profile_data->sleeptime = current->ftrace_sleeptime; in profile_graph_entry()
846 profile_data->calltime = trace_clock_local(); in profile_graph_entry()
860 int size; in profile_graph_return() local
864 if (!stat->hash || !ftrace_profile_enabled) in profile_graph_return()
867 profile_data = fgraph_retrieve_data(gops->idx, &size); in profile_graph_return()
870 if (!profile_data || !profile_data->calltime) in profile_graph_return()
873 calltime = rettime - profile_data->calltime; in profile_graph_return()
876 if (current->ftrace_sleeptime) in profile_graph_return()
877 calltime -= current->ftrace_sleeptime - profile_data->sleeptime; in profile_graph_return()
884 parent_data = fgraph_retrieve_parent_data(gops->idx, &size, 1); in profile_graph_return()
886 parent_data->subtime += calltime; in profile_graph_return()
888 if (profile_data->subtime && profile_data->subtime < calltime) in profile_graph_return()
889 calltime -= profile_data->subtime; in profile_graph_return()
894 rec = ftrace_find_profiled_func(stat, trace->func); in profile_graph_return()
896 rec->time += calltime; in profile_graph_return()
897 rec->time_squared += calltime * calltime; in profile_graph_return()
1034 stat->stat = function_stats; in ftrace_profile_tracefs()
1035 stat->stat.name = name; in ftrace_profile_tracefs()
1036 ret = register_stat_tracer(&stat->stat); in ftrace_profile_tracefs()
1068 # error Dynamic ftrace depends on MCOUNT_RECORD
1102 * Used by the stack unwinder to know about dynamic ftrace trampolines.
1120 if (op->trampoline && op->trampoline_size) in ftrace_ops_trampoline()
1121 if (addr >= op->trampoline && in ftrace_ops_trampoline()
1122 addr < op->trampoline + op->trampoline_size) { in ftrace_ops_trampoline()
1159 if (hash->size_bits > 0) in ftrace_hash_key()
1160 return hash_long(ip, hash->size_bits); in ftrace_hash_key()
1174 hhd = &hash->buckets[key]; in __ftrace_lookup_ip()
1177 if (entry->ip == ip) in __ftrace_lookup_ip()
1184 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash
1208 key = ftrace_hash_key(hash, entry->ip); in __add_hash_entry()
1209 hhd = &hash->buckets[key]; in __add_hash_entry()
1210 hlist_add_head(&entry->hlist, hhd); in __add_hash_entry()
1211 hash->count++; in __add_hash_entry()
1223 entry->ip = ip; in add_hash_entry()
1233 hlist_del(&entry->hlist); in free_hash_entry()
1235 hash->count--; in free_hash_entry()
1242 hlist_del_rcu(&entry->hlist); in remove_hash_entry()
1243 hash->count--; in remove_hash_entry()
1251 int size = 1 << hash->size_bits; in ftrace_hash_clear() local
1254 if (!hash->count) in ftrace_hash_clear()
1257 for (i = 0; i < size; i++) { in ftrace_hash_clear()
1258 hhd = &hash->buckets[i]; in ftrace_hash_clear()
1262 FTRACE_WARN_ON(hash->count); in ftrace_hash_clear()
1267 list_del(&ftrace_mod->list); in free_ftrace_mod()
1268 kfree(ftrace_mod->module); in free_ftrace_mod()
1269 kfree(ftrace_mod->func); in free_ftrace_mod()
1292 kfree(hash->buckets); in free_ftrace_hash()
1308 call_rcu(&hash->rcu, __free_ftrace_hash_rcu); in free_ftrace_hash_rcu()
1312 * ftrace_free_filter - remove all filters for an ftrace_ops
1318 free_ftrace_hash(ops->func_hash->filter_hash); in ftrace_free_filter()
1319 free_ftrace_hash(ops->func_hash->notrace_hash); in ftrace_free_filter()
1326 int size; in alloc_ftrace_hash() local
1332 size = 1 << size_bits; in alloc_ftrace_hash()
1333 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); in alloc_ftrace_hash()
1335 if (!hash->buckets) { in alloc_ftrace_hash()
1340 hash->size_bits = size_bits; in alloc_ftrace_hash()
1351 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace; in ftrace_add_mod()
1355 return -ENOMEM; in ftrace_add_mod()
1357 INIT_LIST_HEAD(&ftrace_mod->list); in ftrace_add_mod()
1358 ftrace_mod->func = kstrdup(func, GFP_KERNEL); in ftrace_add_mod()
1359 ftrace_mod->module = kstrdup(module, GFP_KERNEL); in ftrace_add_mod()
1360 ftrace_mod->enable = enable; in ftrace_add_mod()
1362 if (!ftrace_mod->func || !ftrace_mod->module) in ftrace_add_mod()
1365 list_add(&ftrace_mod->list, mod_head); in ftrace_add_mod()
1372 return -ENOMEM; in ftrace_add_mod()
1380 int size; in alloc_and_copy_ftrace_hash() local
1388 new_hash->flags = hash->flags; in alloc_and_copy_ftrace_hash()
1394 size = 1 << hash->size_bits; in alloc_and_copy_ftrace_hash()
1395 for (i = 0; i < size; i++) { in alloc_and_copy_ftrace_hash()
1396 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in alloc_and_copy_ftrace_hash()
1397 if (add_hash_entry(new_hash, entry->ip) == NULL) in alloc_and_copy_ftrace_hash()
1402 FTRACE_WARN_ON(new_hash->count != hash->count); in alloc_and_copy_ftrace_hash()
1421 static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) in __move_hash() argument
1431 * Use around half the size (max bit of it), but in __move_hash()
1432 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits). in __move_hash()
1434 bits = fls(size / 2); in __move_hash()
1444 new_hash->flags = src->flags; in __move_hash()
1446 size = 1 << src->size_bits; in __move_hash()
1447 for (i = 0; i < size; i++) { in __move_hash()
1448 hhd = &src->buckets[i]; in __move_hash()
1461 int size = src->count; in __ftrace_hash_move() local
1469 return __move_hash(src, size); in __ftrace_hash_move()
1473 * ftrace_hash_move - move a new hash to a filter and do updates
1500 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) in ftrace_hash_move()
1501 return -EINVAL; in ftrace_hash_move()
1505 return -ENOMEM; in ftrace_hash_move()
1539 return (ftrace_hash_empty(hash->filter_hash) || in hash_contains_ip()
1540 __ftrace_lookup_ip(hash->filter_hash, ip)) && in hash_contains_ip()
1541 (ftrace_hash_empty(hash->notrace_hash) || in hash_contains_ip()
1542 !__ftrace_lookup_ip(hash->notrace_hash, ip)); in hash_contains_ip()
1547 * the ops->func or not.
1549 * It's a match if the ip is in the ops->filter_hash or
1552 * the ip is not in the ops->notrace_hash.
1565 * There's a small race when adding ops that the ftrace handler in ftrace_ops_test()
1569 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) in ftrace_ops_test()
1573 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); in ftrace_ops_test()
1574 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); in ftrace_ops_test()
1589 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1591 for (_____i = 0; _____i < pg->index; _____i++) { \
1592 rec = &pg->records[_____i];
1604 if (key->flags < rec->ip) in ftrace_cmp_recs()
1605 return -1; in ftrace_cmp_recs()
1606 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) in ftrace_cmp_recs()
1620 for (pg = ftrace_pages_start; pg; pg = pg->next) { in lookup_rec()
1621 if (pg->index == 0 || in lookup_rec()
1622 end < pg->records[0].ip || in lookup_rec()
1623 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in lookup_rec()
1625 rec = bsearch(&key, pg->records, pg->index, in lookup_rec()
1635 * ftrace_location_range - return the first address of a traced location
1641 * Returns: rec->ip if the related ftrace location is a least partly within
1643 * that is either a NOP or call to the function tracer. It checks the ftrace
1654 ip = rec->ip; in ftrace_location_range()
1661 * ftrace_location - return the ftrace location
1665 * * If @ip matches the ftrace location, return @ip.
1666 * * If @ip matches sym+0, return sym's ftrace location.
1673 unsigned long size; in ftrace_location() local
1677 if (!kallsyms_lookup_size_offset(ip, &size, &offset)) in ftrace_location()
1682 loc = ftrace_location_range(ip, ip + size - 1); in ftrace_location()
1690 * ftrace_text_reserved - return true if range contains an ftrace location
1694 * Returns: 1 if @start and @end contains a ftrace location.
1696 * the function tracer. It checks the ftrace internal tables to
1716 ops != &ftrace_list_end; ops = ops->next) { in test_rec_ops_needs_regs()
1717 /* pass rec in as regs to have non-NULL val */ in test_rec_ops_needs_regs()
1718 if (ftrace_ops_test(ops, rec->ip, rec)) { in test_rec_ops_needs_regs()
1719 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in test_rec_ops_needs_regs()
1744 return rec->flags & FTRACE_FL_DISABLED && in skip_record()
1745 !(rec->flags & FTRACE_FL_ENABLED); in skip_record()
1749 * This is the main engine to the ftrace updates to the dyn_ftrace records.
1751 * It will iterate through all the available ftrace functions
1752 * (the ones that ftrace can have callbacks to) and set the flags
1770 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_rec_update()
1777 hash = ops->func_hash->filter_hash; in __ftrace_hash_rec_update()
1778 notrace_hash = ops->func_hash->notrace_hash; in __ftrace_hash_rec_update()
1795 if (!notrace_hash || !ftrace_lookup_ip(notrace_hash, rec->ip)) in __ftrace_hash_rec_update()
1798 in_hash = !!ftrace_lookup_ip(hash, rec->ip); in __ftrace_hash_rec_update()
1799 in_notrace_hash = !!ftrace_lookup_ip(notrace_hash, rec->ip); in __ftrace_hash_rec_update()
1812 rec->flags++; in __ftrace_hash_rec_update()
1816 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1817 rec->flags |= FTRACE_FL_DIRECT; in __ftrace_hash_rec_update()
1824 if (ftrace_rec_count(rec) == 1 && ops->trampoline) in __ftrace_hash_rec_update()
1825 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1833 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1839 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in __ftrace_hash_rec_update()
1840 rec->flags |= FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1844 rec->flags--; in __ftrace_hash_rec_update()
1852 if (ops->flags & FTRACE_OPS_FL_DIRECT) in __ftrace_hash_rec_update()
1853 rec->flags &= ~FTRACE_FL_DIRECT; in __ftrace_hash_rec_update()
1862 rec->flags & FTRACE_FL_REGS && in __ftrace_hash_rec_update()
1863 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { in __ftrace_hash_rec_update()
1865 rec->flags &= ~FTRACE_FL_REGS; in __ftrace_hash_rec_update()
1877 rec->flags |= FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1879 rec->flags &= ~FTRACE_FL_TRAMP; in __ftrace_hash_rec_update()
1888 * If the rec has a single associated ops, and ops->func can be in __ftrace_hash_rec_update()
1893 ftrace_ops_get_func(ops) == ops->func) in __ftrace_hash_rec_update()
1894 rec->flags |= FTRACE_FL_CALL_OPS; in __ftrace_hash_rec_update()
1896 rec->flags &= ~FTRACE_FL_CALL_OPS; in __ftrace_hash_rec_update()
1904 if (!all && count == hash->count) in __ftrace_hash_rec_update()
1940 * ops->hash = new_hash
1954 if (ops->func_hash != &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1965 if (op->func_hash == &global_ops.local_hash) in ftrace_hash_rec_update_modify()
1982 * or no-needed to update, -EBUSY if it detects a conflict of the flag
1983 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
1985 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
1986 * - If the hash is EMPTY_HASH, it hits nothing
1987 * - Anything else hits the recs which match the hash entries.
1992 * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate
2006 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in __ftrace_hash_update_ipmodify()
2009 is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2010 is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; in __ftrace_hash_update_ipmodify()
2025 return -EINVAL; in __ftrace_hash_update_ipmodify()
2027 /* Update rec->flags */ in __ftrace_hash_update_ipmodify()
2030 if (rec->flags & FTRACE_FL_DISABLED) in __ftrace_hash_update_ipmodify()
2034 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2035 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2040 if (rec->flags & FTRACE_FL_IPMODIFY) { in __ftrace_hash_update_ipmodify()
2047 FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT); in __ftrace_hash_update_ipmodify()
2055 if (!ops->ops_func) in __ftrace_hash_update_ipmodify()
2056 return -EBUSY; in __ftrace_hash_update_ipmodify()
2057 ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); in __ftrace_hash_update_ipmodify()
2061 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2064 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2076 if (rec->flags & FTRACE_FL_DISABLED) in __ftrace_hash_update_ipmodify()
2082 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2083 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); in __ftrace_hash_update_ipmodify()
2088 rec->flags &= ~FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2090 rec->flags |= FTRACE_FL_IPMODIFY; in __ftrace_hash_update_ipmodify()
2094 return -EBUSY; in __ftrace_hash_update_ipmodify()
2099 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_enable()
2110 struct ftrace_hash *hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_disable()
2121 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; in ftrace_hash_ipmodify_update()
2154 pr_info("Initializing ftrace call sites\n"); in print_bug_type()
2157 pr_info("Setting ftrace call site to NOP\n"); in print_bug_type()
2160 pr_info("Setting ftrace call site to call ftrace function\n"); in print_bug_type()
2163 pr_info("Updating ftrace call site to call a different ftrace function\n"); in print_bug_type()
2169 * ftrace_bug - report and shutdown function tracer
2176 * EFAULT - if the problem happens on reading the @ip address
2177 * EINVAL - if what is read at @ip is not what was expected
2178 * EPERM - if the problem happens on writing to the @ip address
2182 unsigned long ip = rec ? rec->ip : 0; in ftrace_bug()
2184 pr_info("------------[ ftrace bug ]------------\n"); in ftrace_bug()
2187 case -EFAULT: in ftrace_bug()
2188 pr_info("ftrace faulted on modifying "); in ftrace_bug()
2191 case -EINVAL: in ftrace_bug()
2192 pr_info("ftrace failed to modify "); in ftrace_bug()
2201 case -EPERM: in ftrace_bug()
2202 pr_info("ftrace faulted on writing "); in ftrace_bug()
2206 pr_info("ftrace faulted on unknown error "); in ftrace_bug()
2213 pr_info("ftrace record flags: %lx\n", rec->flags); in ftrace_bug()
2215 rec->flags & FTRACE_FL_REGS ? " R" : " ", in ftrace_bug()
2216 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " "); in ftrace_bug()
2217 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_bug()
2222 (void *)ops->trampoline, in ftrace_bug()
2223 (void *)ops->func); in ftrace_bug()
2267 if (!(rec->flags & FTRACE_FL_REGS) != in ftrace_check_record()
2268 !(rec->flags & FTRACE_FL_REGS_EN)) in ftrace_check_record()
2271 if (!(rec->flags & FTRACE_FL_TRAMP) != in ftrace_check_record()
2272 !(rec->flags & FTRACE_FL_TRAMP_EN)) in ftrace_check_record()
2287 if (!(rec->flags & FTRACE_FL_DIRECT) != in ftrace_check_record()
2288 !(rec->flags & FTRACE_FL_DIRECT_EN)) in ftrace_check_record()
2290 } else if (rec->flags & FTRACE_FL_DIRECT_EN) { in ftrace_check_record()
2300 if (!(rec->flags & FTRACE_FL_CALL_OPS) != in ftrace_check_record()
2301 !(rec->flags & FTRACE_FL_CALL_OPS_EN)) in ftrace_check_record()
2303 } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) { in ftrace_check_record()
2309 if ((rec->flags & FTRACE_FL_ENABLED) == flag) in ftrace_check_record()
2314 flag ^= rec->flags & FTRACE_FL_ENABLED; in ftrace_check_record()
2317 rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED; in ftrace_check_record()
2319 if (rec->flags & FTRACE_FL_REGS) in ftrace_check_record()
2320 rec->flags |= FTRACE_FL_REGS_EN; in ftrace_check_record()
2322 rec->flags &= ~FTRACE_FL_REGS_EN; in ftrace_check_record()
2325 if (rec->flags & FTRACE_FL_TRAMP) in ftrace_check_record()
2326 rec->flags |= FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2328 rec->flags &= ~FTRACE_FL_TRAMP_EN; in ftrace_check_record()
2332 if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY)) in ftrace_check_record()
2333 rec->flags |= FTRACE_FL_MODIFIED; in ftrace_check_record()
2339 * directly (no ftrace trampoline). in ftrace_check_record()
2342 if (rec->flags & FTRACE_FL_DIRECT) in ftrace_check_record()
2343 rec->flags |= FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2345 rec->flags &= ~FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2351 rec->flags &= ~FTRACE_FL_DIRECT_EN; in ftrace_check_record()
2357 if (rec->flags & FTRACE_FL_CALL_OPS) in ftrace_check_record()
2358 rec->flags |= FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2360 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2366 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; in ftrace_check_record()
2376 * from the save regs, to a non-save regs function or in ftrace_check_record()
2391 rec->flags &= FTRACE_NOCLEAR_FLAGS; in ftrace_check_record()
2397 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | in ftrace_check_record()
2407 * ftrace_update_record - set a record that now is tracing or not
2420 * ftrace_test_record - check if the record has been enabled or not
2437 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any()
2441 if (!op->trampoline) in ftrace_find_tramp_ops_any()
2444 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_any()
2455 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_any_other()
2459 if (op == op_exclude || !op->trampoline) in ftrace_find_tramp_ops_any_other()
2462 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_any_other()
2473 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_next()
2477 if (!op->trampoline) in ftrace_find_tramp_ops_next()
2480 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_next()
2491 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_curr()
2500 if (hash_contains_ip(ip, &removed_ops->old_hash)) in ftrace_find_tramp_ops_curr()
2524 if (!op->trampoline) in ftrace_find_tramp_ops_curr()
2531 if (op->flags & FTRACE_OPS_FL_ADDING) in ftrace_find_tramp_ops_curr()
2540 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && in ftrace_find_tramp_ops_curr()
2541 hash_contains_ip(ip, &op->old_hash)) in ftrace_find_tramp_ops_curr()
2548 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && in ftrace_find_tramp_ops_curr()
2549 hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_curr()
2561 unsigned long ip = rec->ip; in ftrace_find_tramp_ops_new()
2564 /* pass rec in as regs to have non-NULL val */ in ftrace_find_tramp_ops_new()
2565 if (hash_contains_ip(ip, op->func_hash)) in ftrace_find_tramp_ops_new()
2576 unsigned long ip = rec->ip; in ftrace_find_unique_ops()
2580 if (hash_contains_ip(ip, op->func_hash)) { in ftrace_find_unique_ops()
2608 return entry->direct; in ftrace_find_rec_direct()
2614 unsigned long addr = READ_ONCE(ops->direct_call); in call_direct_funcs()
2624 * ftrace_get_addr_new - Get the call address to set to
2625 * @rec: The ftrace record descriptor
2638 if ((rec->flags & FTRACE_FL_DIRECT) && in ftrace_get_addr_new()
2640 addr = ftrace_find_rec_direct(rec->ip); in ftrace_get_addr_new()
2647 if (rec->flags & FTRACE_FL_TRAMP) { in ftrace_get_addr_new()
2649 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { in ftrace_get_addr_new()
2651 (void *)rec->ip, (void *)rec->ip, rec->flags); in ftrace_get_addr_new()
2652 /* Ftrace is shutting down, return anything */ in ftrace_get_addr_new()
2655 return ops->trampoline; in ftrace_get_addr_new()
2658 if (rec->flags & FTRACE_FL_REGS) in ftrace_get_addr_new()
2665 * ftrace_get_addr_curr - Get the call address that is already there
2666 * @rec: The ftrace record descriptor
2680 if (rec->flags & FTRACE_FL_DIRECT_EN) { in ftrace_get_addr_curr()
2681 addr = ftrace_find_rec_direct(rec->ip); in ftrace_get_addr_curr()
2688 if (rec->flags & FTRACE_FL_TRAMP_EN) { in ftrace_get_addr_curr()
2692 (void *)rec->ip, (void *)rec->ip); in ftrace_get_addr_curr()
2693 /* Ftrace is shutting down, return anything */ in ftrace_get_addr_curr()
2696 return ops->trampoline; in ftrace_get_addr_curr()
2699 if (rec->flags & FTRACE_FL_REGS_EN) in ftrace_get_addr_curr()
2738 return -1; /* unknown ftrace bug */ in __ftrace_replace_code()
2774 * ftrace_rec_iter_start - start up iterating over traced functions
2791 iter->pg = ftrace_pages_start; in ftrace_rec_iter_start()
2792 iter->index = 0; in ftrace_rec_iter_start()
2795 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_start()
2796 iter->pg = iter->pg->next; in ftrace_rec_iter_start()
2798 if (!iter->pg) in ftrace_rec_iter_start()
2805 * ftrace_rec_iter_next - get the next record to process.
2812 iter->index++; in ftrace_rec_iter_next()
2814 if (iter->index >= iter->pg->index) { in ftrace_rec_iter_next()
2815 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2816 iter->index = 0; in ftrace_rec_iter_next()
2819 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_next()
2820 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2823 if (!iter->pg) in ftrace_rec_iter_next()
2830 * ftrace_rec_iter_record - get the record at the iterator location
2837 return &iter->pg->records[iter->index]; in ftrace_rec_iter_record()
2944 * ftrace_run_stop_machine - go back to the stop machine method
2945 * @command: The command to tell ftrace what to do
2956 * arch_ftrace_update_code - modify the code to trace or not trace
2985 ops->flags |= FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
2986 ops->old_hash.filter_hash = old_hash->filter_hash; in ftrace_run_modify_code()
2987 ops->old_hash.notrace_hash = old_hash->notrace_hash; in ftrace_run_modify_code()
2989 ops->old_hash.filter_hash = NULL; in ftrace_run_modify_code()
2990 ops->old_hash.notrace_hash = NULL; in ftrace_run_modify_code()
2991 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; in ftrace_run_modify_code()
3007 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list); in ftrace_add_trampoline_to_kallsyms()
3013 list_del_rcu(&ops->list); in ftrace_remove_trampoline_from_kallsyms()
3019 * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is
3027 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) && in ftrace_trampoline_free()
3028 ops->trampoline) { in ftrace_trampoline_free()
3033 perf_event_text_poke((void *)ops->trampoline, in ftrace_trampoline_free()
3034 (void *)ops->trampoline, in ftrace_trampoline_free()
3035 ops->trampoline_size, NULL, 0); in ftrace_trampoline_free()
3037 ops->trampoline, ops->trampoline_size, in ftrace_trampoline_free()
3071 return -ENODEV; in ftrace_startup()
3080 * Note that ftrace probes uses this to start up in ftrace_startup()
3087 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; in ftrace_startup()
3093 ftrace_start_up--; in ftrace_startup()
3094 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_startup()
3095 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) in ftrace_startup()
3106 * If ftrace is in an undefined state, we just remove ops from list in ftrace_startup()
3112 return -ENODEV; in ftrace_startup()
3115 ops->flags &= ~FTRACE_OPS_FL_ADDING; in ftrace_startup()
3125 return -ENODEV; in ftrace_shutdown()
3131 ftrace_start_up--; in ftrace_shutdown()
3133 * Just warn in case of unbalance, no need to kill ftrace, it's not in ftrace_shutdown()
3135 * further ftrace uses. in ftrace_shutdown()
3145 ops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_shutdown()
3159 ops->flags |= FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3163 ops->old_hash.filter_hash = ops->func_hash->filter_hash; in ftrace_shutdown()
3164 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; in ftrace_shutdown()
3169 * If there's no more ops registered with ftrace, run a in ftrace_shutdown()
3178 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS)) in ftrace_shutdown()
3180 (void *)rec->ip, rec->flags); in ftrace_shutdown()
3184 ops->old_hash.filter_hash = NULL; in ftrace_shutdown()
3185 ops->old_hash.notrace_hash = NULL; in ftrace_shutdown()
3188 ops->flags &= ~FTRACE_OPS_FL_REMOVING; in ftrace_shutdown()
3195 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) { in ftrace_shutdown()
3208 * while on a ftrace trampoline. Just scheduling a task on in ftrace_shutdown()
3227 return alloc_and_copy_ftrace_hash(src->size_bits, src); in copy_hash()
3248 int size; in append_hash() local
3262 size = 1 << new_hash->size_bits; in append_hash()
3263 for (i = 0; i < size; i++) { in append_hash()
3264 hlist_for_each_entry(entry, &new_hash->buckets[i], hlist) { in append_hash()
3266 if (!__ftrace_lookup_ip(*hash, entry->ip) && in append_hash()
3267 add_hash_entry(*hash, entry->ip) == NULL) in append_hash()
3268 return -ENOMEM; in append_hash()
3284 int size; in intersect_hash() local
3297 size = 1 << new_hash1->size_bits; in intersect_hash()
3298 for (i = 0; i < size; i++) { in intersect_hash()
3299 hlist_for_each_entry(entry, &new_hash1->buckets[i], hlist) { in intersect_hash()
3301 if (__ftrace_lookup_ip(new_hash2, entry->ip) && in intersect_hash()
3302 add_hash_entry(*hash, entry->ip) == NULL) in intersect_hash()
3303 return -ENOMEM; in intersect_hash()
3314 /* Return a new hash that has a union of all @ops->filter_hash entries */
3321 new_hash = alloc_ftrace_hash(ops->func_hash->filter_hash->size_bits); in append_hashes()
3325 list_for_each_entry(subops, &ops->subop_list, list) { in append_hashes()
3326 ret = append_hash(&new_hash, subops->func_hash->filter_hash); in append_hashes()
3346 list_for_each_entry(subops, &ops->subop_list, list) { in intersect_hashes()
3350 size_bits = subops->func_hash->notrace_hash->size_bits; in intersect_hashes()
3351 new_hash = alloc_and_copy_ftrace_hash(size_bits, ops->func_hash->notrace_hash); in intersect_hashes()
3356 size_bits = new_hash->size_bits; in intersect_hashes()
3359 ret = intersect_hash(&new_hash, next_hash, subops->func_hash->notrace_hash); in intersect_hashes()
3375 int size; in ops_equal() local
3384 if (A->count != B->count) in ops_equal()
3387 size = 1 << A->size_bits; in ops_equal()
3388 for (i = 0; i < size; i++) { in ops_equal()
3389 hlist_for_each_entry(entry, &A->buckets[i], hlist) { in ops_equal()
3390 if (!__ftrace_lookup_ip(B, entry->ip)) in ops_equal()
3411 old_hash_ops.filter_hash = ops->func_hash->filter_hash; in __ftrace_hash_move_and_update_ops()
3412 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; in __ftrace_hash_move_and_update_ops()
3426 if (!ops_equal(filter_hash, ops->func_hash->filter_hash)) { in ftrace_update_ops()
3427 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash, in ftrace_update_ops()
3433 if (!ops_equal(notrace_hash, ops->func_hash->notrace_hash)) { in ftrace_update_ops()
3434 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash, in ftrace_update_ops()
3444 * ftrace_startup_subops - enable tracing for subops of an ops
3463 return -ENODEV; in ftrace_startup_subops()
3468 if (WARN_ON_ONCE(subops->flags & FTRACE_OPS_FL_ENABLED)) in ftrace_startup_subops()
3469 return -EBUSY; in ftrace_startup_subops()
3472 if (!ops->func_hash->filter_hash) in ftrace_startup_subops()
3473 ops->func_hash->filter_hash = EMPTY_HASH; in ftrace_startup_subops()
3474 if (!ops->func_hash->notrace_hash) in ftrace_startup_subops()
3475 ops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_startup_subops()
3476 if (!subops->func_hash->filter_hash) in ftrace_startup_subops()
3477 subops->func_hash->filter_hash = EMPTY_HASH; in ftrace_startup_subops()
3478 if (!subops->func_hash->notrace_hash) in ftrace_startup_subops()
3479 subops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_startup_subops()
3482 if (list_empty(&ops->subop_list)) { in ftrace_startup_subops()
3484 filter_hash = copy_hash(subops->func_hash->filter_hash); in ftrace_startup_subops()
3485 notrace_hash = copy_hash(subops->func_hash->notrace_hash); in ftrace_startup_subops()
3489 return -ENOMEM; in ftrace_startup_subops()
3492 save_filter_hash = ops->func_hash->filter_hash; in ftrace_startup_subops()
3493 save_notrace_hash = ops->func_hash->notrace_hash; in ftrace_startup_subops()
3495 ops->func_hash->filter_hash = filter_hash; in ftrace_startup_subops()
3496 ops->func_hash->notrace_hash = notrace_hash; in ftrace_startup_subops()
3497 list_add(&subops->list, &ops->subop_list); in ftrace_startup_subops()
3500 list_del(&subops->list); in ftrace_startup_subops()
3501 ops->func_hash->filter_hash = save_filter_hash; in ftrace_startup_subops()
3502 ops->func_hash->notrace_hash = save_notrace_hash; in ftrace_startup_subops()
3508 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; in ftrace_startup_subops()
3509 subops->managed = ops; in ftrace_startup_subops()
3521 if (ftrace_hash_empty(ops->func_hash->filter_hash) || in ftrace_startup_subops()
3522 ftrace_hash_empty(subops->func_hash->filter_hash)) { in ftrace_startup_subops()
3525 size_bits = max(ops->func_hash->filter_hash->size_bits, in ftrace_startup_subops()
3526 subops->func_hash->filter_hash->size_bits); in ftrace_startup_subops()
3527 filter_hash = alloc_and_copy_ftrace_hash(size_bits, ops->func_hash->filter_hash); in ftrace_startup_subops()
3529 return -ENOMEM; in ftrace_startup_subops()
3530 ret = append_hash(&filter_hash, subops->func_hash->filter_hash); in ftrace_startup_subops()
3537 if (ftrace_hash_empty(ops->func_hash->notrace_hash) || in ftrace_startup_subops()
3538 ftrace_hash_empty(subops->func_hash->notrace_hash)) { in ftrace_startup_subops()
3541 size_bits = max(ops->func_hash->filter_hash->size_bits, in ftrace_startup_subops()
3542 subops->func_hash->filter_hash->size_bits); in ftrace_startup_subops()
3546 return -ENOMEM; in ftrace_startup_subops()
3549 ret = intersect_hash(¬race_hash, ops->func_hash->filter_hash, in ftrace_startup_subops()
3550 subops->func_hash->filter_hash); in ftrace_startup_subops()
3558 list_add(&subops->list, &ops->subop_list); in ftrace_startup_subops()
3564 list_del(&subops->list); in ftrace_startup_subops()
3566 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; in ftrace_startup_subops()
3567 subops->managed = ops; in ftrace_startup_subops()
3573 * ftrace_shutdown_subops - Remove a subops from a manager ops
3591 return -ENODEV; in ftrace_shutdown_subops()
3593 if (WARN_ON_ONCE(!(subops->flags & FTRACE_OPS_FL_ENABLED))) in ftrace_shutdown_subops()
3594 return -EINVAL; in ftrace_shutdown_subops()
3596 list_del(&subops->list); in ftrace_shutdown_subops()
3598 if (list_empty(&ops->subop_list)) { in ftrace_shutdown_subops()
3603 list_add(&subops->list, &ops->subop_list); in ftrace_shutdown_subops()
3607 subops->flags &= ~FTRACE_OPS_FL_ENABLED; in ftrace_shutdown_subops()
3609 free_ftrace_hash(ops->func_hash->filter_hash); in ftrace_shutdown_subops()
3610 free_ftrace_hash(ops->func_hash->notrace_hash); in ftrace_shutdown_subops()
3611 ops->func_hash->filter_hash = EMPTY_HASH; in ftrace_shutdown_subops()
3612 ops->func_hash->notrace_hash = EMPTY_HASH; in ftrace_shutdown_subops()
3613 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); in ftrace_shutdown_subops()
3614 subops->managed = NULL; in ftrace_shutdown_subops()
3625 list_add(&subops->list, &ops->subop_list); in ftrace_shutdown_subops()
3626 return -ENOMEM; in ftrace_shutdown_subops()
3631 list_add(&subops->list, &ops->subop_list); in ftrace_shutdown_subops()
3633 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); in ftrace_shutdown_subops()
3634 subops->managed = NULL; in ftrace_shutdown_subops()
3646 struct ftrace_ops *ops = subops->managed; in ftrace_hash_move_and_update_subops()
3653 if (WARN_ON_ONCE(!ops || ops->flags & FTRACE_OPS_FL_SUBOP)) in ftrace_hash_move_and_update_subops()
3654 return -EINVAL; in ftrace_hash_move_and_update_subops()
3661 return -ENOMEM; in ftrace_hash_move_and_update_subops()
3666 orig_hash = &ops->func_hash->filter_hash; in ftrace_hash_move_and_update_subops()
3669 orig_hash = &ops->func_hash->notrace_hash; in ftrace_hash_move_and_update_subops()
3701 return ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_traces_mod()
3702 ftrace_hash_empty(ops->func_hash->notrace_hash); in ops_traces_mod()
3720 * an entry in the ftrace data. Now, if ftrace is activated in ftrace_update_code()
3722 * read-only, the modification of enabling ftrace can fail if in ftrace_update_code()
3723 * the read-only is done while ftrace is converting the calls. in ftrace_update_code()
3726 * to read-only. in ftrace_update_code()
3731 for (pg = new_pgs; pg; pg = pg->next) { in ftrace_update_code()
3733 for (i = 0; i < pg->index; i++) { in ftrace_update_code()
3737 return -1; in ftrace_update_code()
3739 p = &pg->records[i]; in ftrace_update_code()
3740 p->flags = rec_flags; in ftrace_update_code()
3754 update_time = stop - start; in ftrace_update_code()
3771 return -EINVAL; in ftrace_allocate_records()
3775 order = fls(pages) - 1; in ftrace_allocate_records()
3778 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); in ftrace_allocate_records()
3780 if (!pg->records) { in ftrace_allocate_records()
3781 /* if we can't allocate this size, try something smaller */ in ftrace_allocate_records()
3783 return -ENOMEM; in ftrace_allocate_records()
3784 order--; in ftrace_allocate_records()
3792 pg->order = order; in ftrace_allocate_records()
3805 if (pg->records) { in ftrace_free_pages()
3806 free_pages((unsigned long)pg->records, pg->order); in ftrace_free_pages()
3807 ftrace_number_of_pages -= 1 << pg->order; in ftrace_free_pages()
3809 pages = pg->next; in ftrace_free_pages()
3812 ftrace_number_of_groups--; in ftrace_free_pages()
3840 num_to_init -= cnt; in ftrace_allocate_pages()
3844 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3845 if (!pg->next) in ftrace_allocate_pages()
3848 pg = pg->next; in ftrace_allocate_pages()
3855 pr_info("ftrace: FAILED to allocate memory for functions\n"); in ftrace_allocate_pages()
3882 struct ftrace_iterator *iter = m->private; in t_probe_next()
3883 struct trace_array *tr = iter->ops->private; in t_probe_next()
3889 int size; in t_probe_next() local
3892 iter->pos = *pos; in t_probe_next()
3897 func_probes = &tr->func_probes; in t_probe_next()
3901 if (!iter->probe) { in t_probe_next()
3902 next = func_probes->next; in t_probe_next()
3903 iter->probe = list_entry(next, struct ftrace_func_probe, list); in t_probe_next()
3906 if (iter->probe_entry) in t_probe_next()
3907 hnd = &iter->probe_entry->hlist; in t_probe_next()
3909 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3918 size = 1 << hash->size_bits; in t_probe_next()
3921 if (iter->pidx >= size) { in t_probe_next()
3922 if (iter->probe->list.next == func_probes) in t_probe_next()
3924 next = iter->probe->list.next; in t_probe_next()
3925 iter->probe = list_entry(next, struct ftrace_func_probe, list); in t_probe_next()
3926 hash = iter->probe->ops.func_hash->filter_hash; in t_probe_next()
3927 size = 1 << hash->size_bits; in t_probe_next()
3928 iter->pidx = 0; in t_probe_next()
3931 hhd = &hash->buckets[iter->pidx]; in t_probe_next()
3934 iter->pidx++; in t_probe_next()
3940 hnd = hhd->first; in t_probe_next()
3942 hnd = hnd->next; in t_probe_next()
3944 iter->pidx++; in t_probe_next()
3952 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist); in t_probe_next()
3959 struct ftrace_iterator *iter = m->private; in t_probe_start()
3963 if (!(iter->flags & FTRACE_ITER_DO_PROBES)) in t_probe_start()
3966 if (iter->mod_pos > *pos) in t_probe_start()
3969 iter->probe = NULL; in t_probe_start()
3970 iter->probe_entry = NULL; in t_probe_start()
3971 iter->pidx = 0; in t_probe_start()
3972 for (l = 0; l <= (*pos - iter->mod_pos); ) { in t_probe_start()
3981 iter->flags |= FTRACE_ITER_PROBE; in t_probe_start()
3993 probe = iter->probe; in t_probe_show()
3994 probe_entry = iter->probe_entry; in t_probe_show()
3997 return -EIO; in t_probe_show()
3999 probe_ops = probe->probe_ops; in t_probe_show()
4001 if (probe_ops->print) in t_probe_show()
4002 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); in t_probe_show()
4004 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, in t_probe_show()
4005 (void *)probe_ops->func); in t_probe_show()
4013 struct ftrace_iterator *iter = m->private; in t_mod_next()
4014 struct trace_array *tr = iter->tr; in t_mod_next()
4017 iter->pos = *pos; in t_mod_next()
4019 iter->mod_list = iter->mod_list->next; in t_mod_next()
4021 if (iter->mod_list == &tr->mod_trace || in t_mod_next()
4022 iter->mod_list == &tr->mod_notrace) { in t_mod_next()
4023 iter->flags &= ~FTRACE_ITER_MOD; in t_mod_next()
4027 iter->mod_pos = *pos; in t_mod_next()
4034 struct ftrace_iterator *iter = m->private; in t_mod_start()
4038 if (iter->func_pos > *pos) in t_mod_start()
4041 iter->mod_pos = iter->func_pos; in t_mod_start()
4044 if (!iter->tr) in t_mod_start()
4047 for (l = 0; l <= (*pos - iter->func_pos); ) { in t_mod_start()
4053 iter->flags &= ~FTRACE_ITER_MOD; in t_mod_start()
4058 iter->flags |= FTRACE_ITER_MOD; in t_mod_start()
4067 struct trace_array *tr = iter->tr; in t_mod_show()
4069 if (WARN_ON_ONCE(!iter->mod_list) || in t_mod_show()
4070 iter->mod_list == &tr->mod_trace || in t_mod_show()
4071 iter->mod_list == &tr->mod_notrace) in t_mod_show()
4072 return -EIO; in t_mod_show()
4074 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list); in t_mod_show()
4076 if (ftrace_mod->func) in t_mod_show()
4077 seq_printf(m, "%s", ftrace_mod->func); in t_mod_show()
4081 seq_printf(m, ":mod:%s\n", ftrace_mod->module); in t_mod_show()
4089 struct ftrace_iterator *iter = m->private; in t_func_next()
4095 if (iter->idx >= iter->pg->index) { in t_func_next()
4096 if (iter->pg->next) { in t_func_next()
4097 iter->pg = iter->pg->next; in t_func_next()
4098 iter->idx = 0; in t_func_next()
4102 rec = &iter->pg->records[iter->idx++]; in t_func_next()
4103 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && in t_func_next()
4104 !ftrace_lookup_ip(iter->hash, rec->ip)) || in t_func_next()
4106 ((iter->flags & FTRACE_ITER_ENABLED) && in t_func_next()
4107 !(rec->flags & FTRACE_FL_ENABLED)) || in t_func_next()
4109 ((iter->flags & FTRACE_ITER_TOUCHED) && in t_func_next()
4110 !(rec->flags & FTRACE_FL_TOUCHED))) { in t_func_next()
4120 iter->pos = iter->func_pos = *pos; in t_func_next()
4121 iter->func = rec; in t_func_next()
4129 struct ftrace_iterator *iter = m->private; in t_next()
4136 if (iter->flags & FTRACE_ITER_PROBE) in t_next()
4139 if (iter->flags & FTRACE_ITER_MOD) in t_next()
4142 if (iter->flags & FTRACE_ITER_PRINTALL) { in t_next()
4158 iter->pos = 0; in reset_iter_read()
4159 iter->func_pos = 0; in reset_iter_read()
4160 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD); in reset_iter_read()
4165 struct ftrace_iterator *iter = m->private; in t_start()
4177 if (*pos < iter->pos) in t_start()
4185 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && in t_start()
4186 ftrace_hash_empty(iter->hash)) { in t_start()
4187 iter->func_pos = 1; /* Account for the message */ in t_start()
4190 iter->flags |= FTRACE_ITER_PRINTALL; in t_start()
4192 iter->flags &= ~FTRACE_ITER_PROBE; in t_start()
4196 if (iter->flags & FTRACE_ITER_MOD) in t_start()
4204 iter->pg = ftrace_pages_start; in t_start()
4205 iter->idx = 0; in t_start()
4236 seq_printf(m, " ->%pS", ptr); in add_trampoline_func()
4256 ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str); in test_for_valid_rec()
4260 rec->flags |= FTRACE_FL_DISABLED; in test_for_valid_rec()
4323 return ret == NULL ? -1 : 0; in print_rec()
4340 struct ftrace_iterator *iter = m->private; in t_show()
4343 if (iter->flags & FTRACE_ITER_PROBE) in t_show()
4346 if (iter->flags & FTRACE_ITER_MOD) in t_show()
4349 if (iter->flags & FTRACE_ITER_PRINTALL) { in t_show()
4350 if (iter->flags & FTRACE_ITER_NOTRACE) in t_show()
4357 rec = iter->func; in t_show()
4362 if (iter->flags & FTRACE_ITER_ADDRS) in t_show()
4363 seq_printf(m, "%lx ", rec->ip); in t_show()
4365 if (print_rec(m, rec->ip)) { in t_show()
4367 WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED)); in t_show()
4372 if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) { in t_show()
4377 rec->flags & FTRACE_FL_REGS ? " R" : " ", in t_show()
4378 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ", in t_show()
4379 rec->flags & FTRACE_FL_DIRECT ? " D" : " ", in t_show()
4380 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ", in t_show()
4381 rec->flags & FTRACE_FL_MODIFIED ? " M " : " "); in t_show()
4382 if (rec->flags & FTRACE_FL_TRAMP_EN) { in t_show()
4387 (void *)ops->trampoline, in t_show()
4388 (void *)ops->func); in t_show()
4397 if (rec->flags & FTRACE_FL_CALL_OPS_EN) { in t_show()
4401 ops, ops->func); in t_show()
4406 if (rec->flags & FTRACE_FL_DIRECT) { in t_show()
4409 direct = ftrace_find_rec_direct(rec->ip); in t_show()
4411 seq_printf(m, "\n\tdirect-->%pS", (void *)direct); in t_show()
4438 return -ENODEV; in ftrace_avail_open()
4442 return -ENOMEM; in ftrace_avail_open()
4444 iter->pg = ftrace_pages_start; in ftrace_avail_open()
4445 iter->ops = &global_ops; in ftrace_avail_open()
4466 return -ENOMEM; in ftrace_enabled_open()
4468 iter->pg = ftrace_pages_start; in ftrace_enabled_open()
4469 iter->flags = FTRACE_ITER_ENABLED; in ftrace_enabled_open()
4470 iter->ops = &global_ops; in ftrace_enabled_open()
4491 return -ENOMEM; in ftrace_touched_open()
4493 iter->pg = ftrace_pages_start; in ftrace_touched_open()
4494 iter->flags = FTRACE_ITER_TOUCHED; in ftrace_touched_open()
4495 iter->ops = &global_ops; in ftrace_touched_open()
4511 return -ENODEV; in ftrace_avail_addrs_open()
4515 return -ENOMEM; in ftrace_avail_addrs_open()
4517 iter->pg = ftrace_pages_start; in ftrace_avail_addrs_open()
4518 iter->flags = FTRACE_ITER_ADDRS; in ftrace_avail_addrs_open()
4519 iter->ops = &global_ops; in ftrace_avail_addrs_open()
4525 * ftrace_regex_open - initialize function tracer filter files
4549 struct trace_array *tr = ops->private; in ftrace_regex_open()
4550 int ret = -ENOMEM; in ftrace_regex_open()
4555 return -ENODEV; in ftrace_regex_open()
4558 return -ENODEV; in ftrace_regex_open()
4564 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) in ftrace_regex_open()
4567 iter->ops = ops; in ftrace_regex_open()
4568 iter->flags = flag; in ftrace_regex_open()
4569 iter->tr = tr; in ftrace_regex_open()
4571 mutex_lock(&ops->func_hash->regex_lock); in ftrace_regex_open()
4574 hash = ops->func_hash->notrace_hash; in ftrace_regex_open()
4575 mod_head = tr ? &tr->mod_notrace : NULL; in ftrace_regex_open()
4577 hash = ops->func_hash->filter_hash; in ftrace_regex_open()
4578 mod_head = tr ? &tr->mod_trace : NULL; in ftrace_regex_open()
4581 iter->mod_list = mod_head; in ftrace_regex_open()
4583 if (file->f_mode & FMODE_WRITE) { in ftrace_regex_open()
4586 if (file->f_flags & O_TRUNC) { in ftrace_regex_open()
4587 iter->hash = alloc_ftrace_hash(size_bits); in ftrace_regex_open()
4590 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); in ftrace_regex_open()
4593 if (!iter->hash) { in ftrace_regex_open()
4594 trace_parser_put(&iter->parser); in ftrace_regex_open()
4598 iter->hash = hash; in ftrace_regex_open()
4602 if (file->f_mode & FMODE_READ) { in ftrace_regex_open()
4603 iter->pg = ftrace_pages_start; in ftrace_regex_open()
4607 struct seq_file *m = file->private_data; in ftrace_regex_open()
4608 m->private = iter; in ftrace_regex_open()
4611 free_ftrace_hash(iter->hash); in ftrace_regex_open()
4612 trace_parser_put(&iter->parser); in ftrace_regex_open()
4615 file->private_data = iter; in ftrace_regex_open()
4618 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_regex_open()
4633 struct ftrace_ops *ops = inode->i_private; in ftrace_filter_open()
4644 struct ftrace_ops *ops = inode->i_private; in ftrace_notrace_open()
4651 /* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */
4659 * If symbols in an architecture don't correspond exactly to the user-visible
4673 str = arch_ftrace_match_adjust(str, g->search); in ftrace_match()
4675 switch (g->type) { in ftrace_match()
4677 if (strcmp(str, g->search) == 0) in ftrace_match()
4681 if (strncmp(str, g->search, g->len) == 0) in ftrace_match()
4685 if (strstr(str, g->search)) in ftrace_match()
4690 if (slen >= g->len && in ftrace_match()
4691 memcmp(str + slen - g->len, g->search, g->len) == 0) in ftrace_match()
4695 if (glob_match(g->search, str)) in ftrace_match()
4709 entry = ftrace_lookup_ip(hash, rec->ip); in enter_record()
4720 if (add_hash_entry(hash, rec->ip) == NULL) in enter_record()
4721 ret = -ENOMEM; in enter_record()
4735 if (kstrtoul(func_g->search, 0, &index) || --index < 0) in add_rec_by_index()
4739 if (pg->index <= index) { in add_rec_by_index()
4740 index -= pg->index; in add_rec_by_index()
4744 rec = &pg->records[index]; in add_rec_by_index()
4758 return -1; in lookup_ip()
4776 if (lookup_ip(rec->ip, &modname, str)) { in ftrace_match_record()
4779 !(rec->flags & FTRACE_FL_DISABLED)); in ftrace_match_record()
4787 if (!mod_g->len) { in ftrace_match_record()
4805 if (!func_g->len) in ftrace_match_record()
4847 if (rec->flags & FTRACE_FL_DISABLED) in match_records()
4876 if (ops->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4886 if (ops->func_hash != &global_ops.local_hash) in ftrace_ops_update_code()
4890 if (op->func_hash == &global_ops.local_hash && in ftrace_ops_update_code()
4891 op->flags & FTRACE_OPS_FL_ENABLED) { in ftrace_ops_update_code()
4904 if (ops->flags & FTRACE_OPS_FL_SUBOP) in ftrace_hash_move_and_update_ops()
4914 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) { in ftrace_hash_move_and_update_ops()
4921 list_for_each_entry(subops, &op->subop_list, list) { in ftrace_hash_move_and_update_ops()
4922 if ((subops->flags & FTRACE_OPS_FL_ENABLED) && in ftrace_hash_move_and_update_ops()
4923 subops->func_hash == ops->func_hash) { in ftrace_hash_move_and_update_ops()
4943 if (n > sizeof(modname) - 1) in module_exists()
4954 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace; in cache_mod()
4960 int ret = -EINVAL; in cache_mod()
4966 if (strcmp(ftrace_mod->module, module) != 0) in cache_mod()
4971 (ftrace_mod->func && in cache_mod()
4972 strcmp(ftrace_mod->func, func) == 0)) { in cache_mod()
4983 return -EINVAL; in cache_mod()
5002 mutex_lock(&ops->func_hash->regex_lock); in process_mod_list()
5005 orig_hash = &ops->func_hash->filter_hash; in process_mod_list()
5007 orig_hash = &ops->func_hash->notrace_hash; in process_mod_list()
5018 if (strcmp(ftrace_mod->module, mod) != 0) in process_mod_list()
5021 if (ftrace_mod->func) in process_mod_list()
5022 func = kstrdup(ftrace_mod->func, GFP_KERNEL); in process_mod_list()
5029 list_move(&ftrace_mod->list, &process_mods); in process_mod_list()
5032 kfree(ftrace_mod->func); in process_mod_list()
5033 ftrace_mod->func = func; in process_mod_list()
5040 func = ftrace_mod->func; in process_mod_list()
5048 new_hash->flags &= ~FTRACE_HASH_FL_MOD; in process_mod_list()
5057 mutex_unlock(&ops->func_hash->regex_lock); in process_mod_list()
5073 if (!list_empty(&tr->mod_trace)) in process_cached_mods()
5074 process_mod_list(&tr->mod_trace, tr->ops, mod, true); in process_cached_mods()
5075 if (!list_empty(&tr->mod_notrace)) in process_cached_mods()
5076 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); in process_cached_mods()
5097 return -ENODEV; in ftrace_mod_callback()
5102 return -ENOMEM; in ftrace_mod_callback()
5139 probe_ops = probe->probe_ops; in function_trace_probe_call()
5147 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); in function_trace_probe_call()
5161 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper
5179 * ftrace_func_mapper_find_ip - Find some data mapped to an ip
5195 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_find_ip()
5200 return &map->data; in ftrace_func_mapper_find_ip()
5204 * ftrace_func_mapper_add_ip - Map some data to an ip
5217 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_add_ip()
5219 return -EBUSY; in ftrace_func_mapper_add_ip()
5223 return -ENOMEM; in ftrace_func_mapper_add_ip()
5225 map->entry.ip = ip; in ftrace_func_mapper_add_ip()
5226 map->data = data; in ftrace_func_mapper_add_ip()
5228 __add_hash_entry(&mapper->hash, &map->entry); in ftrace_func_mapper_add_ip()
5234 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping
5250 entry = ftrace_lookup_ip(&mapper->hash, ip); in ftrace_func_mapper_remove_ip()
5255 data = map->data; in ftrace_func_mapper_remove_ip()
5257 remove_hash_entry(&mapper->hash, entry); in ftrace_func_mapper_remove_ip()
5264 * free_ftrace_func_mapper - free a mapping of ips and data
5277 int size, i; in free_ftrace_func_mapper() local
5282 if (free_func && mapper->hash.count) { in free_ftrace_func_mapper()
5283 size = 1 << mapper->hash.size_bits; in free_ftrace_func_mapper()
5284 for (i = 0; i < size; i++) { in free_ftrace_func_mapper()
5285 hhd = &mapper->hash.buckets[i]; in free_ftrace_func_mapper()
5292 free_ftrace_hash(&mapper->hash); in free_ftrace_func_mapper()
5301 WARN_ON(probe->ref <= 0); in release_probe()
5304 probe->ref--; in release_probe()
5306 if (!probe->ref) { in release_probe()
5307 probe_ops = probe->probe_ops; in release_probe()
5310 * the probe->data itself in release_probe()
5312 if (probe_ops->free) in release_probe()
5313 probe_ops->free(probe_ops, probe->tr, 0, probe->data); in release_probe()
5314 list_del(&probe->list); in release_probe()
5325 probe->ref++; in acquire_probe_locked()
5339 int size; in register_ftrace_function_probe() local
5344 return -EINVAL; in register_ftrace_function_probe()
5348 return -EINVAL; in register_ftrace_function_probe()
5353 list_for_each_entry(iter, &tr->func_probes, list) { in register_ftrace_function_probe()
5354 if (iter->probe_ops == probe_ops) { in register_ftrace_function_probe()
5363 return -ENOMEM; in register_ftrace_function_probe()
5365 probe->probe_ops = probe_ops; in register_ftrace_function_probe()
5366 probe->ops.func = function_trace_probe_call; in register_ftrace_function_probe()
5367 probe->tr = tr; in register_ftrace_function_probe()
5368 ftrace_ops_init(&probe->ops); in register_ftrace_function_probe()
5369 list_add(&probe->list, &tr->func_probes); in register_ftrace_function_probe()
5377 * Note, there's a small window here that the func_hash->filter_hash in register_ftrace_function_probe()
5380 mutex_lock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
5382 orig_hash = &probe->ops.func_hash->filter_hash; in register_ftrace_function_probe()
5387 ret = -ENOMEM; in register_ftrace_function_probe()
5395 ret = -EINVAL; in register_ftrace_function_probe()
5400 size = 1 << hash->size_bits; in register_ftrace_function_probe()
5401 for (i = 0; i < size; i++) { in register_ftrace_function_probe()
5402 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_function_probe()
5403 if (ftrace_lookup_ip(old_hash, entry->ip)) in register_ftrace_function_probe()
5410 if (probe_ops->init) { in register_ftrace_function_probe()
5411 ret = probe_ops->init(probe_ops, tr, in register_ftrace_function_probe()
5412 entry->ip, data, in register_ftrace_function_probe()
5413 &probe->data); in register_ftrace_function_probe()
5415 if (probe_ops->free && count) in register_ftrace_function_probe()
5416 probe_ops->free(probe_ops, tr, in register_ftrace_function_probe()
5417 0, probe->data); in register_ftrace_function_probe()
5418 probe->data = NULL; in register_ftrace_function_probe()
5430 ret = -EINVAL; in register_ftrace_function_probe()
5434 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in register_ftrace_function_probe()
5440 probe->ref += count; in register_ftrace_function_probe()
5442 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) in register_ftrace_function_probe()
5443 ret = ftrace_startup(&probe->ops, 0); in register_ftrace_function_probe()
5451 mutex_unlock(&probe->ops.func_hash->regex_lock); in register_ftrace_function_probe()
5459 if (!probe_ops->free || !count) in register_ftrace_function_probe()
5463 for (i = 0; i < size; i++) { in register_ftrace_function_probe()
5464 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_function_probe()
5465 if (ftrace_lookup_ip(old_hash, entry->ip)) in register_ftrace_function_probe()
5467 probe_ops->free(probe_ops, tr, entry->ip, probe->data); in register_ftrace_function_probe()
5488 int i, ret = -ENODEV; in unregister_ftrace_function_probe_func()
5489 int size; in unregister_ftrace_function_probe_func() local
5502 return -EINVAL; in unregister_ftrace_function_probe_func()
5507 list_for_each_entry(iter, &tr->func_probes, list) { in unregister_ftrace_function_probe_func()
5508 if (iter->probe_ops == probe_ops) { in unregister_ftrace_function_probe_func()
5516 ret = -EINVAL; in unregister_ftrace_function_probe_func()
5517 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) in unregister_ftrace_function_probe_func()
5524 mutex_lock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
5526 orig_hash = &probe->ops.func_hash->filter_hash; in unregister_ftrace_function_probe_func()
5536 ret = -ENOMEM; in unregister_ftrace_function_probe_func()
5543 size = 1 << hash->size_bits; in unregister_ftrace_function_probe_func()
5544 for (i = 0; i < size; i++) { in unregister_ftrace_function_probe_func()
5545 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { in unregister_ftrace_function_probe_func()
5548 kallsyms_lookup(entry->ip, NULL, NULL, in unregister_ftrace_function_probe_func()
5555 hlist_add_head(&entry->hlist, &hhd); in unregister_ftrace_function_probe_func()
5561 ret = -EINVAL; in unregister_ftrace_function_probe_func()
5567 WARN_ON(probe->ref < count); in unregister_ftrace_function_probe_func()
5569 probe->ref -= count; in unregister_ftrace_function_probe_func()
5572 ftrace_shutdown(&probe->ops, 0); in unregister_ftrace_function_probe_func()
5574 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, in unregister_ftrace_function_probe_func()
5579 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, in unregister_ftrace_function_probe_func()
5584 hlist_del(&entry->hlist); in unregister_ftrace_function_probe_func()
5585 if (probe_ops->free) in unregister_ftrace_function_probe_func()
5586 probe_ops->free(probe_ops, tr, entry->ip, probe->data); in unregister_ftrace_function_probe_func()
5592 mutex_unlock(&probe->ops.func_hash->regex_lock); in unregister_ftrace_function_probe_func()
5608 list_for_each_entry_safe(probe, n, &tr->func_probes, list) in clear_ftrace_function_probes()
5609 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops); in clear_ftrace_function_probes()
5616 * Currently we only register ftrace commands from __init, so mark this
5626 if (strcmp(cmd->name, p->name) == 0) { in register_ftrace_command()
5627 ret = -EBUSY; in register_ftrace_command()
5631 list_add(&cmd->list, &ftrace_commands); in register_ftrace_command()
5639 * Currently we only unregister ftrace commands from __init, so mark
5645 int ret = -ENODEV; in unregister_ftrace_command()
5649 if (strcmp(cmd->name, p->name) == 0) { in unregister_ftrace_command()
5651 list_del_init(&p->list); in unregister_ftrace_command()
5664 struct ftrace_hash *hash = iter->hash; in ftrace_process_regex()
5665 struct trace_array *tr = iter->ops->private; in ftrace_process_regex()
5668 int ret = -EINVAL; in ftrace_process_regex()
5675 ret = -EINVAL; in ftrace_process_regex()
5687 if (strcmp(p->name, command) == 0) { in ftrace_process_regex()
5688 ret = p->func(tr, hash, func, command, next, enable); in ftrace_process_regex()
5709 if (file->f_mode & FMODE_READ) { in ftrace_regex_write()
5710 struct seq_file *m = file->private_data; in ftrace_regex_write()
5711 iter = m->private; in ftrace_regex_write()
5713 iter = file->private_data; in ftrace_regex_write()
5716 return -ENODEV; in ftrace_regex_write()
5718 /* iter->hash is a local copy, so we don't need regex_lock */ in ftrace_regex_write()
5720 parser = &iter->parser; in ftrace_regex_write()
5725 ret = ftrace_process_regex(iter, parser->buffer, in ftrace_regex_write()
5726 parser->idx, enable); in ftrace_regex_write()
5758 return -EINVAL; in __ftrace_match_addr()
5763 return -ENOENT; in __ftrace_match_addr()
5769 return entry ? 0 : -ENOMEM; in __ftrace_match_addr()
5802 return -ENODEV; in ftrace_set_hash()
5804 mutex_lock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5807 orig_hash = &ops->func_hash->filter_hash; in ftrace_set_hash()
5809 orig_hash = &ops->func_hash->notrace_hash; in ftrace_set_hash()
5817 ret = -ENOMEM; in ftrace_set_hash()
5822 ret = -EINVAL; in ftrace_set_hash()
5836 mutex_unlock(&ops->func_hash->regex_lock); in ftrace_set_hash()
5867 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in check_direct_multi()
5868 return -EINVAL; in check_direct_multi()
5869 if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS) in check_direct_multi()
5870 return -EINVAL; in check_direct_multi()
5877 int size, i; in remove_direct_functions_hash() local
5879 size = 1 << hash->size_bits; in remove_direct_functions_hash()
5880 for (i = 0; i < size; i++) { in remove_direct_functions_hash()
5881 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in remove_direct_functions_hash()
5882 del = __ftrace_lookup_ip(direct_functions, entry->ip); in remove_direct_functions_hash()
5883 if (del && del->direct == addr) { in remove_direct_functions_hash()
5899 * register_ftrace_direct - Call a custom trampoline directly
5914 * -EINVAL - The @ops object was already registered with this call or
5916 * -EBUSY - Another direct function is already attached (there can be only one)
5917 * -ENODEV - @ip does not point to a ftrace nop location (or not supported)
5918 * -ENOMEM - There was an allocation failure.
5924 int err = -EBUSY, size, i; in register_ftrace_direct() local
5926 if (ops->func || ops->trampoline) in register_ftrace_direct()
5927 return -EINVAL; in register_ftrace_direct()
5928 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) in register_ftrace_direct()
5929 return -EINVAL; in register_ftrace_direct()
5930 if (ops->flags & FTRACE_OPS_FL_ENABLED) in register_ftrace_direct()
5931 return -EINVAL; in register_ftrace_direct()
5933 hash = ops->func_hash->filter_hash; in register_ftrace_direct()
5935 return -EINVAL; in register_ftrace_direct()
5940 size = 1 << hash->size_bits; in register_ftrace_direct()
5941 for (i = 0; i < size; i++) { in register_ftrace_direct()
5942 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_direct()
5943 if (ftrace_find_rec_direct(entry->ip)) in register_ftrace_direct()
5948 err = -ENOMEM; in register_ftrace_direct()
5951 size = hash->count + direct_functions->count; in register_ftrace_direct()
5952 if (size > 32) in register_ftrace_direct()
5953 size = 32; in register_ftrace_direct()
5954 new_hash = alloc_ftrace_hash(fls(size)); in register_ftrace_direct()
5959 size = 1 << direct_functions->size_bits; in register_ftrace_direct()
5960 for (i = 0; i < size; i++) { in register_ftrace_direct()
5961 hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) { in register_ftrace_direct()
5962 new = add_hash_entry(new_hash, entry->ip); in register_ftrace_direct()
5965 new->direct = entry->direct; in register_ftrace_direct()
5970 size = 1 << hash->size_bits; in register_ftrace_direct()
5971 for (i = 0; i < size; i++) { in register_ftrace_direct()
5972 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in register_ftrace_direct()
5973 new = add_hash_entry(new_hash, entry->ip); in register_ftrace_direct()
5977 new->direct = addr; in register_ftrace_direct()
5978 entry->direct = addr; in register_ftrace_direct()
5986 ops->func = call_direct_funcs; in register_ftrace_direct()
5987 ops->flags = MULTI_FLAGS; in register_ftrace_direct()
5988 ops->trampoline = FTRACE_REGS_ADDR; in register_ftrace_direct()
5989 ops->direct_call = addr; in register_ftrace_direct()
5997 call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); in register_ftrace_direct()
6007 * unregister_ftrace_direct - Remove calls to custom trampoline
6019 * -EINVAL - The @ops object was not properly registered.
6024 struct ftrace_hash *hash = ops->func_hash->filter_hash; in unregister_ftrace_direct()
6028 return -EINVAL; in unregister_ftrace_direct()
6029 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in unregister_ftrace_direct()
6030 return -EINVAL; in unregister_ftrace_direct()
6038 ops->func = NULL; in unregister_ftrace_direct()
6039 ops->trampoline = 0; in unregister_ftrace_direct()
6056 int i, size; in __modify_ftrace_direct() local
6063 tmp_ops.func_hash = ops->func_hash; in __modify_ftrace_direct()
6076 hash = ops->func_hash->filter_hash; in __modify_ftrace_direct()
6077 size = 1 << hash->size_bits; in __modify_ftrace_direct()
6078 for (i = 0; i < size; i++) { in __modify_ftrace_direct()
6079 hlist_for_each_entry(iter, &hash->buckets[i], hlist) { in __modify_ftrace_direct()
6080 entry = __ftrace_lookup_ip(direct_functions, iter->ip); in __modify_ftrace_direct()
6083 entry->direct = addr; in __modify_ftrace_direct()
6087 WRITE_ONCE(ops->direct_call, addr); in __modify_ftrace_direct()
6098 * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
6113 * -EINVAL - The @ops object was not properly registered.
6118 return -EINVAL; in modify_ftrace_direct_nolock()
6119 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct_nolock()
6120 return -EINVAL; in modify_ftrace_direct_nolock()
6127 * modify_ftrace_direct - Modify an existing direct 'multi' call
6139 * -EINVAL - The @ops object was not properly registered.
6146 return -EINVAL; in modify_ftrace_direct()
6147 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in modify_ftrace_direct()
6148 return -EINVAL; in modify_ftrace_direct()
6159 * ftrace_set_filter_ip - set a function to filter on in ftrace by address
6181 * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses
6204 * ftrace_ops_set_global_filter - setup ops to use global filters
6207 * ftrace users who need global function trace filtering should call this.
6212 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) in ftrace_ops_set_global_filter()
6216 ops->func_hash = &global_ops.local_hash; in ftrace_ops_set_global_filter()
6228 * ftrace_set_filter - set a function to filter on in ftrace
6232 * @reset: non-zero to reset all filters before applying this filter.
6250 * ftrace_set_notrace - set a function to not trace in ftrace
6254 * @reset: non-zero to reset all filters before applying this filter.
6272 * ftrace_set_global_filter - set a function to filter on with global tracers
6275 * @reset: non-zero to reset all filters before applying this filter.
6287 * ftrace_set_global_notrace - set a function to not trace with global tracers
6290 * @reset: non-zero to reset all filters before applying this filter.
6370 printk(KERN_DEBUG "ftrace: function %s not " in set_ftrace_early_graph()
6410 struct seq_file *m = (struct seq_file *)file->private_data; in ftrace_regex_release()
6416 if (file->f_mode & FMODE_READ) { in ftrace_regex_release()
6417 iter = m->private; in ftrace_regex_release()
6420 iter = file->private_data; in ftrace_regex_release()
6422 parser = &iter->parser; in ftrace_regex_release()
6424 int enable = !(iter->flags & FTRACE_ITER_NOTRACE); in ftrace_regex_release()
6426 ftrace_process_regex(iter, parser->buffer, in ftrace_regex_release()
6427 parser->idx, enable); in ftrace_regex_release()
6432 mutex_lock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
6434 if (file->f_mode & FMODE_WRITE) { in ftrace_regex_release()
6435 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); in ftrace_regex_release()
6438 orig_hash = &iter->ops->func_hash->filter_hash; in ftrace_regex_release()
6439 if (iter->tr) { in ftrace_regex_release()
6440 if (list_empty(&iter->tr->mod_trace)) in ftrace_regex_release()
6441 iter->hash->flags &= ~FTRACE_HASH_FL_MOD; in ftrace_regex_release()
6443 iter->hash->flags |= FTRACE_HASH_FL_MOD; in ftrace_regex_release()
6446 orig_hash = &iter->ops->func_hash->notrace_hash; in ftrace_regex_release()
6449 ftrace_hash_move_and_update_ops(iter->ops, orig_hash, in ftrace_regex_release()
6450 iter->hash, filter_hash); in ftrace_regex_release()
6454 iter->hash = NULL; in ftrace_regex_release()
6457 mutex_unlock(&iter->ops->func_hash->regex_lock); in ftrace_regex_release()
6458 free_ftrace_hash(iter->hash); in ftrace_regex_release()
6459 if (iter->tr) in ftrace_regex_release()
6460 trace_array_put(iter->tr); in ftrace_regex_release()
6537 struct ftrace_graph_data *fgd = m->private; in __g_next()
6538 struct ftrace_func_entry *entry = fgd->entry; in __g_next()
6540 int i, idx = fgd->idx; in __g_next()
6542 if (*pos >= fgd->hash->count) in __g_next()
6547 fgd->entry = entry; in __g_next()
6554 for (i = idx; i < 1 << fgd->hash->size_bits; i++) { in __g_next()
6555 head = &fgd->hash->buckets[i]; in __g_next()
6557 fgd->entry = entry; in __g_next()
6558 fgd->idx = i; in __g_next()
6574 struct ftrace_graph_data *fgd = m->private; in g_start()
6578 if (fgd->type == GRAPH_FILTER_FUNCTION) in g_start()
6579 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, in g_start()
6582 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, in g_start()
6586 if (ftrace_hash_empty(fgd->hash) && !*pos) in g_start()
6589 fgd->idx = 0; in g_start()
6590 fgd->entry = NULL; in g_start()
6607 struct ftrace_graph_data *fgd = m->private; in g_show()
6609 if (fgd->type == GRAPH_FILTER_FUNCTION) in g_show()
6616 seq_printf(m, "%ps\n", (void *)entry->ip); in g_show()
6639 if (file->f_mode & FMODE_WRITE) { in __ftrace_graph_open()
6642 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX)) in __ftrace_graph_open()
6643 return -ENOMEM; in __ftrace_graph_open()
6645 if (file->f_flags & O_TRUNC) in __ftrace_graph_open()
6649 fgd->hash); in __ftrace_graph_open()
6651 ret = -ENOMEM; in __ftrace_graph_open()
6656 if (file->f_mode & FMODE_READ) { in __ftrace_graph_open()
6659 struct seq_file *m = file->private_data; in __ftrace_graph_open()
6660 m->private = fgd; in __ftrace_graph_open()
6667 file->private_data = fgd; in __ftrace_graph_open()
6670 if (ret < 0 && file->f_mode & FMODE_WRITE) in __ftrace_graph_open()
6671 trace_parser_put(&fgd->parser); in __ftrace_graph_open()
6673 fgd->new_hash = new_hash; in __ftrace_graph_open()
6676 * All uses of fgd->hash must be taken with the graph_lock in __ftrace_graph_open()
6678 * fgd->hash to be reinitialized when it is taken again. in __ftrace_graph_open()
6680 fgd->hash = NULL; in __ftrace_graph_open()
6692 return -ENODEV; in ftrace_graph_open()
6696 return -ENOMEM; in ftrace_graph_open()
6700 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, in ftrace_graph_open()
6702 fgd->type = GRAPH_FILTER_FUNCTION; in ftrace_graph_open()
6703 fgd->seq_ops = &ftrace_graph_seq_ops; in ftrace_graph_open()
6720 return -ENODEV; in ftrace_graph_notrace_open()
6724 return -ENOMEM; in ftrace_graph_notrace_open()
6728 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, in ftrace_graph_notrace_open()
6730 fgd->type = GRAPH_FILTER_NOTRACE; in ftrace_graph_notrace_open()
6731 fgd->seq_ops = &ftrace_graph_seq_ops; in ftrace_graph_notrace_open()
6749 if (file->f_mode & FMODE_READ) { in ftrace_graph_release()
6750 struct seq_file *m = file->private_data; in ftrace_graph_release()
6752 fgd = m->private; in ftrace_graph_release()
6755 fgd = file->private_data; in ftrace_graph_release()
6759 if (file->f_mode & FMODE_WRITE) { in ftrace_graph_release()
6761 parser = &fgd->parser; in ftrace_graph_release()
6764 ret = ftrace_graph_set_hash(fgd->new_hash, in ftrace_graph_release()
6765 parser->buffer); in ftrace_graph_release()
6770 new_hash = __ftrace_hash_move(fgd->new_hash); in ftrace_graph_release()
6772 ret = -ENOMEM; in ftrace_graph_release()
6778 if (fgd->type == GRAPH_FILTER_FUNCTION) { in ftrace_graph_release()
6805 free_ftrace_hash(fgd->new_hash); in ftrace_graph_release()
6830 return -ENODEV; in ftrace_graph_set_hash()
6834 if (rec->flags & FTRACE_FL_DISABLED) in ftrace_graph_set_hash()
6838 entry = ftrace_lookup_ip(hash, rec->ip); in ftrace_graph_set_hash()
6845 if (add_hash_entry(hash, rec->ip) == NULL) in ftrace_graph_set_hash()
6856 return fail ? -EINVAL : 0; in ftrace_graph_set_hash()
6864 struct ftrace_graph_data *fgd = file->private_data; in ftrace_graph_write()
6871 if (file->f_mode & FMODE_READ) { in ftrace_graph_write()
6872 struct seq_file *m = file->private_data; in ftrace_graph_write()
6873 fgd = m->private; in ftrace_graph_write()
6876 parser = &fgd->parser; in ftrace_graph_write()
6883 ret = ftrace_graph_set_hash(fgd->new_hash, in ftrace_graph_write()
6884 parser->buffer); in ftrace_graph_write()
6935 if (ops->flags & FTRACE_OPS_FL_ENABLED) in ftrace_destroy_filter_files()
6937 ops->flags |= FTRACE_OPS_FL_DELETED; in ftrace_destroy_filter_files()
6979 return -1; in ftrace_cmp_ips()
6989 if (WARN(start[i - 1] > start[i], in test_is_sorted()
6991 (void *)start[i - 1], start[i - 1], in test_is_sorted()
6996 pr_info("ftrace section at %px sorted properly\n", start); in test_is_sorted()
7017 int ret = -ENOMEM; in ftrace_process_locs()
7019 count = end - start; in ftrace_process_locs()
7038 return -ENOMEM; in ftrace_process_locs()
7055 if (WARN_ON(ftrace_pages->next)) { in ftrace_process_locs()
7057 while (ftrace_pages->next) in ftrace_process_locs()
7058 ftrace_pages = ftrace_pages->next; in ftrace_process_locs()
7061 ftrace_pages->next = start_pg; in ftrace_process_locs()
7080 end_offset = (pg->index+1) * sizeof(pg->records[0]); in ftrace_process_locs()
7081 if (end_offset > PAGE_SIZE << pg->order) { in ftrace_process_locs()
7083 if (WARN_ON(!pg->next)) in ftrace_process_locs()
7085 pg = pg->next; in ftrace_process_locs()
7088 rec = &pg->records[pg->index++]; in ftrace_process_locs()
7089 rec->ip = addr; in ftrace_process_locs()
7092 if (pg->next) { in ftrace_process_locs()
7093 pg_unuse = pg->next; in ftrace_process_locs()
7094 pg->next = NULL; in ftrace_process_locs()
7131 unsigned int size; member
7152 if (!op->trampoline || symnum--) in ftrace_get_trampoline_kallsym()
7154 *value = op->trampoline; in ftrace_get_trampoline_kallsym()
7162 return -ERANGE; in ftrace_get_trampoline_kallsym()
7177 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) in ops_references_ip()
7185 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && in ops_references_ip()
7186 !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) in ops_references_ip()
7190 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) in ops_references_ip()
7208 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { in referenced_filters()
7209 if (ops_references_ip(ops, rec->ip)) { in referenced_filters()
7210 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) in referenced_filters()
7212 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in referenced_filters()
7215 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) in referenced_filters()
7216 rec->flags |= FTRACE_FL_REGS; in referenced_filters()
7217 if (cnt == 1 && ops->trampoline) in referenced_filters()
7218 rec->flags |= FTRACE_FL_TRAMP; in referenced_filters()
7220 rec->flags &= ~FTRACE_FL_TRAMP; in referenced_filters()
7237 for (i = 0; i < pg->index; i++) { in clear_mod_from_hash()
7238 rec = &pg->records[i]; in clear_mod_from_hash()
7239 entry = __ftrace_lookup_ip(hash, rec->ip); in clear_mod_from_hash()
7246 entry->ip = 0; in clear_mod_from_hash()
7257 if (!tr->ops || !tr->ops->func_hash) in clear_mod_from_hashes()
7259 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
7260 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); in clear_mod_from_hashes()
7261 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); in clear_mod_from_hashes()
7262 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_mod_from_hashes()
7274 list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) { in ftrace_free_mod_map()
7275 kfree(mod_func->name); in ftrace_free_mod_map()
7276 list_del(&mod_func->list); in ftrace_free_mod_map()
7298 if (mod_map->mod == mod) { in ftrace_release_mod()
7299 list_del_rcu(&mod_map->list); in ftrace_release_mod()
7300 call_rcu(&mod_map->rcu, ftrace_free_mod_map); in ftrace_release_mod()
7311 rec = &pg->records[0]; in ftrace_release_mod()
7312 if (within_module(rec->ip, mod)) { in ftrace_release_mod()
7324 ftrace_update_tot_cnt -= pg->index; in ftrace_release_mod()
7325 *last_pg = pg->next; in ftrace_release_mod()
7327 pg->next = tmp_page; in ftrace_release_mod()
7330 last_pg = &pg->next; in ftrace_release_mod()
7343 if (pg->records) { in ftrace_release_mod()
7344 free_pages((unsigned long)pg->records, pg->order); in ftrace_release_mod()
7345 ftrace_number_of_pages -= 1 << pg->order; in ftrace_release_mod()
7347 tmp_page = pg->next; in ftrace_release_mod()
7349 ftrace_number_of_groups--; in ftrace_release_mod()
7373 * text to read-only, as we now need to set it back to read-write in ftrace_module_enable()
7387 if (!within_module(rec->ip, mod)) in ftrace_module_enable()
7393 rec->flags = FTRACE_FL_DISABLED; in ftrace_module_enable()
7408 rec->flags &= ~FTRACE_FL_DISABLED; in ftrace_module_enable()
7409 rec->flags += cnt; in ftrace_module_enable()
7428 process_cached_mods(mod->name); in ftrace_module_enable()
7435 if (ftrace_disabled || !mod->num_ftrace_callsites) in ftrace_module_init()
7438 ret = ftrace_process_locs(mod, mod->ftrace_callsites, in ftrace_module_init()
7439 mod->ftrace_callsites + mod->num_ftrace_callsites); in ftrace_module_init()
7441 pr_warn("ftrace: failed to allocate entries for module '%s' functions\n", in ftrace_module_init()
7442 mod->name); in ftrace_module_init()
7455 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str); in save_ftrace_mod_rec()
7463 mod_func->name = kstrdup(str, GFP_KERNEL); in save_ftrace_mod_rec()
7464 if (!mod_func->name) { in save_ftrace_mod_rec()
7469 mod_func->ip = rec->ip - offset; in save_ftrace_mod_rec()
7470 mod_func->size = symsize; in save_ftrace_mod_rec()
7472 mod_map->num_funcs++; in save_ftrace_mod_rec()
7474 list_add_rcu(&mod_func->list, &mod_map->funcs); in save_ftrace_mod_rec()
7487 mod_map->mod = mod; in allocate_ftrace_mod_map()
7488 mod_map->start_addr = start; in allocate_ftrace_mod_map()
7489 mod_map->end_addr = end; in allocate_ftrace_mod_map()
7490 mod_map->num_funcs = 0; in allocate_ftrace_mod_map()
7492 INIT_LIST_HEAD_RCU(&mod_map->funcs); in allocate_ftrace_mod_map()
7494 list_add_rcu(&mod_map->list, &ftrace_mod_maps); in allocate_ftrace_mod_map()
7501 unsigned long addr, unsigned long *size, in ftrace_func_address_lookup() argument
7507 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { in ftrace_func_address_lookup()
7508 if (addr >= mod_func->ip && in ftrace_func_address_lookup()
7509 addr < mod_func->ip + mod_func->size) { in ftrace_func_address_lookup()
7516 if (size) in ftrace_func_address_lookup()
7517 *size = found_func->size; in ftrace_func_address_lookup()
7519 *off = addr - found_func->ip; in ftrace_func_address_lookup()
7520 return strscpy(sym, found_func->name, KSYM_NAME_LEN); in ftrace_func_address_lookup()
7527 ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, in ftrace_mod_address_lookup() argument
7536 ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym); in ftrace_mod_address_lookup()
7539 *modname = mod_map->mod->name; in ftrace_mod_address_lookup()
7559 if (symnum >= mod_map->num_funcs) { in ftrace_mod_get_kallsym()
7560 symnum -= mod_map->num_funcs; in ftrace_mod_get_kallsym()
7564 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { in ftrace_mod_get_kallsym()
7566 symnum--; in ftrace_mod_get_kallsym()
7570 *value = mod_func->ip; in ftrace_mod_get_kallsym()
7572 strscpy(name, mod_func->name, KSYM_NAME_LEN); in ftrace_mod_get_kallsym()
7573 strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN); in ftrace_mod_get_kallsym()
7621 entry = ftrace_lookup_ip(hash, func->ip); in clear_func_from_hash()
7628 entry->ip = 0; in clear_func_from_hash()
7638 if (!tr->ops || !tr->ops->func_hash) in clear_func_from_hashes()
7640 mutex_lock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7641 clear_func_from_hash(func, tr->ops->func_hash->filter_hash); in clear_func_from_hashes()
7642 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash); in clear_func_from_hashes()
7643 mutex_unlock(&tr->ops->func_hash->regex_lock); in clear_func_from_hashes()
7655 MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n"); in add_to_clear_hash_list()
7659 func->ip = rec->ip; in add_to_clear_hash_list()
7660 list_add(&func->list, clear_list); in add_to_clear_hash_list()
7689 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { in ftrace_free_mem()
7690 if (end < pg->records[0].ip || in ftrace_free_mem()
7691 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in ftrace_free_mem()
7694 rec = bsearch(&key, pg->records, pg->index, in ftrace_free_mem()
7706 pg->index--; in ftrace_free_mem()
7707 ftrace_update_tot_cnt--; in ftrace_free_mem()
7708 if (!pg->index) { in ftrace_free_mem()
7709 *last_pg = pg->next; in ftrace_free_mem()
7710 pg->next = tmp_page; in ftrace_free_mem()
7718 (pg->index - (rec - pg->records)) * sizeof(*rec)); in ftrace_free_mem()
7763 count = __stop_mcount_loc - __start_mcount_loc; in ftrace_init()
7765 pr_info("ftrace: No functions to be traced?\n"); in ftrace_init()
7769 pr_info("ftrace: allocating %ld entries in %ld pages\n", in ftrace_init()
7776 pr_warn("ftrace: failed to allocate entries for functions\n"); in ftrace_init()
7780 pr_info("ftrace: allocated %ld pages with %ld groups\n", in ftrace_init()
7799 unsigned long trampoline = ops->trampoline; in ftrace_update_trampoline()
7802 if (ops->trampoline && ops->trampoline != trampoline && in ftrace_update_trampoline()
7803 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) { in ftrace_update_trampoline()
7807 ops->trampoline, ops->trampoline_size, false, in ftrace_update_trampoline()
7813 perf_event_text_poke((void *)ops->trampoline, NULL, 0, in ftrace_update_trampoline()
7814 (void *)ops->trampoline, in ftrace_update_trampoline()
7815 ops->trampoline_size); in ftrace_update_trampoline()
7821 INIT_LIST_HEAD(&tr->func_probes); in ftrace_init_trace_array()
7822 INIT_LIST_HEAD(&tr->mod_trace); in ftrace_init_trace_array()
7823 INIT_LIST_HEAD(&tr->mod_notrace); in ftrace_init_trace_array()
7851 tr->ops = &global_ops; in ftrace_init_global_array_ops()
7852 tr->ops->private = tr; in ftrace_init_global_array_ops()
7854 init_array_fgraph_ops(tr, tr->ops); in ftrace_init_global_array_ops()
7860 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { in ftrace_init_array_ops()
7861 if (WARN_ON(tr->ops->func != ftrace_stub)) in ftrace_init_array_ops()
7862 printk("ftrace ops had %pS for function\n", in ftrace_init_array_ops()
7863 tr->ops->func); in ftrace_init_array_ops()
7865 tr->ops->func = func; in ftrace_init_array_ops()
7866 tr->ops->private = tr; in ftrace_init_array_ops()
7871 tr->ops->func = ftrace_stub; in ftrace_reset_array_ops()
7893 if (op->flags & FTRACE_OPS_FL_STUB) in __ftrace_ops_list_func()
7900 * If any of the above fails then the op->func() is not executed. in __ftrace_ops_list_func()
7902 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && in __ftrace_ops_list_func()
7904 if (FTRACE_WARN_ON(!op->func)) { in __ftrace_ops_list_func()
7908 op->func(ip, parent_ip, op, fregs); in __ftrace_ops_list_func()
7960 if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) in ftrace_ops_assist_func()
7961 op->func(ip, parent_ip, op, fregs); in ftrace_ops_assist_func()
7968 * ftrace_ops_get_func - get the function a trampoline should call
7971 * Normally the mcount trampoline will call the ops->func, but there
7984 if (ops->flags & (FTRACE_OPS_FL_RECURSION | in ftrace_ops_get_func()
7988 return ops->func; in ftrace_ops_get_func()
8001 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_filter_pid_sched_switch_probe()
8002 no_pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_filter_pid_sched_switch_probe()
8005 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ftrace_filter_pid_sched_switch_probe()
8008 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ftrace_filter_pid_sched_switch_probe()
8009 next->pid); in ftrace_filter_pid_sched_switch_probe()
8020 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_pid_follow_sched_process_fork()
8023 pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_pid_follow_sched_process_fork()
8033 pid_list = rcu_dereference_sched(tr->function_pids); in ftrace_pid_follow_sched_process_exit()
8036 pid_list = rcu_dereference_sched(tr->function_no_pids); in ftrace_pid_follow_sched_process_exit()
8061 pid_list = rcu_dereference_protected(tr->function_pids, in clear_ftrace_pids()
8063 no_pid_list = rcu_dereference_protected(tr->function_no_pids, in clear_ftrace_pids()
8074 per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE; in clear_ftrace_pids()
8078 rcu_assign_pointer(tr->function_pids, NULL); in clear_ftrace_pids()
8081 rcu_assign_pointer(tr->function_no_pids, NULL); in clear_ftrace_pids()
8120 struct trace_array *tr = m->private; in fpid_start()
8125 pid_list = rcu_dereference_sched(tr->function_pids); in fpid_start()
8135 struct trace_array *tr = m->private; in fpid_next()
8136 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); in fpid_next()
8173 struct trace_array *tr = m->private; in fnpid_start()
8178 pid_list = rcu_dereference_sched(tr->function_no_pids); in fnpid_start()
8188 struct trace_array *tr = m->private; in fnpid_next()
8189 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids); in fnpid_next()
8208 struct trace_array *tr = inode->i_private; in pid_open()
8216 if ((file->f_mode & FMODE_WRITE) && in pid_open()
8217 (file->f_flags & O_TRUNC)) in pid_open()
8230 return -EINVAL; in pid_open()
8237 m = file->private_data; in pid_open()
8239 m->private = tr; in pid_open()
8267 pid_list = rcu_dereference_protected(tr->function_pids, in ignore_task_cpu()
8269 no_pid_list = rcu_dereference_protected(tr->function_no_pids, in ignore_task_cpu()
8273 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ignore_task_cpu()
8276 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, in ignore_task_cpu()
8277 current->pid); in ignore_task_cpu()
8284 struct seq_file *m = filp->private_data; in pid_write()
8285 struct trace_array *tr = m->private; in pid_write()
8298 filtered_pids = rcu_dereference_protected(tr->function_pids, in pid_write()
8300 other_pids = rcu_dereference_protected(tr->function_no_pids, in pid_write()
8304 filtered_pids = rcu_dereference_protected(tr->function_no_pids, in pid_write()
8306 other_pids = rcu_dereference_protected(tr->function_pids, in pid_write()
8310 ret = -EINVAL; in pid_write()
8321 rcu_assign_pointer(tr->function_pids, pid_list); in pid_write()
8324 rcu_assign_pointer(tr->function_no_pids, pid_list); in pid_write()
8372 struct trace_array *tr = inode->i_private; in ftrace_pid_release()
8407 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); in ftrace_init_tracefs_toplevel()
8414 * ftrace_kill - kill ftrace
8416 * This function should be used by panic code. It stops ftrace
8417 * but in a not so nice way. If you need to simply kill ftrace
8418 * from a non-atomic section, use ftrace_kill.
8429 * ftrace_is_dead - Test if ftrace is dead or not.
8431 * Returns: 1 if ftrace is "dead", zero otherwise.
8456 int size, i, ret; in prepare_direct_functions_for_ipmodify() local
8460 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in prepare_direct_functions_for_ipmodify()
8463 hash = ops->func_hash->filter_hash; in prepare_direct_functions_for_ipmodify()
8464 size = 1 << hash->size_bits; in prepare_direct_functions_for_ipmodify()
8465 for (i = 0; i < size; i++) { in prepare_direct_functions_for_ipmodify()
8466 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in prepare_direct_functions_for_ipmodify()
8467 unsigned long ip = entry->ip; in prepare_direct_functions_for_ipmodify()
8472 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) in prepare_direct_functions_for_ipmodify()
8482 if (!op->ops_func) in prepare_direct_functions_for_ipmodify()
8483 return -EBUSY; in prepare_direct_functions_for_ipmodify()
8485 ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); in prepare_direct_functions_for_ipmodify()
8505 int size, i; in cleanup_direct_functions_after_ipmodify() local
8507 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) in cleanup_direct_functions_after_ipmodify()
8512 hash = ops->func_hash->filter_hash; in cleanup_direct_functions_after_ipmodify()
8513 size = 1 << hash->size_bits; in cleanup_direct_functions_after_ipmodify()
8514 for (i = 0; i < size; i++) { in cleanup_direct_functions_after_ipmodify()
8515 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { in cleanup_direct_functions_after_ipmodify()
8516 unsigned long ip = entry->ip; in cleanup_direct_functions_after_ipmodify()
8521 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) in cleanup_direct_functions_after_ipmodify()
8531 if (found_op && op->ops_func) in cleanup_direct_functions_after_ipmodify()
8532 op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); in cleanup_direct_functions_after_ipmodify()
8576 * register_ftrace_function - register a function for profiling
8582 * Note: @ops->func and all the functions it calls must be labeled
8604 * unregister_ftrace_function - unregister a function for profiling.
8607 * Unregister a function that was added to be called by ftrace profiling.
8647 sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp); in kallsyms_callback()
8651 idx = sym - args->syms; in kallsyms_callback()
8652 if (args->addrs[idx]) in kallsyms_callback()
8658 args->addrs[idx] = addr; in kallsyms_callback()
8659 args->found++; in kallsyms_callback()
8660 return args->found == args->cnt ? 1 : 0; in kallsyms_callback()
8664 * ftrace_lookup_symbols - Lookup addresses for array of symbols
8676 * Returns: 0 if all provided symbols are found, -ESRCH otherwise.
8693 return found_all ? 0 : -ESRCH; in ftrace_lookup_symbols()
8708 /* ftrace_start_up is true if we want ftrace running */ in ftrace_startup_sysctl()
8724 /* ftrace_start_up is true if ftrace is running */ in ftrace_shutdown_sysctl()
8742 if (op->flags & FTRACE_OPS_FL_PERMANENT) in is_permanent_ops_registered()
8753 int ret = -ENODEV; in ftrace_enable_sysctl()
8767 /* we are starting ftrace again */ in ftrace_enable_sysctl()
8777 ret = -EBUSY; in ftrace_enable_sysctl()
8781 /* stopping ftrace calls (just send to ftrace_stub) */ in ftrace_enable_sysctl()