Lines Matching defs:rec
459 struct ftrace_profile *rec = v;
462 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
466 rec++;
468 if ((void *)rec >= (void *)&pg->records[pg->index]) {
472 rec = &pg->records[0];
473 if (!rec->counter)
477 return rec;
538 struct ftrace_profile *rec = v;
550 if (unlikely(rec->counter == 0))
554 avg = div64_ul(rec->time, rec->counter);
562 if (core_kernel_text(rec->ip)) {
564 offset = rec->ip - (unsigned long)_text;
569 mod = __module_text_address(rec->ip);
573 offset = rec->ip - (unsigned long)mod->mem[MOD_TEXT].base;
580 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
582 seq_printf(m, " %-30.30s %10lu", str, rec->counter);
595 stddev_denom = rec->counter * (rec->counter - 1) * 1000;
597 stddev = rec->counter * rec->time_squared -
598 rec->time * rec->time;
603 trace_print_graph_duration(rec->time, &s);
739 struct ftrace_profile *rec;
749 hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
750 if (rec->ip == ip)
751 return rec;
758 struct ftrace_profile *rec)
762 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
763 hlist_add_head_rcu(&rec->node, &stat->hash[key]);
772 struct ftrace_profile *rec = NULL;
782 rec = ftrace_find_profiled_func(stat, ip);
783 if (rec)
792 rec = &stat->pages->records[stat->pages->index++];
793 rec->ip = ip;
794 ftrace_add_profile(stat, rec);
799 return rec;
807 struct ftrace_profile *rec;
818 rec = ftrace_find_profiled_func(stat, ip);
819 if (!rec) {
820 rec = ftrace_profile_alloc(stat, ip);
821 if (!rec)
825 rec->counter++;
875 struct ftrace_profile *rec;
911 rec = ftrace_find_profiled_func(stat, trace->func);
912 if (rec) {
913 rec->time += calltime;
914 rec->time_squared += calltime * calltime;
1592 #define do_for_each_ftrace_rec(pg, rec) \
1596 rec = &pg->records[_____i];
1606 const struct dyn_ftrace *rec = b;
1608 if (key->flags < rec->ip)
1610 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1618 struct dyn_ftrace *rec = NULL;
1629 rec = bsearch(&key, pg->records, pg->index,
1632 if (rec)
1635 return rec;
1645 * Returns: rec->ip if the related ftrace location is a least partly within
1652 struct dyn_ftrace *rec;
1656 rec = lookup_rec(start, end);
1657 if (rec)
1658 ip = rec->ip;
1711 /* Test if ops registered to this rec needs regs */
1712 static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
1719 /* pass rec in as regs to have non-NULL val */
1720 if (ftrace_ops_test(ops, rec->ip, rec)) {
1732 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
1734 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
1736 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
1738 static bool skip_record(struct dyn_ftrace *rec)
1746 return rec->flags & FTRACE_FL_DISABLED &&
1747 !(rec->flags & FTRACE_FL_ENABLED);
1766 struct dyn_ftrace *rec;
1784 do_for_each_ftrace_rec(pg, rec) {
1789 if (skip_record(rec))
1797 if (!notrace_hash || !ftrace_lookup_ip(notrace_hash, rec->ip))
1800 in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1801 in_notrace_hash = !!ftrace_lookup_ip(notrace_hash, rec->ip);
1814 rec->flags++;
1815 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
1819 rec->flags |= FTRACE_FL_DIRECT;
1826 if (ftrace_rec_count(rec) == 1 && ops->trampoline)
1827 rec->flags |= FTRACE_FL_TRAMP;
1835 rec->flags &= ~FTRACE_FL_TRAMP;
1842 rec->flags |= FTRACE_FL_REGS;
1844 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
1846 rec->flags--;
1855 rec->flags &= ~FTRACE_FL_DIRECT;
1858 * If the rec had REGS enabled and the ops that is
1863 if (ftrace_rec_count(rec) > 0 &&
1864 rec->flags & FTRACE_FL_REGS &&
1866 if (!test_rec_ops_needs_regs(rec))
1867 rec->flags &= ~FTRACE_FL_REGS;
1871 * The TRAMP needs to be set only if rec count
1877 if (ftrace_rec_count(rec) == 1 &&
1878 ftrace_find_tramp_ops_any_other(rec, ops))
1879 rec->flags |= FTRACE_FL_TRAMP;
1881 rec->flags &= ~FTRACE_FL_TRAMP;
1885 * if rec count is zero.
1890 * If the rec has a single associated ops, and ops->func can be
1894 ftrace_rec_count(rec) == 1 &&
1896 rec->flags |= FTRACE_FL_CALL_OPS;
1898 rec->flags &= ~FTRACE_FL_CALL_OPS;
1903 update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
2004 struct dyn_ftrace *rec, *end = NULL;
2030 /* Update rec->flags */
2031 do_for_each_ftrace_rec(pg, rec) {
2033 if (rec->flags & FTRACE_FL_DISABLED)
2040 in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
2041 in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
2046 if (rec->flags & FTRACE_FL_IPMODIFY) {
2049 /* Cannot have two ipmodify on same rec */
2062 (rec->flags & FTRACE_FL_DIRECT));
2076 rec->flags |= FTRACE_FL_IPMODIFY;
2079 rec->flags &= ~FTRACE_FL_IPMODIFY;
2086 end = rec;
2089 do_for_each_ftrace_rec(pg, rec) {
2091 if (rec->flags & FTRACE_FL_DISABLED)
2094 if (rec == end)
2097 in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
2098 in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
2103 rec->flags &= ~FTRACE_FL_IPMODIFY;
2105 rec->flags |= FTRACE_FL_IPMODIFY;
2185 * @rec: The record that failed
2194 void ftrace_bug(int failed, struct dyn_ftrace *rec)
2196 unsigned long ip = rec ? rec->ip : 0;
2224 if (rec) {
2227 pr_info("ftrace record flags: %lx\n", rec->flags);
2228 pr_cont(" (%ld)%s%s", ftrace_rec_count(rec),
2229 rec->flags & FTRACE_FL_REGS ? " R" : " ",
2230 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ");
2231 if (rec->flags & FTRACE_FL_TRAMP_EN) {
2232 ops = ftrace_find_tramp_ops_any(rec);
2238 ops = ftrace_find_tramp_ops_next(rec, ops);
2244 ip = ftrace_get_addr_curr(rec);
2251 static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
2257 if (skip_record(rec))
2271 if (enable && ftrace_rec_count(rec))
2281 if (!(rec->flags & FTRACE_FL_REGS) !=
2282 !(rec->flags & FTRACE_FL_REGS_EN))
2285 if (!(rec->flags & FTRACE_FL_TRAMP) !=
2286 !(rec->flags & FTRACE_FL_TRAMP_EN))
2300 if (ftrace_rec_count(rec) == 1) {
2301 if (!(rec->flags & FTRACE_FL_DIRECT) !=
2302 !(rec->flags & FTRACE_FL_DIRECT_EN))
2304 } else if (rec->flags & FTRACE_FL_DIRECT_EN) {
2313 if (ftrace_rec_count(rec) == 1) {
2314 if (!(rec->flags & FTRACE_FL_CALL_OPS) !=
2315 !(rec->flags & FTRACE_FL_CALL_OPS_EN))
2317 } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
2323 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
2327 /* Save off if rec is being enabled (for return value) */
2328 flag ^= rec->flags & FTRACE_FL_ENABLED;
2331 rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED;
2333 if (rec->flags & FTRACE_FL_REGS)
2334 rec->flags |= FTRACE_FL_REGS_EN;
2336 rec->flags &= ~FTRACE_FL_REGS_EN;
2339 if (rec->flags & FTRACE_FL_TRAMP)
2340 rec->flags |= FTRACE_FL_TRAMP_EN;
2342 rec->flags &= ~FTRACE_FL_TRAMP_EN;
2346 if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY))
2347 rec->flags |= FTRACE_FL_MODIFIED;
2355 if (ftrace_rec_count(rec) == 1) {
2356 if (rec->flags & FTRACE_FL_DIRECT)
2357 rec->flags |= FTRACE_FL_DIRECT_EN;
2359 rec->flags &= ~FTRACE_FL_DIRECT_EN;
2365 rec->flags &= ~FTRACE_FL_DIRECT_EN;
2370 if (ftrace_rec_count(rec) == 1) {
2371 if (rec->flags & FTRACE_FL_CALL_OPS)
2372 rec->flags |= FTRACE_FL_CALL_OPS_EN;
2374 rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
2380 rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
2404 if (!ftrace_rec_count(rec))
2405 rec->flags &= FTRACE_NOCLEAR_FLAGS;
2411 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
2422 * @rec: the record to update
2428 int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
2430 return ftrace_check_record(rec, enable, true);
2435 * @rec: the record to test
2442 int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
2444 return ftrace_check_record(rec, enable, false);
2448 ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
2451 unsigned long ip = rec->ip;
2466 ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
2469 unsigned long ip = rec->ip;
2484 ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
2487 unsigned long ip = rec->ip;
2502 ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
2505 unsigned long ip = rec->ip;
2509 * If they are being removed, and this rec has a tramp,
2510 * and this rec is in the ops list, then it would be the
2519 * Need to find the current trampoline for a rec.
2520 * Now, a trampoline is only attached to a rec if there
2522 * when we are adding another op to the rec or removing the
2524 * ignore it because it hasn't attached itself to the rec
2572 ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
2575 unsigned long ip = rec->ip;
2578 /* pass rec in as regs to have non-NULL val */
2587 ftrace_find_unique_ops(struct dyn_ftrace *rec)
2590 unsigned long ip = rec->ip;
2639 * @rec: The ftrace record descriptor
2647 unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
2652 if ((rec->flags & FTRACE_FL_DIRECT) &&
2653 (ftrace_rec_count(rec) == 1)) {
2654 addr = ftrace_find_rec_direct(rec->ip);
2661 if (rec->flags & FTRACE_FL_TRAMP) {
2662 ops = ftrace_find_tramp_ops_new(rec);
2665 (void *)rec->ip, (void *)rec->ip, rec->flags);
2672 if (rec->flags & FTRACE_FL_REGS)
2680 * @rec: The ftrace record descriptor
2688 unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
2694 if (rec->flags & FTRACE_FL_DIRECT_EN) {
2695 addr = ftrace_find_rec_direct(rec->ip);
2702 if (rec->flags & FTRACE_FL_TRAMP_EN) {
2703 ops = ftrace_find_tramp_ops_curr(rec);
2706 (void *)rec->ip, (void *)rec->ip);
2713 if (rec->flags & FTRACE_FL_REGS_EN)
2720 __ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
2726 ftrace_addr = ftrace_get_addr_new(rec);
2729 ftrace_old_addr = ftrace_get_addr_curr(rec);
2731 ret = ftrace_update_record(rec, enable);
2741 return ftrace_make_call(rec, ftrace_addr);
2745 return ftrace_make_nop(NULL, rec, ftrace_old_addr);
2749 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
2757 struct dyn_ftrace *rec;
2766 do_for_each_ftrace_rec(pg, rec) {
2768 if (skip_record(rec))
2771 failed = __ftrace_replace_code(rec, enable);
2773 ftrace_bug(failed, rec);
2855 ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
2862 ret = ftrace_init_nop(mod, rec);
2865 ftrace_bug(ret, rec);
3184 * sanity check to make sure all rec flags are cleared.
3189 struct dyn_ftrace *rec;
3191 do_for_each_ftrace_rec(pg, rec) {
3192 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS))
3194 (void *)rec->ip, rec->flags);
4163 struct dyn_ftrace *rec = NULL;
4175 rec = &iter->pg->records[iter->idx++];
4177 !ftrace_lookup_ip(iter->hash, rec->ip)) ||
4180 !(rec->flags & FTRACE_FL_ENABLED)) ||
4183 !(rec->flags & FTRACE_FL_TOUCHED))) {
4185 rec = NULL;
4190 if (!rec)
4194 iter->func = rec;
4297 arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
4303 struct dyn_ftrace *rec)
4307 ptr = arch_ftrace_trampoline_func(ops, rec);
4323 static int test_for_valid_rec(struct dyn_ftrace *rec)
4329 ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str);
4333 rec->flags |= FTRACE_FL_DISABLED;
4348 struct dyn_ftrace *rec;
4351 do_for_each_ftrace_rec(pg, rec) {
4352 test_for_valid_rec(rec);
4399 static inline int test_for_valid_rec(struct dyn_ftrace *rec)
4411 static void print_subops(struct seq_file *m, struct ftrace_ops *ops, struct dyn_ftrace *rec)
4418 hash_contains_ip(rec->ip, subops->func_hash)))
4439 add_trampoline_func(m, subops, rec);
4450 struct dyn_ftrace *rec;
4466 rec = iter->func;
4468 if (!rec)
4472 seq_printf(m, "%lx ", rec->ip);
4474 if (print_rec(m, rec->ip)) {
4475 /* This should only happen when a rec is disabled */
4476 WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED));
4485 ftrace_rec_count(rec),
4486 rec->flags & FTRACE_FL_REGS ? " R" : " ",
4487 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ",
4488 rec->flags & FTRACE_FL_DIRECT ? " D" : " ",
4489 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ",
4490 rec->flags & FTRACE_FL_MODIFIED ? " M " : " ");
4491 if (rec->flags & FTRACE_FL_TRAMP_EN) {
4492 ops = ftrace_find_tramp_ops_any(rec);
4498 add_trampoline_func(m, ops, rec);
4499 print_subops(m, ops, rec);
4500 ops = ftrace_find_tramp_ops_next(rec, ops);
4505 add_trampoline_func(m, NULL, rec);
4507 if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
4508 ops = ftrace_find_unique_ops(rec);
4512 print_subops(m, ops, rec);
4517 if (rec->flags & FTRACE_FL_DIRECT) {
4520 direct = ftrace_find_rec_direct(rec->ip);
4822 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
4827 entry = ftrace_lookup_ip(hash, rec->ip);
4838 if (add_hash_entry(hash, rec->ip) == NULL)
4850 struct dyn_ftrace *rec;
4856 do_for_each_ftrace_rec(pg, rec) {
4862 rec = &pg->records[index];
4863 enter_record(hash, rec, clear_filter);
4888 ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
4894 if (lookup_ip(rec->ip, &modname, str)) {
4895 /* This should only happen when a rec is disabled */
4897 !(rec->flags & FTRACE_FL_DISABLED));
4934 struct dyn_ftrace *rec;
4963 do_for_each_ftrace_rec(pg, rec) {
4965 if (rec->flags & FTRACE_FL_DISABLED)
4968 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
4969 ret = enter_record(hash, rec, clear_filter);
6984 struct dyn_ftrace *rec;
7001 do_for_each_ftrace_rec(pg, rec) {
7003 if (rec->flags & FTRACE_FL_DISABLED)
7006 if (ftrace_match_record(rec, &func_g, NULL, 0)) {
7007 entry = ftrace_lookup_ip(hash, rec->ip);
7014 if (add_hash_entry(hash, rec->ip) == NULL)
7181 struct dyn_ftrace *rec;
7276 rec = &pg->records[pg->index++];
7277 rec->ip = addr;
7421 static int referenced_filters(struct dyn_ftrace *rec)
7427 if (ops_references_ip(ops, rec->ip)) {
7434 rec->flags |= FTRACE_FL_REGS;
7436 rec->flags |= FTRACE_FL_TRAMP;
7438 rec->flags &= ~FTRACE_FL_TRAMP;
7449 struct dyn_ftrace *rec;
7456 rec = &pg->records[i];
7457 entry = __ftrace_lookup_ip(hash, rec->ip);
7459 * Do not allow this rec to match again.
7505 struct dyn_ftrace *rec;
7533 rec = &pg->records[0];
7534 if (within_module(rec->ip, mod)) {
7577 struct dyn_ftrace *rec;
7601 do_for_each_ftrace_rec(pg, rec) {
7609 if (!within_module(rec->ip, mod))
7615 if (!test_for_valid_rec(rec)) {
7617 rec->flags = FTRACE_FL_DISABLED;
7630 cnt += referenced_filters(rec);
7632 rec->flags &= ~FTRACE_FL_DISABLED;
7633 rec->flags += cnt;
7636 int failed = __ftrace_replace_code(rec, 1);
7638 ftrace_bug(failed, rec);
7670 struct dyn_ftrace *rec)
7679 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
7693 mod_func->ip = rec->ip - offset;
7816 struct dyn_ftrace *rec) { }
7850 * Do not allow this rec to match again.
7876 struct dyn_ftrace *rec)
7886 func->ip = rec->ip;
7897 struct dyn_ftrace *rec;
7921 rec = bsearch(&key, pg->records, pg->index,
7924 if (!rec)
7927 /* rec will be cleared from hashes after ftrace_lock unlock */
7928 add_to_clear_hash_list(&clear_hash, rec);
7931 save_ftrace_mod_rec(mod_map, rec);
7944 memmove(rec, rec + 1,
7945 (pg->index - (rec - pg->records)) * sizeof(*rec));