1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_TRACE_RECURSION_H
3 #define _LINUX_TRACE_RECURSION_H
4
5 #include <linux/interrupt.h>
6 #include <linux/sched.h>
7
8 #ifdef CONFIG_TRACING
9
10 /* Only current can touch trace_recursion */
11
12 /*
13 * For function tracing recursion:
14 * The order of these bits are important.
15 *
16 * When function tracing occurs, the following steps are made:
17 * If arch does not support a ftrace feature:
18 * call internal function (uses INTERNAL bits) which calls...
19 * The function callback, which can use the FTRACE bits to
20 * check for recursion.
21 */
22 enum {
23 /* Function recursion bits */
24 TRACE_FTRACE_BIT,
25 TRACE_FTRACE_NMI_BIT,
26 TRACE_FTRACE_IRQ_BIT,
27 TRACE_FTRACE_SIRQ_BIT,
28 TRACE_FTRACE_TRANSITION_BIT,
29
30 /* Internal use recursion bits */
31 TRACE_INTERNAL_BIT,
32 TRACE_INTERNAL_NMI_BIT,
33 TRACE_INTERNAL_IRQ_BIT,
34 TRACE_INTERNAL_SIRQ_BIT,
35 TRACE_INTERNAL_TRANSITION_BIT,
36
37 /* Internal event use recursion bits */
38 TRACE_INTERNAL_EVENT_BIT,
39 TRACE_INTERNAL_EVENT_NMI_BIT,
40 TRACE_INTERNAL_EVENT_IRQ_BIT,
41 TRACE_INTERNAL_EVENT_SIRQ_BIT,
42 TRACE_INTERNAL_EVENT_TRANSITION_BIT,
43
44 TRACE_BRANCH_BIT,
45 /*
46 * Abuse of the trace_recursion.
47 * As we need a way to maintain state if we are tracing the function
48 * graph in irq because we want to trace a particular function that
49 * was called in irq context but we have irq tracing off. Since this
50 * can only be modified by current, we can reuse trace_recursion.
51 */
52 TRACE_IRQ_BIT,
53
54 /* Used to prevent recursion recording from recursing. */
55 TRACE_RECORD_RECURSION_BIT,
56 };
57
58 #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
59 #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
60 #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
61
62 #define TRACE_CONTEXT_BITS 4
63
64 #define TRACE_FTRACE_START TRACE_FTRACE_BIT
65
66 #define TRACE_LIST_START TRACE_INTERNAL_BIT
67
68 #define TRACE_EVENT_START TRACE_INTERNAL_EVENT_BIT
69
70 #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
71
72 /*
73 * Used for setting context
74 * NMI = 0
75 * IRQ = 1
76 * SOFTIRQ = 2
77 * NORMAL = 3
78 */
79 enum {
80 TRACE_CTX_NMI,
81 TRACE_CTX_IRQ,
82 TRACE_CTX_SOFTIRQ,
83 TRACE_CTX_NORMAL,
84 TRACE_CTX_TRANSITION,
85 };
86
trace_get_context_bit(void)87 static __always_inline int trace_get_context_bit(void)
88 {
89 unsigned char bit = interrupt_context_level();
90
91 return TRACE_CTX_NORMAL - bit;
92 }
93
94 #ifdef CONFIG_FTRACE_RECORD_RECURSION
95 extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
96 # define do_ftrace_record_recursion(ip, pip) \
97 do { \
98 if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
99 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
100 ftrace_record_recursion(ip, pip); \
101 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
102 } \
103 } while (0)
104 #else
105 # define do_ftrace_record_recursion(ip, pip) do { } while (0)
106 #endif
107
108 #ifdef CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING
109 # define trace_warn_on_no_rcu(ip) \
110 ({ \
111 bool __ret = !rcu_is_watching(); \
112 if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
113 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
114 WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \
115 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
116 } \
117 __ret; \
118 })
119 #else
120 # define trace_warn_on_no_rcu(ip) false
121 #endif
122
123 /*
124 * Preemption is promised to be disabled when return bit >= 0.
125 */
trace_test_and_set_recursion(unsigned long ip,unsigned long pip,int start)126 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
127 int start)
128 {
129 unsigned int val = READ_ONCE(current->trace_recursion);
130 int bit;
131
132 if (trace_warn_on_no_rcu(ip))
133 return -1;
134
135 bit = trace_get_context_bit() + start;
136 if (unlikely(val & (1 << bit))) {
137 /*
138 * If an interrupt occurs during a trace, and another trace
139 * happens in that interrupt but before the preempt_count is
140 * updated to reflect the new interrupt context, then this
141 * will think a recursion occurred, and the event will be dropped.
142 * Let a single instance happen via the TRANSITION_BIT to
143 * not drop those events.
144 */
145 bit = TRACE_CTX_TRANSITION + start;
146 if (val & (1 << bit)) {
147 do_ftrace_record_recursion(ip, pip);
148 return -1;
149 }
150 }
151
152 val |= 1 << bit;
153 current->trace_recursion = val;
154 barrier();
155
156 preempt_disable_notrace();
157
158 return bit;
159 }
160
161 /*
162 * Preemption will be enabled (if it was previously enabled).
163 */
trace_clear_recursion(int bit)164 static __always_inline void trace_clear_recursion(int bit)
165 {
166 preempt_enable_notrace();
167 barrier();
168 trace_recursion_clear(bit);
169 }
170
171 /**
172 * ftrace_test_recursion_trylock - tests for recursion in same context
173 *
174 * Use this for ftrace callbacks. This will detect if the function
175 * tracing recursed in the same context (normal vs interrupt),
176 *
177 * Returns: -1 if a recursion happened.
178 * >= 0 if no recursion.
179 */
ftrace_test_recursion_trylock(unsigned long ip,unsigned long parent_ip)180 static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
181 unsigned long parent_ip)
182 {
183 return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
184 }
185
186 /**
187 * ftrace_test_recursion_unlock - called when function callback is complete
188 * @bit: The return of a successful ftrace_test_recursion_trylock()
189 *
190 * This is used at the end of a ftrace callback.
191 */
ftrace_test_recursion_unlock(int bit)192 static __always_inline void ftrace_test_recursion_unlock(int bit)
193 {
194 trace_clear_recursion(bit);
195 }
196
197 #endif /* CONFIG_TRACING */
198 #endif /* _LINUX_TRACE_RECURSION_H */
199