1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_TRACE_RECURSION_H 3 #define _LINUX_TRACE_RECURSION_H 4 5 #include <linux/interrupt.h> 6 #include <linux/sched.h> 7 8 #ifdef CONFIG_TRACING 9 10 /* Only current can touch trace_recursion */ 11 12 /* 13 * For function tracing recursion: 14 * The order of these bits are important. 15 * 16 * When function tracing occurs, the following steps are made: 17 * If arch does not support a ftrace feature: 18 * call internal function (uses INTERNAL bits) which calls... 19 * The function callback, which can use the FTRACE bits to 20 * check for recursion. 21 */ 22 enum { 23 /* Function recursion bits */ 24 TRACE_FTRACE_BIT, 25 TRACE_FTRACE_NMI_BIT, 26 TRACE_FTRACE_IRQ_BIT, 27 TRACE_FTRACE_SIRQ_BIT, 28 TRACE_FTRACE_TRANSITION_BIT, 29 30 /* Internal use recursion bits */ 31 TRACE_INTERNAL_BIT, 32 TRACE_INTERNAL_NMI_BIT, 33 TRACE_INTERNAL_IRQ_BIT, 34 TRACE_INTERNAL_SIRQ_BIT, 35 TRACE_INTERNAL_TRANSITION_BIT, 36 37 TRACE_BRANCH_BIT, 38 /* 39 * Abuse of the trace_recursion. 40 * As we need a way to maintain state if we are tracing the function 41 * graph in irq because we want to trace a particular function that 42 * was called in irq context but we have irq tracing off. Since this 43 * can only be modified by current, we can reuse trace_recursion. 44 */ 45 TRACE_IRQ_BIT, 46 47 /* Used to prevent recursion recording from recursing. */ 48 TRACE_RECORD_RECURSION_BIT, 49 }; 50 51 #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0) 52 #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0) 53 #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit))) 54 55 #define TRACE_CONTEXT_BITS 4 56 57 #define TRACE_FTRACE_START TRACE_FTRACE_BIT 58 59 #define TRACE_LIST_START TRACE_INTERNAL_BIT 60 61 #define TRACE_CONTEXT_MASK ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1) 62 63 /* 64 * Used for setting context 65 * NMI = 0 66 * IRQ = 1 67 * SOFTIRQ = 2 68 * NORMAL = 3 69 */ 70 enum { 71 TRACE_CTX_NMI, 72 TRACE_CTX_IRQ, 73 TRACE_CTX_SOFTIRQ, 74 TRACE_CTX_NORMAL, 75 TRACE_CTX_TRANSITION, 76 }; 77 trace_get_context_bit(void)78 static __always_inline int trace_get_context_bit(void) 79 { 80 unsigned char bit = interrupt_context_level(); 81 82 return TRACE_CTX_NORMAL - bit; 83 } 84 85 #ifdef CONFIG_FTRACE_RECORD_RECURSION 86 extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip); 87 # define do_ftrace_record_recursion(ip, pip) \ 88 do { \ 89 if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ 90 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ 91 ftrace_record_recursion(ip, pip); \ 92 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ 93 } \ 94 } while (0) 95 #else 96 # define do_ftrace_record_recursion(ip, pip) do { } while (0) 97 #endif 98 99 #ifdef CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING 100 # define trace_warn_on_no_rcu(ip) \ 101 ({ \ 102 bool __ret = !rcu_is_watching(); \ 103 if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \ 104 trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \ 105 WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \ 106 trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \ 107 } \ 108 __ret; \ 109 }) 110 #else 111 # define trace_warn_on_no_rcu(ip) false 112 #endif 113 114 /* 115 * Preemption is promised to be disabled when return bit >= 0. 116 */ trace_test_and_set_recursion(unsigned long ip,unsigned long pip,int start)117 static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip, 118 int start) 119 { 120 unsigned int val = READ_ONCE(current->trace_recursion); 121 int bit; 122 123 if (trace_warn_on_no_rcu(ip)) 124 return -1; 125 126 bit = trace_get_context_bit() + start; 127 if (unlikely(val & (1 << bit))) { 128 /* 129 * If an interrupt occurs during a trace, and another trace 130 * happens in that interrupt but before the preempt_count is 131 * updated to reflect the new interrupt context, then this 132 * will think a recursion occurred, and the event will be dropped. 133 * Let a single instance happen via the TRANSITION_BIT to 134 * not drop those events. 135 */ 136 bit = TRACE_CTX_TRANSITION + start; 137 if (val & (1 << bit)) { 138 do_ftrace_record_recursion(ip, pip); 139 return -1; 140 } 141 } 142 143 val |= 1 << bit; 144 current->trace_recursion = val; 145 barrier(); 146 147 preempt_disable_notrace(); 148 149 return bit; 150 } 151 152 /* 153 * Preemption will be enabled (if it was previously enabled). 154 */ trace_clear_recursion(int bit)155 static __always_inline void trace_clear_recursion(int bit) 156 { 157 preempt_enable_notrace(); 158 barrier(); 159 trace_recursion_clear(bit); 160 } 161 162 /** 163 * ftrace_test_recursion_trylock - tests for recursion in same context 164 * 165 * Use this for ftrace callbacks. This will detect if the function 166 * tracing recursed in the same context (normal vs interrupt), 167 * 168 * Returns: -1 if a recursion happened. 169 * >= 0 if no recursion. 170 */ ftrace_test_recursion_trylock(unsigned long ip,unsigned long parent_ip)171 static __always_inline int ftrace_test_recursion_trylock(unsigned long ip, 172 unsigned long parent_ip) 173 { 174 return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START); 175 } 176 177 /** 178 * ftrace_test_recursion_unlock - called when function callback is complete 179 * @bit: The return of a successful ftrace_test_recursion_trylock() 180 * 181 * This is used at the end of a ftrace callback. 182 */ ftrace_test_recursion_unlock(int bit)183 static __always_inline void ftrace_test_recursion_unlock(int bit) 184 { 185 trace_clear_recursion(bit); 186 } 187 188 #endif /* CONFIG_TRACING */ 189 #endif /* _LINUX_TRACE_RECURSION_H */ 190