1 // SPDX-License-Identifier: GPL-2.0 2 3 #define pr_fmt(fmt) "rethook: " fmt 4 5 #include <linux/bug.h> 6 #include <linux/kallsyms.h> 7 #include <linux/kprobes.h> 8 #include <linux/preempt.h> 9 #include <linux/rethook.h> 10 #include <linux/slab.h> 11 #include <linux/sort.h> 12 13 /* Return hook list (shadow stack by list) */ 14 15 /* 16 * This function is called from delayed_put_task_struct() when a task is 17 * dead and cleaned up to recycle any kretprobe instances associated with 18 * this task. These left over instances represent probed functions that 19 * have been called but will never return. 20 */ 21 void rethook_flush_task(struct task_struct *tk) 22 { 23 struct rethook_node *rhn; 24 struct llist_node *node; 25 26 node = __llist_del_all(&tk->rethooks); 27 while (node) { 28 rhn = container_of(node, struct rethook_node, llist); 29 node = node->next; 30 preempt_disable(); 31 rethook_recycle(rhn); 32 preempt_enable(); 33 } 34 } 35 36 static void rethook_free_rcu(struct rcu_head *head) 37 { 38 struct rethook *rh = container_of(head, struct rethook, rcu); 39 struct rethook_node *rhn; 40 struct freelist_node *node; 41 int count = 1; 42 43 node = rh->pool.head; 44 while (node) { 45 rhn = container_of(node, struct rethook_node, freelist); 46 node = node->next; 47 kfree(rhn); 48 count++; 49 } 50 51 /* The rh->ref is the number of pooled node + 1 */ 52 if (refcount_sub_and_test(count, &rh->ref)) 53 kfree(rh); 54 } 55 56 /** 57 * rethook_free() - Free struct rethook. 58 * @rh: the struct rethook to be freed. 59 * 60 * Free the rethook. Before calling this function, user must ensure the 61 * @rh::data is cleaned if needed (or, the handler can access it after 62 * calling this function.) This function will set the @rh to be freed 63 * after all rethook_node are freed (not soon). And the caller must 64 * not touch @rh after calling this. 65 */ 66 void rethook_free(struct rethook *rh) 67 { 68 WRITE_ONCE(rh->handler, NULL); 69 70 call_rcu(&rh->rcu, rethook_free_rcu); 71 } 72 73 /** 74 * rethook_alloc() - Allocate struct rethook. 75 * @data: a data to pass the @handler when hooking the return. 76 * @handler: the return hook callback function. 77 * 78 * Allocate and initialize a new rethook with @data and @handler. 79 * Return NULL if memory allocation fails or @handler is NULL. 80 * Note that @handler == NULL means this rethook is going to be freed. 81 */ 82 struct rethook *rethook_alloc(void *data, rethook_handler_t handler) 83 { 84 struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL); 85 86 if (!rh || !handler) { 87 kfree(rh); 88 return NULL; 89 } 90 91 rh->data = data; 92 rh->handler = handler; 93 rh->pool.head = NULL; 94 refcount_set(&rh->ref, 1); 95 96 return rh; 97 } 98 99 /** 100 * rethook_add_node() - Add a new node to the rethook. 101 * @rh: the struct rethook. 102 * @node: the struct rethook_node to be added. 103 * 104 * Add @node to @rh. User must allocate @node (as a part of user's 105 * data structure.) The @node fields are initialized in this function. 106 */ 107 void rethook_add_node(struct rethook *rh, struct rethook_node *node) 108 { 109 node->rethook = rh; 110 freelist_add(&node->freelist, &rh->pool); 111 refcount_inc(&rh->ref); 112 } 113 114 static void free_rethook_node_rcu(struct rcu_head *head) 115 { 116 struct rethook_node *node = container_of(head, struct rethook_node, rcu); 117 118 if (refcount_dec_and_test(&node->rethook->ref)) 119 kfree(node->rethook); 120 kfree(node); 121 } 122 123 /** 124 * rethook_recycle() - return the node to rethook. 125 * @node: The struct rethook_node to be returned. 126 * 127 * Return back the @node to @node::rethook. If the @node::rethook is already 128 * marked as freed, this will free the @node. 129 */ 130 void rethook_recycle(struct rethook_node *node) 131 { 132 lockdep_assert_preemption_disabled(); 133 134 if (likely(READ_ONCE(node->rethook->handler))) 135 freelist_add(&node->freelist, &node->rethook->pool); 136 else 137 call_rcu(&node->rcu, free_rethook_node_rcu); 138 } 139 NOKPROBE_SYMBOL(rethook_recycle); 140 141 /** 142 * rethook_try_get() - get an unused rethook node. 143 * @rh: The struct rethook which pools the nodes. 144 * 145 * Get an unused rethook node from @rh. If the node pool is empty, this 146 * will return NULL. Caller must disable preemption. 147 */ 148 struct rethook_node *rethook_try_get(struct rethook *rh) 149 { 150 rethook_handler_t handler = READ_ONCE(rh->handler); 151 struct freelist_node *fn; 152 153 lockdep_assert_preemption_disabled(); 154 155 /* Check whether @rh is going to be freed. */ 156 if (unlikely(!handler)) 157 return NULL; 158 159 /* 160 * This expects the caller will set up a rethook on a function entry. 161 * When the function returns, the rethook will eventually be reclaimed 162 * or released in the rethook_recycle() with call_rcu(). 163 * This means the caller must be run in the RCU-availabe context. 164 */ 165 if (unlikely(!rcu_is_watching())) 166 return NULL; 167 168 fn = freelist_try_get(&rh->pool); 169 if (!fn) 170 return NULL; 171 172 return container_of(fn, struct rethook_node, freelist); 173 } 174 NOKPROBE_SYMBOL(rethook_try_get); 175 176 /** 177 * rethook_hook() - Hook the current function return. 178 * @node: The struct rethook node to hook the function return. 179 * @regs: The struct pt_regs for the function entry. 180 * @mcount: True if this is called from mcount(ftrace) context. 181 * 182 * Hook the current running function return. This must be called when the 183 * function entry (or at least @regs must be the registers of the function 184 * entry.) @mcount is used for identifying the context. If this is called 185 * from ftrace (mcount) callback, @mcount must be set true. If this is called 186 * from the real function entry (e.g. kprobes) @mcount must be set false. 187 * This is because the way to hook the function return depends on the context. 188 */ 189 void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount) 190 { 191 arch_rethook_prepare(node, regs, mcount); 192 __llist_add(&node->llist, ¤t->rethooks); 193 } 194 NOKPROBE_SYMBOL(rethook_hook); 195 196 /* This assumes the 'tsk' is the current task or is not running. */ 197 static unsigned long __rethook_find_ret_addr(struct task_struct *tsk, 198 struct llist_node **cur) 199 { 200 struct rethook_node *rh = NULL; 201 struct llist_node *node = *cur; 202 203 if (!node) 204 node = tsk->rethooks.first; 205 else 206 node = node->next; 207 208 while (node) { 209 rh = container_of(node, struct rethook_node, llist); 210 if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) { 211 *cur = node; 212 return rh->ret_addr; 213 } 214 node = node->next; 215 } 216 return 0; 217 } 218 NOKPROBE_SYMBOL(__rethook_find_ret_addr); 219 220 /** 221 * rethook_find_ret_addr -- Find correct return address modified by rethook 222 * @tsk: Target task 223 * @frame: A frame pointer 224 * @cur: a storage of the loop cursor llist_node pointer for next call 225 * 226 * Find the correct return address modified by a rethook on @tsk in unsigned 227 * long type. 228 * The @tsk must be 'current' or a task which is not running. @frame is a hint 229 * to get the currect return address - which is compared with the 230 * rethook::frame field. The @cur is a loop cursor for searching the 231 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the 232 * first call, but '@cur' itself must NOT NULL. 233 * 234 * Returns found address value or zero if not found. 235 */ 236 unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame, 237 struct llist_node **cur) 238 { 239 struct rethook_node *rhn = NULL; 240 unsigned long ret; 241 242 if (WARN_ON_ONCE(!cur)) 243 return 0; 244 245 if (WARN_ON_ONCE(tsk != current && task_is_running(tsk))) 246 return 0; 247 248 do { 249 ret = __rethook_find_ret_addr(tsk, cur); 250 if (!ret) 251 break; 252 rhn = container_of(*cur, struct rethook_node, llist); 253 } while (rhn->frame != frame); 254 255 return ret; 256 } 257 NOKPROBE_SYMBOL(rethook_find_ret_addr); 258 259 void __weak arch_rethook_fixup_return(struct pt_regs *regs, 260 unsigned long correct_ret_addr) 261 { 262 /* 263 * Do nothing by default. If the architecture which uses a 264 * frame pointer to record real return address on the stack, 265 * it should fill this function to fixup the return address 266 * so that stacktrace works from the rethook handler. 267 */ 268 } 269 270 /* This function will be called from each arch-defined trampoline. */ 271 unsigned long rethook_trampoline_handler(struct pt_regs *regs, 272 unsigned long frame) 273 { 274 struct llist_node *first, *node = NULL; 275 unsigned long correct_ret_addr; 276 rethook_handler_t handler; 277 struct rethook_node *rhn; 278 279 correct_ret_addr = __rethook_find_ret_addr(current, &node); 280 if (!correct_ret_addr) { 281 pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n"); 282 BUG_ON(1); 283 } 284 285 instruction_pointer_set(regs, correct_ret_addr); 286 287 /* 288 * These loops must be protected from rethook_free_rcu() because those 289 * are accessing 'rhn->rethook'. 290 */ 291 preempt_disable(); 292 293 /* 294 * Run the handler on the shadow stack. Do not unlink the list here because 295 * stackdump inside the handlers needs to decode it. 296 */ 297 first = current->rethooks.first; 298 while (first) { 299 rhn = container_of(first, struct rethook_node, llist); 300 if (WARN_ON_ONCE(rhn->frame != frame)) 301 break; 302 handler = READ_ONCE(rhn->rethook->handler); 303 if (handler) 304 handler(rhn, rhn->rethook->data, regs); 305 306 if (first == node) 307 break; 308 first = first->next; 309 } 310 311 /* Fixup registers for returning to correct address. */ 312 arch_rethook_fixup_return(regs, correct_ret_addr); 313 314 /* Unlink used shadow stack */ 315 first = current->rethooks.first; 316 current->rethooks.first = node->next; 317 node->next = NULL; 318 319 while (first) { 320 rhn = container_of(first, struct rethook_node, llist); 321 first = first->next; 322 rethook_recycle(rhn); 323 } 324 preempt_enable(); 325 326 return correct_ret_addr; 327 } 328 NOKPROBE_SYMBOL(rethook_trampoline_handler); 329