rethook.c (c900529f3d9161bfde5cca0754f83b4d3c3e0220) rethook.c (4bbd9345565933823f38a419df65661f12adbe5e)
1// SPDX-License-Identifier: GPL-2.0
2
3#define pr_fmt(fmt) "rethook: " fmt
4
5#include <linux/bug.h>
6#include <linux/kallsyms.h>
7#include <linux/kprobes.h>
8#include <linux/preempt.h>
9#include <linux/rethook.h>
10#include <linux/slab.h>
11#include <linux/sort.h>
1// SPDX-License-Identifier: GPL-2.0
2
3#define pr_fmt(fmt) "rethook: " fmt
4
5#include <linux/bug.h>
6#include <linux/kallsyms.h>
7#include <linux/kprobes.h>
8#include <linux/preempt.h>
9#include <linux/rethook.h>
10#include <linux/slab.h>
11#include <linux/sort.h>
12#include <linux/smp.h>
12
13/* Return hook list (shadow stack by list) */
14
15/*
16 * This function is called from delayed_put_task_struct() when a task is
17 * dead and cleaned up to recycle any kretprobe instances associated with
18 * this task. These left over instances represent probed functions that
19 * have been called but will never return.

--- 11 unchanged lines hidden (view full) ---

31 rethook_recycle(rhn);
32 preempt_enable();
33 }
34}
35
36static void rethook_free_rcu(struct rcu_head *head)
37{
38 struct rethook *rh = container_of(head, struct rethook, rcu);
13
14/* Return hook list (shadow stack by list) */
15
16/*
17 * This function is called from delayed_put_task_struct() when a task is
18 * dead and cleaned up to recycle any kretprobe instances associated with
19 * this task. These left over instances represent probed functions that
20 * have been called but will never return.

--- 11 unchanged lines hidden (view full) ---

32 rethook_recycle(rhn);
33 preempt_enable();
34 }
35}
36
37static void rethook_free_rcu(struct rcu_head *head)
38{
39 struct rethook *rh = container_of(head, struct rethook, rcu);
39 struct rethook_node *rhn;
40 struct freelist_node *node;
41 int count = 1;
42
43 node = rh->pool.head;
44 while (node) {
45 rhn = container_of(node, struct rethook_node, freelist);
46 node = node->next;
47 kfree(rhn);
48 count++;
49 }
50
51 /* The rh->ref is the number of pooled node + 1 */
52 if (refcount_sub_and_test(count, &rh->ref))
53 kfree(rh);
40 objpool_fini(&rh->pool);
54}
55
56/**
57 * rethook_stop() - Stop using a rethook.
58 * @rh: the struct rethook to stop.
59 *
60 * Stop using a rethook to prepare for freeing it. If you want to wait for
61 * all running rethook handler before calling rethook_free(), you need to

--- 16 unchanged lines hidden (view full) ---

78 */
79void rethook_free(struct rethook *rh)
80{
81 WRITE_ONCE(rh->handler, NULL);
82
83 call_rcu(&rh->rcu, rethook_free_rcu);
84}
85
41}
42
43/**
44 * rethook_stop() - Stop using a rethook.
45 * @rh: the struct rethook to stop.
46 *
47 * Stop using a rethook to prepare for freeing it. If you want to wait for
48 * all running rethook handler before calling rethook_free(), you need to

--- 16 unchanged lines hidden (view full) ---

65 */
66void rethook_free(struct rethook *rh)
67{
68 WRITE_ONCE(rh->handler, NULL);
69
70 call_rcu(&rh->rcu, rethook_free_rcu);
71}
72
73static int rethook_init_node(void *nod, void *context)
74{
75 struct rethook_node *node = nod;
76
77 node->rethook = context;
78 return 0;
79}
80
81static int rethook_fini_pool(struct objpool_head *head, void *context)
82{
83 kfree(context);
84 return 0;
85}
86
86/**
87 * rethook_alloc() - Allocate struct rethook.
88 * @data: a data to pass the @handler when hooking the return.
87/**
88 * rethook_alloc() - Allocate struct rethook.
89 * @data: a data to pass the @handler when hooking the return.
89 * @handler: the return hook callback function.
90 * @handler: the return hook callback function, must NOT be NULL
91 * @size: node size: rethook node and additional data
92 * @num: number of rethook nodes to be preallocated
90 *
91 * Allocate and initialize a new rethook with @data and @handler.
93 *
94 * Allocate and initialize a new rethook with @data and @handler.
92 * Return NULL if memory allocation fails or @handler is NULL.
95 * Return pointer of new rethook, or error codes for failures.
96 *
93 * Note that @handler == NULL means this rethook is going to be freed.
94 */
97 * Note that @handler == NULL means this rethook is going to be freed.
98 */
95struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
99struct rethook *rethook_alloc(void *data, rethook_handler_t handler,
100 int size, int num)
96{
101{
97 struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
102 struct rethook *rh;
98
103
99 if (!rh || !handler) {
100 kfree(rh);
101 return NULL;
102 }
104 if (!handler || num <= 0 || size < sizeof(struct rethook_node))
105 return ERR_PTR(-EINVAL);
103
106
107 rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
108 if (!rh)
109 return ERR_PTR(-ENOMEM);
110
104 rh->data = data;
105 rh->handler = handler;
111 rh->data = data;
112 rh->handler = handler;
106 rh->pool.head = NULL;
107 refcount_set(&rh->ref, 1);
108
113
114 /* initialize the objpool for rethook nodes */
115 if (objpool_init(&rh->pool, num, size, GFP_KERNEL, rh,
116 rethook_init_node, rethook_fini_pool)) {
117 kfree(rh);
118 return ERR_PTR(-ENOMEM);
119 }
109 return rh;
110}
111
120 return rh;
121}
122
112/**
113 * rethook_add_node() - Add a new node to the rethook.
114 * @rh: the struct rethook.
115 * @node: the struct rethook_node to be added.
116 *
117 * Add @node to @rh. User must allocate @node (as a part of user's
118 * data structure.) The @node fields are initialized in this function.
119 */
120void rethook_add_node(struct rethook *rh, struct rethook_node *node)
121{
122 node->rethook = rh;
123 freelist_add(&node->freelist, &rh->pool);
124 refcount_inc(&rh->ref);
125}
126
127static void free_rethook_node_rcu(struct rcu_head *head)
128{
129 struct rethook_node *node = container_of(head, struct rethook_node, rcu);
123static void free_rethook_node_rcu(struct rcu_head *head)
124{
125 struct rethook_node *node = container_of(head, struct rethook_node, rcu);
126 struct rethook *rh = node->rethook;
130
127
131 if (refcount_dec_and_test(&node->rethook->ref))
132 kfree(node->rethook);
133 kfree(node);
128 objpool_drop(node, &rh->pool);
134}
135
136/**
137 * rethook_recycle() - return the node to rethook.
138 * @node: The struct rethook_node to be returned.
139 *
140 * Return back the @node to @node::rethook. If the @node::rethook is already
141 * marked as freed, this will free the @node.
142 */
143void rethook_recycle(struct rethook_node *node)
144{
145 lockdep_assert_preemption_disabled();
146
147 if (likely(READ_ONCE(node->rethook->handler)))
129}
130
131/**
132 * rethook_recycle() - return the node to rethook.
133 * @node: The struct rethook_node to be returned.
134 *
135 * Return back the @node to @node::rethook. If the @node::rethook is already
136 * marked as freed, this will free the @node.
137 */
138void rethook_recycle(struct rethook_node *node)
139{
140 lockdep_assert_preemption_disabled();
141
142 if (likely(READ_ONCE(node->rethook->handler)))
148 freelist_add(&node->freelist, &node->rethook->pool);
143 objpool_push(node, &node->rethook->pool);
149 else
150 call_rcu(&node->rcu, free_rethook_node_rcu);
151}
152NOKPROBE_SYMBOL(rethook_recycle);
153
154/**
155 * rethook_try_get() - get an unused rethook node.
156 * @rh: The struct rethook which pools the nodes.
157 *
158 * Get an unused rethook node from @rh. If the node pool is empty, this
159 * will return NULL. Caller must disable preemption.
160 */
161struct rethook_node *rethook_try_get(struct rethook *rh)
162{
163 rethook_handler_t handler = READ_ONCE(rh->handler);
144 else
145 call_rcu(&node->rcu, free_rethook_node_rcu);
146}
147NOKPROBE_SYMBOL(rethook_recycle);
148
149/**
150 * rethook_try_get() - get an unused rethook node.
151 * @rh: The struct rethook which pools the nodes.
152 *
153 * Get an unused rethook node from @rh. If the node pool is empty, this
154 * will return NULL. Caller must disable preemption.
155 */
156struct rethook_node *rethook_try_get(struct rethook *rh)
157{
158 rethook_handler_t handler = READ_ONCE(rh->handler);
164 struct freelist_node *fn;
165
166 lockdep_assert_preemption_disabled();
167
168 /* Check whether @rh is going to be freed. */
169 if (unlikely(!handler))
170 return NULL;
171
172 /*
173 * This expects the caller will set up a rethook on a function entry.
174 * When the function returns, the rethook will eventually be reclaimed
175 * or released in the rethook_recycle() with call_rcu().
176 * This means the caller must be run in the RCU-availabe context.
177 */
178 if (unlikely(!rcu_is_watching()))
179 return NULL;
180
159
160 lockdep_assert_preemption_disabled();
161
162 /* Check whether @rh is going to be freed. */
163 if (unlikely(!handler))
164 return NULL;
165
166 /*
167 * This expects the caller will set up a rethook on a function entry.
168 * When the function returns, the rethook will eventually be reclaimed
169 * or released in the rethook_recycle() with call_rcu().
170 * This means the caller must be run in the RCU-availabe context.
171 */
172 if (unlikely(!rcu_is_watching()))
173 return NULL;
174
181 fn = freelist_try_get(&rh->pool);
182 if (!fn)
183 return NULL;
184
185 return container_of(fn, struct rethook_node, freelist);
175 return (struct rethook_node *)objpool_pop(&rh->pool);
186}
187NOKPROBE_SYMBOL(rethook_try_get);
188
189/**
190 * rethook_hook() - Hook the current function return.
191 * @node: The struct rethook node to hook the function return.
192 * @regs: The struct pt_regs for the function entry.
193 * @mcount: True if this is called from mcount(ftrace) context.

--- 149 unchanged lines hidden --- 		176}
177NOKPROBE_SYMBOL(rethook_try_get);
178
179/**
180 * rethook_hook() - Hook the current function return.
181 * @node: The struct rethook node to hook the function return.
182 * @regs: The struct pt_regs for the function entry.
183 * @mcount: True if this is called from mcount(ftrace) context.

--- 149 unchanged lines hidden ---