xref: /linux/kernel/livepatch/patch.c (revision b77e0ce62d63a761ffb7f7245a215a49f5921c2f) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * patch.c - livepatch patching functions
4  *
5  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
6  * Copyright (C) 2014 SUSE
7  * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
8  */
9 
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 
12 #include <linux/livepatch.h>
13 #include <linux/list.h>
14 #include <linux/ftrace.h>
15 #include <linux/rculist.h>
16 #include <linux/slab.h>
17 #include <linux/bug.h>
18 #include <linux/printk.h>
19 #include "core.h"
20 #include "patch.h"
21 #include "transition.h"
22 
23 static LIST_HEAD(klp_ops);
24 
25 struct klp_ops *klp_find_ops(void *old_func)
26 {
27 	struct klp_ops *ops;
28 	struct klp_func *func;
29 
30 	list_for_each_entry(ops, &klp_ops, node) {
31 		func = list_first_entry(&ops->func_stack, struct klp_func,
32 					stack_node);
33 		if (func->old_func == old_func)
34 			return ops;
35 	}
36 
37 	return NULL;
38 }
39 
40 static void notrace klp_ftrace_handler(unsigned long ip,
41 				       unsigned long parent_ip,
42 				       struct ftrace_ops *fops,
43 				       struct ftrace_regs *fregs)
44 {
45 	struct klp_ops *ops;
46 	struct klp_func *func;
47 	int patch_state;
48 	int bit;
49 
50 	ops = container_of(fops, struct klp_ops, fops);
51 
52 	bit = ftrace_test_recursion_trylock(ip, parent_ip);
53 	if (WARN_ON_ONCE(bit < 0))
54 		return;
55 	/*
56 	 * A variant of synchronize_rcu() is used to allow patching functions
57 	 * where RCU is not watching, see klp_synchronize_transition().
58 	 */
59 	preempt_disable_notrace();
60 
61 	func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
62 				      stack_node);
63 
64 	/*
65 	 * func should never be NULL because preemption should be disabled here
66 	 * and unregister_ftrace_function() does the equivalent of a
67 	 * synchronize_rcu() before the func_stack removal.
68 	 */
69 	if (WARN_ON_ONCE(!func))
70 		goto unlock;
71 
72 	/*
73 	 * In the enable path, enforce the order of the ops->func_stack and
74 	 * func->transition reads.  The corresponding write barrier is in
75 	 * __klp_enable_patch().
76 	 *
77 	 * (Note that this barrier technically isn't needed in the disable
78 	 * path.  In the rare case where klp_update_patch_state() runs before
79 	 * this handler, its TIF_PATCH_PENDING read and this func->transition
80 	 * read need to be ordered.  But klp_update_patch_state() already
81 	 * enforces that.)
82 	 */
83 	smp_rmb();
84 
85 	if (unlikely(func->transition)) {
86 
87 		/*
88 		 * Enforce the order of the func->transition and
89 		 * current->patch_state reads.  Otherwise we could read an
90 		 * out-of-date task state and pick the wrong function.  The
91 		 * corresponding write barrier is in klp_init_transition().
92 		 */
93 		smp_rmb();
94 
95 		patch_state = current->patch_state;
96 
97 		WARN_ON_ONCE(patch_state == KLP_UNDEFINED);
98 
99 		if (patch_state == KLP_UNPATCHED) {
100 			/*
101 			 * Use the previously patched version of the function.
102 			 * If no previous patches exist, continue with the
103 			 * original function.
104 			 */
105 			func = list_entry_rcu(func->stack_node.next,
106 					      struct klp_func, stack_node);
107 
108 			if (&func->stack_node == &ops->func_stack)
109 				goto unlock;
110 		}
111 	}
112 
113 	/*
114 	 * NOPs are used to replace existing patches with original code.
115 	 * Do nothing! Setting pc would cause an infinite loop.
116 	 */
117 	if (func->nop)
118 		goto unlock;
119 
120 	klp_arch_set_pc(fregs, (unsigned long)func->new_func);
121 
122 unlock:
123 	preempt_enable_notrace();
124 	ftrace_test_recursion_unlock(bit);
125 }
126 
127 /*
128  * Convert a function address into the appropriate ftrace location.
129  *
130  * Usually this is just the address of the function, but on some architectures
131  * it's more complicated so allow them to provide a custom behaviour.
132  */
133 #ifndef klp_get_ftrace_location
134 static unsigned long klp_get_ftrace_location(unsigned long faddr)
135 {
136 	return faddr;
137 }
138 #endif
139 
140 static void klp_unpatch_func(struct klp_func *func)
141 {
142 	struct klp_ops *ops;
143 
144 	if (WARN_ON(!func->patched))
145 		return;
146 	if (WARN_ON(!func->old_func))
147 		return;
148 
149 	ops = klp_find_ops(func->old_func);
150 	if (WARN_ON(!ops))
151 		return;
152 
153 	if (list_is_singular(&ops->func_stack)) {
154 		unsigned long ftrace_loc;
155 
156 		ftrace_loc =
157 			klp_get_ftrace_location((unsigned long)func->old_func);
158 		if (WARN_ON(!ftrace_loc))
159 			return;
160 
161 		WARN_ON(unregister_ftrace_function(&ops->fops));
162 		WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
163 
164 		list_del_rcu(&func->stack_node);
165 		list_del(&ops->node);
166 		kfree(ops);
167 	} else {
168 		list_del_rcu(&func->stack_node);
169 	}
170 
171 	func->patched = false;
172 }
173 
174 static int klp_patch_func(struct klp_func *func)
175 {
176 	struct klp_ops *ops;
177 	int ret;
178 
179 	if (WARN_ON(!func->old_func))
180 		return -EINVAL;
181 
182 	if (WARN_ON(func->patched))
183 		return -EINVAL;
184 
185 	ops = klp_find_ops(func->old_func);
186 	if (!ops) {
187 		unsigned long ftrace_loc;
188 
189 		ftrace_loc =
190 			klp_get_ftrace_location((unsigned long)func->old_func);
191 		if (!ftrace_loc) {
192 			pr_err("failed to find location for function '%s'\n",
193 				func->old_name);
194 			return -EINVAL;
195 		}
196 
197 		ops = kzalloc(sizeof(*ops), GFP_KERNEL);
198 		if (!ops)
199 			return -ENOMEM;
200 
201 		ops->fops.func = klp_ftrace_handler;
202 		ops->fops.flags = FTRACE_OPS_FL_DYNAMIC |
203 #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
204 				  FTRACE_OPS_FL_SAVE_REGS |
205 #endif
206 				  FTRACE_OPS_FL_IPMODIFY |
207 				  FTRACE_OPS_FL_PERMANENT;
208 
209 		list_add(&ops->node, &klp_ops);
210 
211 		INIT_LIST_HEAD(&ops->func_stack);
212 		list_add_rcu(&func->stack_node, &ops->func_stack);
213 
214 		ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
215 		if (ret) {
216 			pr_err("failed to set ftrace filter for function '%s' (%d)\n",
217 			       func->old_name, ret);
218 			goto err;
219 		}
220 
221 		ret = register_ftrace_function(&ops->fops);
222 		if (ret) {
223 			pr_err("failed to register ftrace handler for function '%s' (%d)\n",
224 			       func->old_name, ret);
225 			ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
226 			goto err;
227 		}
228 
229 
230 	} else {
231 		list_add_rcu(&func->stack_node, &ops->func_stack);
232 	}
233 
234 	func->patched = true;
235 
236 	return 0;
237 
238 err:
239 	list_del_rcu(&func->stack_node);
240 	list_del(&ops->node);
241 	kfree(ops);
242 	return ret;
243 }
244 
245 static void __klp_unpatch_object(struct klp_object *obj, bool nops_only)
246 {
247 	struct klp_func *func;
248 
249 	klp_for_each_func(obj, func) {
250 		if (nops_only && !func->nop)
251 			continue;
252 
253 		if (func->patched)
254 			klp_unpatch_func(func);
255 	}
256 
257 	if (obj->dynamic || !nops_only)
258 		obj->patched = false;
259 }
260 
261 
262 void klp_unpatch_object(struct klp_object *obj)
263 {
264 	__klp_unpatch_object(obj, false);
265 }
266 
267 int klp_patch_object(struct klp_object *obj)
268 {
269 	struct klp_func *func;
270 	int ret;
271 
272 	if (WARN_ON(obj->patched))
273 		return -EINVAL;
274 
275 	klp_for_each_func(obj, func) {
276 		ret = klp_patch_func(func);
277 		if (ret) {
278 			klp_unpatch_object(obj);
279 			return ret;
280 		}
281 	}
282 	obj->patched = true;
283 
284 	return 0;
285 }
286 
287 static void __klp_unpatch_objects(struct klp_patch *patch, bool nops_only)
288 {
289 	struct klp_object *obj;
290 
291 	klp_for_each_object(patch, obj)
292 		if (obj->patched)
293 			__klp_unpatch_object(obj, nops_only);
294 }
295 
296 void klp_unpatch_objects(struct klp_patch *patch)
297 {
298 	__klp_unpatch_objects(patch, false);
299 }
300 
301 void klp_unpatch_objects_dynamic(struct klp_patch *patch)
302 {
303 	__klp_unpatch_objects(patch, true);
304 }
305