xref: /linux/kernel/livepatch/patch.c (revision 69bfec7548f4c1595bac0e3ddfc0458a5af31f4c) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * patch.c - livepatch patching functions
4  *
5  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
6  * Copyright (C) 2014 SUSE
7  * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
8  */
9 
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 
12 #include <linux/livepatch.h>
13 #include <linux/list.h>
14 #include <linux/ftrace.h>
15 #include <linux/rculist.h>
16 #include <linux/slab.h>
17 #include <linux/bug.h>
18 #include <linux/printk.h>
19 #include "core.h"
20 #include "patch.h"
21 #include "transition.h"
22 
23 static LIST_HEAD(klp_ops);
24 
25 struct klp_ops *klp_find_ops(void *old_func)
26 {
27 	struct klp_ops *ops;
28 	struct klp_func *func;
29 
30 	list_for_each_entry(ops, &klp_ops, node) {
31 		func = list_first_entry(&ops->func_stack, struct klp_func,
32 					stack_node);
33 		if (func->old_func == old_func)
34 			return ops;
35 	}
36 
37 	return NULL;
38 }
39 
40 static void notrace klp_ftrace_handler(unsigned long ip,
41 				       unsigned long parent_ip,
42 				       struct ftrace_ops *fops,
43 				       struct ftrace_regs *fregs)
44 {
45 	struct klp_ops *ops;
46 	struct klp_func *func;
47 	int patch_state;
48 	int bit;
49 
50 	ops = container_of(fops, struct klp_ops, fops);
51 
52 	/*
53 	 * The ftrace_test_recursion_trylock() will disable preemption,
54 	 * which is required for the variant of synchronize_rcu() that is
55 	 * used to allow patching functions where RCU is not watching.
56 	 * See klp_synchronize_transition() for more details.
57 	 */
58 	bit = ftrace_test_recursion_trylock(ip, parent_ip);
59 	if (WARN_ON_ONCE(bit < 0))
60 		return;
61 
62 	func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
63 				      stack_node);
64 
65 	/*
66 	 * func should never be NULL because preemption should be disabled here
67 	 * and unregister_ftrace_function() does the equivalent of a
68 	 * synchronize_rcu() before the func_stack removal.
69 	 */
70 	if (WARN_ON_ONCE(!func))
71 		goto unlock;
72 
73 	/*
74 	 * In the enable path, enforce the order of the ops->func_stack and
75 	 * func->transition reads.  The corresponding write barrier is in
76 	 * __klp_enable_patch().
77 	 *
78 	 * (Note that this barrier technically isn't needed in the disable
79 	 * path.  In the rare case where klp_update_patch_state() runs before
80 	 * this handler, its TIF_PATCH_PENDING read and this func->transition
81 	 * read need to be ordered.  But klp_update_patch_state() already
82 	 * enforces that.)
83 	 */
84 	smp_rmb();
85 
86 	if (unlikely(func->transition)) {
87 
88 		/*
89 		 * Enforce the order of the func->transition and
90 		 * current->patch_state reads.  Otherwise we could read an
91 		 * out-of-date task state and pick the wrong function.  The
92 		 * corresponding write barrier is in klp_init_transition().
93 		 */
94 		smp_rmb();
95 
96 		patch_state = current->patch_state;
97 
98 		WARN_ON_ONCE(patch_state == KLP_UNDEFINED);
99 
100 		if (patch_state == KLP_UNPATCHED) {
101 			/*
102 			 * Use the previously patched version of the function.
103 			 * If no previous patches exist, continue with the
104 			 * original function.
105 			 */
106 			func = list_entry_rcu(func->stack_node.next,
107 					      struct klp_func, stack_node);
108 
109 			if (&func->stack_node == &ops->func_stack)
110 				goto unlock;
111 		}
112 	}
113 
114 	/*
115 	 * NOPs are used to replace existing patches with original code.
116 	 * Do nothing! Setting pc would cause an infinite loop.
117 	 */
118 	if (func->nop)
119 		goto unlock;
120 
121 	ftrace_regs_set_instruction_pointer(fregs, (unsigned long)func->new_func);
122 
123 unlock:
124 	ftrace_test_recursion_unlock(bit);
125 }
126 
127 static void klp_unpatch_func(struct klp_func *func)
128 {
129 	struct klp_ops *ops;
130 
131 	if (WARN_ON(!func->patched))
132 		return;
133 	if (WARN_ON(!func->old_func))
134 		return;
135 
136 	ops = klp_find_ops(func->old_func);
137 	if (WARN_ON(!ops))
138 		return;
139 
140 	if (list_is_singular(&ops->func_stack)) {
141 		unsigned long ftrace_loc;
142 
143 		ftrace_loc = ftrace_location((unsigned long)func->old_func);
144 		if (WARN_ON(!ftrace_loc))
145 			return;
146 
147 		WARN_ON(unregister_ftrace_function(&ops->fops));
148 		WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
149 
150 		list_del_rcu(&func->stack_node);
151 		list_del(&ops->node);
152 		kfree(ops);
153 	} else {
154 		list_del_rcu(&func->stack_node);
155 	}
156 
157 	func->patched = false;
158 }
159 
160 static int klp_patch_func(struct klp_func *func)
161 {
162 	struct klp_ops *ops;
163 	int ret;
164 
165 	if (WARN_ON(!func->old_func))
166 		return -EINVAL;
167 
168 	if (WARN_ON(func->patched))
169 		return -EINVAL;
170 
171 	ops = klp_find_ops(func->old_func);
172 	if (!ops) {
173 		unsigned long ftrace_loc;
174 
175 		ftrace_loc = ftrace_location((unsigned long)func->old_func);
176 		if (!ftrace_loc) {
177 			pr_err("failed to find location for function '%s'\n",
178 				func->old_name);
179 			return -EINVAL;
180 		}
181 
182 		ops = kzalloc(sizeof(*ops), GFP_KERNEL);
183 		if (!ops)
184 			return -ENOMEM;
185 
186 		ops->fops.func = klp_ftrace_handler;
187 		ops->fops.flags = FTRACE_OPS_FL_DYNAMIC |
188 #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
189 				  FTRACE_OPS_FL_SAVE_REGS |
190 #endif
191 				  FTRACE_OPS_FL_IPMODIFY |
192 				  FTRACE_OPS_FL_PERMANENT;
193 
194 		list_add(&ops->node, &klp_ops);
195 
196 		INIT_LIST_HEAD(&ops->func_stack);
197 		list_add_rcu(&func->stack_node, &ops->func_stack);
198 
199 		ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
200 		if (ret) {
201 			pr_err("failed to set ftrace filter for function '%s' (%d)\n",
202 			       func->old_name, ret);
203 			goto err;
204 		}
205 
206 		ret = register_ftrace_function(&ops->fops);
207 		if (ret) {
208 			pr_err("failed to register ftrace handler for function '%s' (%d)\n",
209 			       func->old_name, ret);
210 			ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
211 			goto err;
212 		}
213 
214 
215 	} else {
216 		list_add_rcu(&func->stack_node, &ops->func_stack);
217 	}
218 
219 	func->patched = true;
220 
221 	return 0;
222 
223 err:
224 	list_del_rcu(&func->stack_node);
225 	list_del(&ops->node);
226 	kfree(ops);
227 	return ret;
228 }
229 
230 static void __klp_unpatch_object(struct klp_object *obj, bool nops_only)
231 {
232 	struct klp_func *func;
233 
234 	klp_for_each_func(obj, func) {
235 		if (nops_only && !func->nop)
236 			continue;
237 
238 		if (func->patched)
239 			klp_unpatch_func(func);
240 	}
241 
242 	if (obj->dynamic || !nops_only)
243 		obj->patched = false;
244 }
245 
246 
247 void klp_unpatch_object(struct klp_object *obj)
248 {
249 	__klp_unpatch_object(obj, false);
250 }
251 
252 int klp_patch_object(struct klp_object *obj)
253 {
254 	struct klp_func *func;
255 	int ret;
256 
257 	if (WARN_ON(obj->patched))
258 		return -EINVAL;
259 
260 	klp_for_each_func(obj, func) {
261 		ret = klp_patch_func(func);
262 		if (ret) {
263 			klp_unpatch_object(obj);
264 			return ret;
265 		}
266 	}
267 	obj->patched = true;
268 
269 	return 0;
270 }
271 
272 static void __klp_unpatch_objects(struct klp_patch *patch, bool nops_only)
273 {
274 	struct klp_object *obj;
275 
276 	klp_for_each_object(patch, obj)
277 		if (obj->patched)
278 			__klp_unpatch_object(obj, nops_only);
279 }
280 
281 void klp_unpatch_objects(struct klp_patch *patch)
282 {
283 	__klp_unpatch_objects(patch, false);
284 }
285 
286 void klp_unpatch_objects_dynamic(struct klp_patch *patch)
287 {
288 	__klp_unpatch_objects(patch, true);
289 }
290