xref: /linux/mm/execmem.c (revision c34e9ab9a612ee8b18273398ef75c207b01f516d) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2002 Richard Henderson
4  * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5  * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6  * Copyright (C) 2024 Mike Rapoport IBM.
7  */
8 
9 #define pr_fmt(fmt) "execmem: " fmt
10 
11 #include <linux/mm.h>
12 #include <linux/mutex.h>
13 #include <linux/vmalloc.h>
14 #include <linux/execmem.h>
15 #include <linux/maple_tree.h>
16 #include <linux/set_memory.h>
17 #include <linux/moduleloader.h>
18 #include <linux/text-patching.h>
19 
20 #include <asm/tlbflush.h>
21 
22 #include "internal.h"
23 
24 static struct execmem_info *execmem_info __ro_after_init;
25 static struct execmem_info default_execmem_info __ro_after_init;
26 
27 #ifdef CONFIG_MMU
28 static void *execmem_vmalloc(struct execmem_range *range, size_t size,
29 			     pgprot_t pgprot, unsigned long vm_flags)
30 {
31 	bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
32 	gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
33 	unsigned int align = range->alignment;
34 	unsigned long start = range->start;
35 	unsigned long end = range->end;
36 	void *p;
37 
38 	if (kasan)
39 		vm_flags |= VM_DEFER_KMEMLEAK;
40 
41 	if (vm_flags & VM_ALLOW_HUGE_VMAP)
42 		align = PMD_SIZE;
43 
44 	p = __vmalloc_node_range(size, align, start, end, gfp_flags,
45 				 pgprot, vm_flags, NUMA_NO_NODE,
46 				 __builtin_return_address(0));
47 	if (!p && range->fallback_start) {
48 		start = range->fallback_start;
49 		end = range->fallback_end;
50 		p = __vmalloc_node_range(size, align, start, end, gfp_flags,
51 					 pgprot, vm_flags, NUMA_NO_NODE,
52 					 __builtin_return_address(0));
53 	}
54 
55 	if (!p) {
56 		pr_warn_ratelimited("unable to allocate memory\n");
57 		return NULL;
58 	}
59 
60 	if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
61 		vfree(p);
62 		return NULL;
63 	}
64 
65 	return p;
66 }
67 
68 struct vm_struct *execmem_vmap(size_t size)
69 {
70 	struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
71 	struct vm_struct *area;
72 
73 	area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
74 				  range->start, range->end, NUMA_NO_NODE,
75 				  GFP_KERNEL, __builtin_return_address(0));
76 	if (!area && range->fallback_start)
77 		area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
78 					  range->fallback_start, range->fallback_end,
79 					  NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0));
80 
81 	return area;
82 }
83 #else
84 static void *execmem_vmalloc(struct execmem_range *range, size_t size,
85 			     pgprot_t pgprot, unsigned long vm_flags)
86 {
87 	return vmalloc(size);
88 }
89 #endif /* CONFIG_MMU */
90 
91 #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
92 struct execmem_cache {
93 	struct mutex mutex;
94 	struct maple_tree busy_areas;
95 	struct maple_tree free_areas;
96 };
97 
98 static struct execmem_cache execmem_cache = {
99 	.mutex = __MUTEX_INITIALIZER(execmem_cache.mutex),
100 	.busy_areas = MTREE_INIT_EXT(busy_areas, MT_FLAGS_LOCK_EXTERN,
101 				     execmem_cache.mutex),
102 	.free_areas = MTREE_INIT_EXT(free_areas, MT_FLAGS_LOCK_EXTERN,
103 				     execmem_cache.mutex),
104 };
105 
106 static inline unsigned long mas_range_len(struct ma_state *mas)
107 {
108 	return mas->last - mas->index + 1;
109 }
110 
111 static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid)
112 {
113 	unsigned int nr = (1 << get_vm_area_page_order(vm));
114 	unsigned int updated = 0;
115 	int err = 0;
116 
117 	for (int i = 0; i < vm->nr_pages; i += nr) {
118 		err = set_direct_map_valid_noflush(vm->pages[i], nr, valid);
119 		if (err)
120 			goto err_restore;
121 		updated += nr;
122 	}
123 
124 	return 0;
125 
126 err_restore:
127 	for (int i = 0; i < updated; i += nr)
128 		set_direct_map_valid_noflush(vm->pages[i], nr, !valid);
129 
130 	return err;
131 }
132 
133 static void execmem_cache_clean(struct work_struct *work)
134 {
135 	struct maple_tree *free_areas = &execmem_cache.free_areas;
136 	struct mutex *mutex = &execmem_cache.mutex;
137 	MA_STATE(mas, free_areas, 0, ULONG_MAX);
138 	void *area;
139 
140 	mutex_lock(mutex);
141 	mas_for_each(&mas, area, ULONG_MAX) {
142 		size_t size = mas_range_len(&mas);
143 
144 		if (IS_ALIGNED(size, PMD_SIZE) &&
145 		    IS_ALIGNED(mas.index, PMD_SIZE)) {
146 			struct vm_struct *vm = find_vm_area(area);
147 
148 			execmem_set_direct_map_valid(vm, true);
149 			mas_store_gfp(&mas, NULL, GFP_KERNEL);
150 			vfree(area);
151 		}
152 	}
153 	mutex_unlock(mutex);
154 }
155 
156 static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean);
157 
158 static int execmem_cache_add(void *ptr, size_t size)
159 {
160 	struct maple_tree *free_areas = &execmem_cache.free_areas;
161 	struct mutex *mutex = &execmem_cache.mutex;
162 	unsigned long addr = (unsigned long)ptr;
163 	MA_STATE(mas, free_areas, addr - 1, addr + 1);
164 	unsigned long lower, upper;
165 	void *area = NULL;
166 	int err;
167 
168 	lower = addr;
169 	upper = addr + size - 1;
170 
171 	mutex_lock(mutex);
172 	area = mas_walk(&mas);
173 	if (area && mas.last == addr - 1)
174 		lower = mas.index;
175 
176 	area = mas_next(&mas, ULONG_MAX);
177 	if (area && mas.index == addr + size)
178 		upper = mas.last;
179 
180 	mas_set_range(&mas, lower, upper);
181 	err = mas_store_gfp(&mas, (void *)lower, GFP_KERNEL);
182 	mutex_unlock(mutex);
183 	if (err)
184 		return err;
185 
186 	return 0;
187 }
188 
189 static bool within_range(struct execmem_range *range, struct ma_state *mas,
190 			 size_t size)
191 {
192 	unsigned long addr = mas->index;
193 
194 	if (addr >= range->start && addr + size < range->end)
195 		return true;
196 
197 	if (range->fallback_start &&
198 	    addr >= range->fallback_start && addr + size < range->fallback_end)
199 		return true;
200 
201 	return false;
202 }
203 
204 static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
205 {
206 	struct maple_tree *free_areas = &execmem_cache.free_areas;
207 	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
208 	MA_STATE(mas_free, free_areas, 0, ULONG_MAX);
209 	MA_STATE(mas_busy, busy_areas, 0, ULONG_MAX);
210 	struct mutex *mutex = &execmem_cache.mutex;
211 	unsigned long addr, last, area_size = 0;
212 	void *area, *ptr = NULL;
213 	int err;
214 
215 	mutex_lock(mutex);
216 	mas_for_each(&mas_free, area, ULONG_MAX) {
217 		area_size = mas_range_len(&mas_free);
218 
219 		if (area_size >= size && within_range(range, &mas_free, size))
220 			break;
221 	}
222 
223 	if (area_size < size)
224 		goto out_unlock;
225 
226 	addr = mas_free.index;
227 	last = mas_free.last;
228 
229 	/* insert allocated size to busy_areas at range [addr, addr + size) */
230 	mas_set_range(&mas_busy, addr, addr + size - 1);
231 	err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL);
232 	if (err)
233 		goto out_unlock;
234 
235 	mas_store_gfp(&mas_free, NULL, GFP_KERNEL);
236 	if (area_size > size) {
237 		void *ptr = (void *)(addr + size);
238 
239 		/*
240 		 * re-insert remaining free size to free_areas at range
241 		 * [addr + size, last]
242 		 */
243 		mas_set_range(&mas_free, addr + size, last);
244 		err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL);
245 		if (err) {
246 			mas_store_gfp(&mas_busy, NULL, GFP_KERNEL);
247 			goto out_unlock;
248 		}
249 	}
250 	ptr = (void *)addr;
251 
252 out_unlock:
253 	mutex_unlock(mutex);
254 	return ptr;
255 }
256 
257 static int execmem_cache_populate(struct execmem_range *range, size_t size)
258 {
259 	unsigned long vm_flags = VM_ALLOW_HUGE_VMAP;
260 	unsigned long start, end;
261 	struct vm_struct *vm;
262 	size_t alloc_size;
263 	int err = -ENOMEM;
264 	void *p;
265 
266 	alloc_size = round_up(size, PMD_SIZE);
267 	p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
268 	if (!p)
269 		return err;
270 
271 	vm = find_vm_area(p);
272 	if (!vm)
273 		goto err_free_mem;
274 
275 	/* fill memory with instructions that will trap */
276 	execmem_fill_trapping_insns(p, alloc_size, /* writable = */ true);
277 
278 	start = (unsigned long)p;
279 	end = start + alloc_size;
280 
281 	vunmap_range(start, end);
282 
283 	err = execmem_set_direct_map_valid(vm, false);
284 	if (err)
285 		goto err_free_mem;
286 
287 	err = vmap_pages_range_noflush(start, end, range->pgprot, vm->pages,
288 				       PMD_SHIFT);
289 	if (err)
290 		goto err_free_mem;
291 
292 	err = execmem_cache_add(p, alloc_size);
293 	if (err)
294 		goto err_free_mem;
295 
296 	return 0;
297 
298 err_free_mem:
299 	vfree(p);
300 	return err;
301 }
302 
303 static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
304 {
305 	void *p;
306 	int err;
307 
308 	p = __execmem_cache_alloc(range, size);
309 	if (p)
310 		return p;
311 
312 	err = execmem_cache_populate(range, size);
313 	if (err)
314 		return NULL;
315 
316 	return __execmem_cache_alloc(range, size);
317 }
318 
319 static bool execmem_cache_free(void *ptr)
320 {
321 	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
322 	struct mutex *mutex = &execmem_cache.mutex;
323 	unsigned long addr = (unsigned long)ptr;
324 	MA_STATE(mas, busy_areas, addr, addr);
325 	size_t size;
326 	void *area;
327 
328 	mutex_lock(mutex);
329 	area = mas_walk(&mas);
330 	if (!area) {
331 		mutex_unlock(mutex);
332 		return false;
333 	}
334 	size = mas_range_len(&mas);
335 
336 	mas_store_gfp(&mas, NULL, GFP_KERNEL);
337 	mutex_unlock(mutex);
338 
339 	execmem_fill_trapping_insns(ptr, size, /* writable = */ false);
340 
341 	execmem_cache_add(ptr, size);
342 
343 	schedule_work(&execmem_cache_clean_work);
344 
345 	return true;
346 }
347 #else /* CONFIG_ARCH_HAS_EXECMEM_ROX */
348 static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
349 {
350 	return NULL;
351 }
352 
353 static bool execmem_cache_free(void *ptr)
354 {
355 	return false;
356 }
357 #endif /* CONFIG_ARCH_HAS_EXECMEM_ROX */
358 
359 void *execmem_alloc(enum execmem_type type, size_t size)
360 {
361 	struct execmem_range *range = &execmem_info->ranges[type];
362 	bool use_cache = range->flags & EXECMEM_ROX_CACHE;
363 	unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
364 	pgprot_t pgprot = range->pgprot;
365 	void *p;
366 
367 	if (use_cache)
368 		p = execmem_cache_alloc(range, size);
369 	else
370 		p = execmem_vmalloc(range, size, pgprot, vm_flags);
371 
372 	return kasan_reset_tag(p);
373 }
374 
375 void execmem_free(void *ptr)
376 {
377 	/*
378 	 * This memory may be RO, and freeing RO memory in an interrupt is not
379 	 * supported by vmalloc.
380 	 */
381 	WARN_ON(in_interrupt());
382 
383 	if (!execmem_cache_free(ptr))
384 		vfree(ptr);
385 }
386 
387 void *execmem_update_copy(void *dst, const void *src, size_t size)
388 {
389 	return text_poke_copy(dst, src, size);
390 }
391 
392 bool execmem_is_rox(enum execmem_type type)
393 {
394 	return !!(execmem_info->ranges[type].flags & EXECMEM_ROX_CACHE);
395 }
396 
397 static bool execmem_validate(struct execmem_info *info)
398 {
399 	struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
400 
401 	if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
402 		pr_crit("Invalid parameters for execmem allocator, module loading will fail");
403 		return false;
404 	}
405 
406 	if (!IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX)) {
407 		for (int i = EXECMEM_DEFAULT; i < EXECMEM_TYPE_MAX; i++) {
408 			r = &info->ranges[i];
409 
410 			if (r->flags & EXECMEM_ROX_CACHE) {
411 				pr_warn_once("ROX cache is not supported\n");
412 				r->flags &= ~EXECMEM_ROX_CACHE;
413 			}
414 		}
415 	}
416 
417 	return true;
418 }
419 
420 static void execmem_init_missing(struct execmem_info *info)
421 {
422 	struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
423 
424 	for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
425 		struct execmem_range *r = &info->ranges[i];
426 
427 		if (!r->start) {
428 			if (i == EXECMEM_MODULE_DATA)
429 				r->pgprot = PAGE_KERNEL;
430 			else
431 				r->pgprot = default_range->pgprot;
432 			r->alignment = default_range->alignment;
433 			r->start = default_range->start;
434 			r->end = default_range->end;
435 			r->flags = default_range->flags;
436 			r->fallback_start = default_range->fallback_start;
437 			r->fallback_end = default_range->fallback_end;
438 		}
439 	}
440 }
441 
442 struct execmem_info * __weak execmem_arch_setup(void)
443 {
444 	return NULL;
445 }
446 
447 static void __init __execmem_init(void)
448 {
449 	struct execmem_info *info = execmem_arch_setup();
450 
451 	if (!info) {
452 		info = execmem_info = &default_execmem_info;
453 		info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
454 		info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
455 		info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
456 		info->ranges[EXECMEM_DEFAULT].alignment = 1;
457 	}
458 
459 	if (!execmem_validate(info))
460 		return;
461 
462 	execmem_init_missing(info);
463 
464 	execmem_info = info;
465 }
466 
467 #ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
468 static int __init execmem_late_init(void)
469 {
470 	__execmem_init();
471 	return 0;
472 }
473 core_initcall(execmem_late_init);
474 #else
475 void __init execmem_init(void)
476 {
477 	__execmem_init();
478 }
479 #endif
480