xref: /linux/mm/execmem.c (revision 1b78070aaef63512688aebfbc82365ef9d6660f1) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2002 Richard Henderson
4  * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5  * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6  * Copyright (C) 2024 Mike Rapoport IBM.
7  */
8 
9 #define pr_fmt(fmt) "execmem: " fmt
10 
11 #include <linux/mm.h>
12 #include <linux/mutex.h>
13 #include <linux/vmalloc.h>
14 #include <linux/execmem.h>
15 #include <linux/maple_tree.h>
16 #include <linux/set_memory.h>
17 #include <linux/moduleloader.h>
18 #include <linux/text-patching.h>
19 
20 #include <asm/tlbflush.h>
21 
22 #include "internal.h"
23 #include "vmalloc.h"
24 
25 static struct execmem_info *execmem_info __ro_after_init;
26 static struct execmem_info default_execmem_info __ro_after_init;
27 
28 #ifdef CONFIG_MMU
29 static void *execmem_vmalloc(struct execmem_range *range, size_t size,
30 			     pgprot_t pgprot, unsigned long vm_flags)
31 {
32 	bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
33 	gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
34 	unsigned int align = range->alignment;
35 	unsigned long start = range->start;
36 	unsigned long end = range->end;
37 	void *p;
38 
39 	if (kasan)
40 		vm_flags |= VM_DEFER_KMEMLEAK;
41 
42 	p = __vmalloc_node_range(size, align, start, end, gfp_flags,
43 				 pgprot, vm_flags, NUMA_NO_NODE,
44 				 __builtin_return_address(0));
45 	if (!p && range->fallback_start) {
46 		start = range->fallback_start;
47 		end = range->fallback_end;
48 		p = __vmalloc_node_range(size, align, start, end, gfp_flags,
49 					 pgprot, vm_flags, NUMA_NO_NODE,
50 					 __builtin_return_address(0));
51 	}
52 
53 	if (!p) {
54 		pr_warn_ratelimited("unable to allocate memory\n");
55 		return NULL;
56 	}
57 
58 	if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
59 		vfree(p);
60 		return NULL;
61 	}
62 
63 	return p;
64 }
65 
66 struct vm_struct *execmem_vmap(size_t size)
67 {
68 	struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
69 	struct vm_struct *area;
70 
71 	area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
72 				  range->start, range->end, NUMA_NO_NODE,
73 				  GFP_KERNEL, __builtin_return_address(0));
74 	if (!area && range->fallback_start)
75 		area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
76 					  range->fallback_start, range->fallback_end,
77 					  NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0));
78 
79 	return area;
80 }
81 #else
82 static void *execmem_vmalloc(struct execmem_range *range, size_t size,
83 			     pgprot_t pgprot, unsigned long vm_flags)
84 {
85 	return vmalloc(size);
86 }
87 #endif /* CONFIG_MMU */
88 
89 #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
90 struct execmem_cache {
91 	struct mutex mutex;
92 	struct maple_tree busy_areas;
93 	struct maple_tree free_areas;
94 	unsigned int pending_free_cnt;	/* protected by mutex */
95 };
96 
97 /* delay to schedule asynchronous free if fast path free fails */
98 #define FREE_DELAY	(msecs_to_jiffies(10))
99 
100 /* mark entries in busy_areas that should be freed asynchronously */
101 #define PENDING_FREE_MASK	(1 << (PAGE_SHIFT - 1))
102 
103 static struct execmem_cache execmem_cache = {
104 	.mutex = __MUTEX_INITIALIZER(execmem_cache.mutex),
105 	.busy_areas = MTREE_INIT_EXT(busy_areas, MT_FLAGS_LOCK_EXTERN,
106 				     execmem_cache.mutex),
107 	.free_areas = MTREE_INIT_EXT(free_areas, MT_FLAGS_LOCK_EXTERN,
108 				     execmem_cache.mutex),
109 };
110 
111 static inline unsigned long mas_range_len(struct ma_state *mas)
112 {
113 	return mas->last - mas->index + 1;
114 }
115 
116 static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid)
117 {
118 	unsigned int nr = (1 << get_vm_area_page_order(vm));
119 	unsigned int updated = 0;
120 	int err = 0;
121 
122 	for (int i = 0; i < vm->nr_pages; i += nr) {
123 		err = set_direct_map_valid_noflush(vm->pages[i], nr, valid);
124 		if (err)
125 			goto err_restore;
126 		updated += nr;
127 	}
128 
129 	return 0;
130 
131 err_restore:
132 	for (int i = 0; i < updated; i += nr)
133 		set_direct_map_valid_noflush(vm->pages[i], nr, !valid);
134 
135 	return err;
136 }
137 
138 static int execmem_force_rw(void *ptr, size_t size)
139 {
140 	unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
141 	unsigned long addr = (unsigned long)ptr;
142 	int ret;
143 
144 	ret = set_memory_nx(addr, nr);
145 	if (ret)
146 		return ret;
147 
148 	return set_memory_rw(addr, nr);
149 }
150 
151 int execmem_restore_rox(void *ptr, size_t size)
152 {
153 	unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
154 	unsigned long addr = (unsigned long)ptr;
155 
156 	return set_memory_rox(addr, nr);
157 }
158 
159 static void execmem_cache_clean(struct work_struct *work)
160 {
161 	struct maple_tree *free_areas = &execmem_cache.free_areas;
162 	struct mutex *mutex = &execmem_cache.mutex;
163 	MA_STATE(mas, free_areas, 0, ULONG_MAX);
164 	void *area;
165 
166 	mutex_lock(mutex);
167 	mas_for_each(&mas, area, ULONG_MAX) {
168 		size_t size = mas_range_len(&mas);
169 
170 		if (IS_ALIGNED(size, PMD_SIZE) &&
171 		    IS_ALIGNED(mas.index, PMD_SIZE)) {
172 			struct vm_struct *vm = find_vm_area(area);
173 
174 			execmem_set_direct_map_valid(vm, true);
175 			mas_store_gfp(&mas, NULL, GFP_KERNEL);
176 			vfree(area);
177 		}
178 	}
179 	mutex_unlock(mutex);
180 }
181 
182 static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean);
183 
184 static int execmem_cache_add_locked(void *ptr, size_t size, gfp_t gfp_mask)
185 {
186 	struct maple_tree *free_areas = &execmem_cache.free_areas;
187 	unsigned long addr = (unsigned long)ptr;
188 	MA_STATE(mas, free_areas, addr - 1, addr + 1);
189 	unsigned long lower, upper;
190 	void *area = NULL;
191 
192 	lower = addr;
193 	upper = addr + size - 1;
194 
195 	area = mas_walk(&mas);
196 	if (area && mas.last == addr - 1)
197 		lower = mas.index;
198 
199 	area = mas_next(&mas, ULONG_MAX);
200 	if (area && mas.index == addr + size)
201 		upper = mas.last;
202 
203 	mas_set_range(&mas, lower, upper);
204 	return mas_store_gfp(&mas, (void *)lower, gfp_mask);
205 }
206 
207 static bool within_range(struct execmem_range *range, struct ma_state *mas,
208 			 size_t size)
209 {
210 	unsigned long addr = mas->index;
211 
212 	if (addr >= range->start && addr + size < range->end)
213 		return true;
214 
215 	if (range->fallback_start &&
216 	    addr >= range->fallback_start && addr + size < range->fallback_end)
217 		return true;
218 
219 	return false;
220 }
221 
222 static void *execmem_cache_alloc_locked(struct execmem_range *range, size_t size)
223 {
224 	struct maple_tree *free_areas = &execmem_cache.free_areas;
225 	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
226 	MA_STATE(mas_free, free_areas, 0, ULONG_MAX);
227 	MA_STATE(mas_busy, busy_areas, 0, ULONG_MAX);
228 	unsigned long addr, last, area_size = 0;
229 	void *area, *ptr = NULL;
230 	int err;
231 
232 	mas_for_each(&mas_free, area, ULONG_MAX) {
233 		area_size = mas_range_len(&mas_free);
234 
235 		if (area_size >= size && within_range(range, &mas_free, size))
236 			break;
237 	}
238 
239 	if (area_size < size)
240 		return NULL;
241 
242 	addr = mas_free.index;
243 	last = mas_free.last;
244 
245 	/* insert allocated size to busy_areas at range [addr, addr + size) */
246 	mas_set_range(&mas_busy, addr, addr + size - 1);
247 	err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL);
248 	if (err)
249 		return NULL;
250 
251 	mas_store_gfp(&mas_free, NULL, GFP_KERNEL);
252 	if (area_size > size) {
253 		void *ptr = (void *)(addr + size);
254 
255 		/*
256 		 * re-insert remaining free size to free_areas at range
257 		 * [addr + size, last]
258 		 */
259 		mas_set_range(&mas_free, addr + size, last);
260 		err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL);
261 		if (err) {
262 			mas_store_gfp(&mas_busy, NULL, GFP_KERNEL);
263 			return NULL;
264 		}
265 	}
266 	ptr = (void *)addr;
267 
268 	return ptr;
269 }
270 
271 static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
272 {
273 	guard(mutex)(&execmem_cache.mutex);
274 
275 	return execmem_cache_alloc_locked(range, size);
276 }
277 
278 static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size)
279 {
280 	unsigned long vm_flags = VM_ALLOW_HUGE_VMAP;
281 	struct mutex *mutex = &execmem_cache.mutex;
282 	struct vm_struct *vm;
283 	size_t alloc_size;
284 	int err = -ENOMEM;
285 	void *p;
286 
287 	alloc_size = round_up(size, PMD_SIZE);
288 	p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
289 	if (!p) {
290 		alloc_size = size;
291 		p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
292 	}
293 
294 	if (!p)
295 		return NULL;
296 
297 	vm = find_vm_area(p);
298 	if (!vm)
299 		goto err_free_mem;
300 
301 	/* fill memory with instructions that will trap */
302 	execmem_fill_trapping_insns(p, alloc_size);
303 
304 	err = set_memory_rox((unsigned long)p, vm->nr_pages);
305 	if (err)
306 		goto err_free_mem;
307 
308 	/*
309 	 * New memory blocks must be allocated and added to the cache
310 	 * as an atomic operation, otherwise they may be consumed
311 	 * by a parallel call to the execmem_cache_alloc function.
312 	 */
313 	mutex_lock(mutex);
314 	err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL);
315 	if (err)
316 		goto err_reset_direct_map;
317 
318 	p = execmem_cache_alloc_locked(range, size);
319 
320 	mutex_unlock(mutex);
321 
322 	return p;
323 
324 err_reset_direct_map:
325 	mutex_unlock(mutex);
326 	execmem_set_direct_map_valid(vm, true);
327 err_free_mem:
328 	vfree(p);
329 	return NULL;
330 }
331 
332 static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
333 {
334 	void *p;
335 
336 	p = __execmem_cache_alloc(range, size);
337 	if (p)
338 		return p;
339 
340 	return execmem_cache_populate_alloc(range, size);
341 }
342 
343 static inline bool is_pending_free(void *ptr)
344 {
345 	return ((unsigned long)ptr & PENDING_FREE_MASK);
346 }
347 
348 static inline void *pending_free_set(void *ptr)
349 {
350 	return (void *)((unsigned long)ptr | PENDING_FREE_MASK);
351 }
352 
353 static inline void *pending_free_clear(void *ptr)
354 {
355 	return (void *)((unsigned long)ptr & ~PENDING_FREE_MASK);
356 }
357 
358 static int __execmem_cache_free(struct ma_state *mas, void *ptr, gfp_t gfp_mask)
359 {
360 	size_t size = mas_range_len(mas);
361 	int err;
362 
363 	err = execmem_force_rw(ptr, size);
364 	if (err)
365 		return err;
366 
367 	execmem_fill_trapping_insns(ptr, size);
368 	execmem_restore_rox(ptr, size);
369 
370 	err = execmem_cache_add_locked(ptr, size, gfp_mask);
371 	if (err)
372 		return err;
373 
374 	mas_store_gfp(mas, NULL, gfp_mask);
375 	return 0;
376 }
377 
378 static void execmem_cache_free_slow(struct work_struct *work);
379 static DECLARE_DELAYED_WORK(execmem_cache_free_work, execmem_cache_free_slow);
380 
381 static void execmem_cache_free_slow(struct work_struct *work)
382 {
383 	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
384 	MA_STATE(mas, busy_areas, 0, ULONG_MAX);
385 	void *area;
386 
387 	guard(mutex)(&execmem_cache.mutex);
388 
389 	if (!execmem_cache.pending_free_cnt)
390 		return;
391 
392 	mas_for_each(&mas, area, ULONG_MAX) {
393 		if (!is_pending_free(area))
394 			continue;
395 
396 		area = pending_free_clear(area);
397 		if (__execmem_cache_free(&mas, area, GFP_KERNEL))
398 			continue;
399 
400 		execmem_cache.pending_free_cnt--;
401 	}
402 
403 	if (execmem_cache.pending_free_cnt)
404 		schedule_delayed_work(&execmem_cache_free_work, FREE_DELAY);
405 	else
406 		schedule_work(&execmem_cache_clean_work);
407 }
408 
409 static bool execmem_cache_free(void *ptr)
410 {
411 	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
412 	unsigned long addr = (unsigned long)ptr;
413 	MA_STATE(mas, busy_areas, addr, addr);
414 	void *area;
415 	int err;
416 
417 	guard(mutex)(&execmem_cache.mutex);
418 
419 	area = mas_walk(&mas);
420 	if (!area)
421 		return false;
422 
423 	err = __execmem_cache_free(&mas, area, GFP_KERNEL | __GFP_NORETRY);
424 	if (err) {
425 		/*
426 		 * mas points to exact slot we've got the area from, nothing
427 		 * else can modify the tree because of the mutex, so there
428 		 * won't be any allocations in mas_store_gfp() and it will just
429 		 * change the pointer.
430 		 */
431 		area = pending_free_set(area);
432 		mas_store_gfp(&mas, area, GFP_KERNEL);
433 		execmem_cache.pending_free_cnt++;
434 		schedule_delayed_work(&execmem_cache_free_work, FREE_DELAY);
435 		return true;
436 	}
437 
438 	schedule_work(&execmem_cache_clean_work);
439 
440 	return true;
441 }
442 
443 #else /* CONFIG_ARCH_HAS_EXECMEM_ROX */
444 /*
445  * when ROX cache is not used the permissions defined by architectures for
446  * execmem ranges that are updated before use (e.g. EXECMEM_MODULE_TEXT) must
447  * be writable anyway
448  */
449 static inline int execmem_force_rw(void *ptr, size_t size)
450 {
451 	return 0;
452 }
453 
454 static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
455 {
456 	return NULL;
457 }
458 
459 static bool execmem_cache_free(void *ptr)
460 {
461 	return false;
462 }
463 #endif /* CONFIG_ARCH_HAS_EXECMEM_ROX */
464 
465 void *execmem_alloc(enum execmem_type type, size_t size)
466 {
467 	struct execmem_range *range = &execmem_info->ranges[type];
468 	bool use_cache = range->flags & EXECMEM_ROX_CACHE;
469 	unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
470 	pgprot_t pgprot = range->pgprot;
471 	void *p = NULL;
472 
473 	size = PAGE_ALIGN(size);
474 
475 	if (use_cache)
476 		p = execmem_cache_alloc(range, size);
477 	else
478 		p = execmem_vmalloc(range, size, pgprot, vm_flags);
479 
480 	return kasan_reset_tag(p);
481 }
482 
483 void *execmem_alloc_rw(enum execmem_type type, size_t size)
484 {
485 	void *p __free(execmem) = execmem_alloc(type, size);
486 	int err;
487 
488 	if (!p)
489 		return NULL;
490 
491 	err = execmem_force_rw(p, size);
492 	if (err)
493 		return NULL;
494 
495 	return no_free_ptr(p);
496 }
497 
498 void execmem_free(void *ptr)
499 {
500 	/*
501 	 * This memory may be RO, and freeing RO memory in an interrupt is not
502 	 * supported by vmalloc.
503 	 */
504 	WARN_ON(in_interrupt());
505 
506 	if (!execmem_cache_free(ptr))
507 		vfree(ptr);
508 }
509 
510 bool execmem_is_rox(enum execmem_type type)
511 {
512 	return !!(execmem_info->ranges[type].flags & EXECMEM_ROX_CACHE);
513 }
514 
515 static bool execmem_validate(struct execmem_info *info)
516 {
517 	struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
518 
519 	if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
520 		pr_crit("Invalid parameters for execmem allocator, module loading will fail");
521 		return false;
522 	}
523 
524 	if (!IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX)) {
525 		for (int i = EXECMEM_DEFAULT; i < EXECMEM_TYPE_MAX; i++) {
526 			r = &info->ranges[i];
527 
528 			if (r->flags & EXECMEM_ROX_CACHE) {
529 				pr_warn_once("ROX cache is not supported\n");
530 				r->flags &= ~EXECMEM_ROX_CACHE;
531 			}
532 		}
533 	}
534 
535 	return true;
536 }
537 
538 static void execmem_init_missing(struct execmem_info *info)
539 {
540 	struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
541 
542 	for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
543 		struct execmem_range *r = &info->ranges[i];
544 
545 		if (!r->start) {
546 			if (i == EXECMEM_MODULE_DATA)
547 				r->pgprot = PAGE_KERNEL;
548 			else
549 				r->pgprot = default_range->pgprot;
550 			r->alignment = default_range->alignment;
551 			r->start = default_range->start;
552 			r->end = default_range->end;
553 			r->flags = default_range->flags;
554 			r->fallback_start = default_range->fallback_start;
555 			r->fallback_end = default_range->fallback_end;
556 		}
557 	}
558 }
559 
560 struct execmem_info * __weak execmem_arch_setup(void)
561 {
562 	return NULL;
563 }
564 
565 static void __init __execmem_init(void)
566 {
567 	struct execmem_info *info = execmem_arch_setup();
568 
569 	if (!info) {
570 		info = execmem_info = &default_execmem_info;
571 		info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
572 		info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
573 		info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
574 		info->ranges[EXECMEM_DEFAULT].alignment = 1;
575 	}
576 
577 	if (!execmem_validate(info))
578 		return;
579 
580 	execmem_init_missing(info);
581 
582 	execmem_info = info;
583 }
584 
585 #ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
586 static int __init execmem_late_init(void)
587 {
588 	__execmem_init();
589 	return 0;
590 }
591 core_initcall(execmem_late_init);
592 #else
593 void __init execmem_init(void)
594 {
595 	__execmem_init();
596 }
597 #endif
598