xref: /linux/mm/execmem.c (revision cb2e1c2136f71618142557ceca3a8802e87a44cd) 
1  // SPDX-License-Identifier: GPL-2.0
2  /*
3   * Copyright (C) 2002 Richard Henderson
4   * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5   * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6   * Copyright (C) 2024 Mike Rapoport IBM.
7   */
8  
9  #define pr_fmt(fmt) "execmem: " fmt
10  
11  #include <linux/mm.h>
12  #include <linux/mutex.h>
13  #include <linux/vmalloc.h>
14  #include <linux/execmem.h>
15  #include <linux/maple_tree.h>
16  #include <linux/set_memory.h>
17  #include <linux/moduleloader.h>
18  #include <linux/text-patching.h>
19  
20  #include <asm/tlbflush.h>
21  
22  #include "internal.h"
23  
24  static struct execmem_info *execmem_info __ro_after_init;
25  static struct execmem_info default_execmem_info __ro_after_init;
26  
27  #ifdef CONFIG_MMU
28  static void *execmem_vmalloc(struct execmem_range *range, size_t size,
29  			     pgprot_t pgprot, unsigned long vm_flags)
30  {
31  	bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
32  	gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
33  	unsigned int align = range->alignment;
34  	unsigned long start = range->start;
35  	unsigned long end = range->end;
36  	void *p;
37  
38  	if (kasan)
39  		vm_flags |= VM_DEFER_KMEMLEAK;
40  
41  	if (vm_flags & VM_ALLOW_HUGE_VMAP)
42  		align = PMD_SIZE;
43  
44  	p = __vmalloc_node_range(size, align, start, end, gfp_flags,
45  				 pgprot, vm_flags, NUMA_NO_NODE,
46  				 __builtin_return_address(0));
47  	if (!p && range->fallback_start) {
48  		start = range->fallback_start;
49  		end = range->fallback_end;
50  		p = __vmalloc_node_range(size, align, start, end, gfp_flags,
51  					 pgprot, vm_flags, NUMA_NO_NODE,
52  					 __builtin_return_address(0));
53  	}
54  
55  	if (!p) {
56  		pr_warn_ratelimited("unable to allocate memory\n");
57  		return NULL;
58  	}
59  
60  	if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
61  		vfree(p);
62  		return NULL;
63  	}
64  
65  	return p;
66  }
67  
68  struct vm_struct *execmem_vmap(size_t size)
69  {
70  	struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
71  	struct vm_struct *area;
72  
73  	area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
74  				  range->start, range->end, NUMA_NO_NODE,
75  				  GFP_KERNEL, __builtin_return_address(0));
76  	if (!area && range->fallback_start)
77  		area = __get_vm_area_node(size, range->alignment, PAGE_SHIFT, VM_ALLOC,
78  					  range->fallback_start, range->fallback_end,
79  					  NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0));
80  
81  	return area;
82  }
83  #else
84  static void *execmem_vmalloc(struct execmem_range *range, size_t size,
85  			     pgprot_t pgprot, unsigned long vm_flags)
86  {
87  	return vmalloc(size);
88  }
89  #endif /* CONFIG_MMU */
90  
91  #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
92  struct execmem_cache {
93  	struct mutex mutex;
94  	struct maple_tree busy_areas;
95  	struct maple_tree free_areas;
96  };
97  
98  static struct execmem_cache execmem_cache = {
99  	.mutex = __MUTEX_INITIALIZER(execmem_cache.mutex),
100  	.busy_areas = MTREE_INIT_EXT(busy_areas, MT_FLAGS_LOCK_EXTERN,
101  				     execmem_cache.mutex),
102  	.free_areas = MTREE_INIT_EXT(free_areas, MT_FLAGS_LOCK_EXTERN,
103  				     execmem_cache.mutex),
104  };
105  
106  static inline unsigned long mas_range_len(struct ma_state *mas)
107  {
108  	return mas->last - mas->index + 1;
109  }
110  
111  static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid)
112  {
113  	unsigned int nr = (1 << get_vm_area_page_order(vm));
114  	unsigned int updated = 0;
115  	int err = 0;
116  
117  	for (int i = 0; i < vm->nr_pages; i += nr) {
118  		err = set_direct_map_valid_noflush(vm->pages[i], nr, valid);
119  		if (err)
120  			goto err_restore;
121  		updated += nr;
122  	}
123  
124  	return 0;
125  
126  err_restore:
127  	for (int i = 0; i < updated; i += nr)
128  		set_direct_map_valid_noflush(vm->pages[i], nr, !valid);
129  
130  	return err;
131  }
132  
133  static void execmem_cache_clean(struct work_struct *work)
134  {
135  	struct maple_tree *free_areas = &execmem_cache.free_areas;
136  	struct mutex *mutex = &execmem_cache.mutex;
137  	MA_STATE(mas, free_areas, 0, ULONG_MAX);
138  	void *area;
139  
140  	mutex_lock(mutex);
141  	mas_for_each(&mas, area, ULONG_MAX) {
142  		size_t size = mas_range_len(&mas);
143  
144  		if (IS_ALIGNED(size, PMD_SIZE) &&
145  		    IS_ALIGNED(mas.index, PMD_SIZE)) {
146  			struct vm_struct *vm = find_vm_area(area);
147  
148  			execmem_set_direct_map_valid(vm, true);
149  			mas_store_gfp(&mas, NULL, GFP_KERNEL);
150  			vfree(area);
151  		}
152  	}
153  	mutex_unlock(mutex);
154  }
155  
156  static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean);
157  
158  static int execmem_cache_add(void *ptr, size_t size)
159  {
160  	struct maple_tree *free_areas = &execmem_cache.free_areas;
161  	struct mutex *mutex = &execmem_cache.mutex;
162  	unsigned long addr = (unsigned long)ptr;
163  	MA_STATE(mas, free_areas, addr - 1, addr + 1);
164  	unsigned long lower, upper;
165  	void *area = NULL;
166  	int err;
167  
168  	lower = addr;
169  	upper = addr + size - 1;
170  
171  	mutex_lock(mutex);
172  	area = mas_walk(&mas);
173  	if (area && mas.last == addr - 1)
174  		lower = mas.index;
175  
176  	area = mas_next(&mas, ULONG_MAX);
177  	if (area && mas.index == addr + size)
178  		upper = mas.last;
179  
180  	mas_set_range(&mas, lower, upper);
181  	err = mas_store_gfp(&mas, (void *)lower, GFP_KERNEL);
182  	mutex_unlock(mutex);
183  	if (err)
184  		return err;
185  
186  	return 0;
187  }
188  
189  static bool within_range(struct execmem_range *range, struct ma_state *mas,
190  			 size_t size)
191  {
192  	unsigned long addr = mas->index;
193  
194  	if (addr >= range->start && addr + size < range->end)
195  		return true;
196  
197  	if (range->fallback_start &&
198  	    addr >= range->fallback_start && addr + size < range->fallback_end)
199  		return true;
200  
201  	return false;
202  }
203  
204  static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
205  {
206  	struct maple_tree *free_areas = &execmem_cache.free_areas;
207  	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
208  	MA_STATE(mas_free, free_areas, 0, ULONG_MAX);
209  	MA_STATE(mas_busy, busy_areas, 0, ULONG_MAX);
210  	struct mutex *mutex = &execmem_cache.mutex;
211  	unsigned long addr, last, area_size = 0;
212  	void *area, *ptr = NULL;
213  	int err;
214  
215  	mutex_lock(mutex);
216  	mas_for_each(&mas_free, area, ULONG_MAX) {
217  		area_size = mas_range_len(&mas_free);
218  
219  		if (area_size >= size && within_range(range, &mas_free, size))
220  			break;
221  	}
222  
223  	if (area_size < size)
224  		goto out_unlock;
225  
226  	addr = mas_free.index;
227  	last = mas_free.last;
228  
229  	/* insert allocated size to busy_areas at range [addr, addr + size) */
230  	mas_set_range(&mas_busy, addr, addr + size - 1);
231  	err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL);
232  	if (err)
233  		goto out_unlock;
234  
235  	mas_store_gfp(&mas_free, NULL, GFP_KERNEL);
236  	if (area_size > size) {
237  		void *ptr = (void *)(addr + size);
238  
239  		/*
240  		 * re-insert remaining free size to free_areas at range
241  		 * [addr + size, last]
242  		 */
243  		mas_set_range(&mas_free, addr + size, last);
244  		err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL);
245  		if (err) {
246  			mas_store_gfp(&mas_busy, NULL, GFP_KERNEL);
247  			goto out_unlock;
248  		}
249  	}
250  	ptr = (void *)addr;
251  
252  out_unlock:
253  	mutex_unlock(mutex);
254  	return ptr;
255  }
256  
257  static int execmem_cache_populate(struct execmem_range *range, size_t size)
258  {
259  	unsigned long vm_flags = VM_ALLOW_HUGE_VMAP;
260  	unsigned long start, end;
261  	struct vm_struct *vm;
262  	size_t alloc_size;
263  	int err = -ENOMEM;
264  	void *p;
265  
266  	alloc_size = round_up(size, PMD_SIZE);
267  	p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
268  	if (!p)
269  		return err;
270  
271  	vm = find_vm_area(p);
272  	if (!vm)
273  		goto err_free_mem;
274  
275  	/* fill memory with instructions that will trap */
276  	execmem_fill_trapping_insns(p, alloc_size, /* writable = */ true);
277  
278  	start = (unsigned long)p;
279  	end = start + alloc_size;
280  
281  	vunmap_range(start, end);
282  
283  	err = execmem_set_direct_map_valid(vm, false);
284  	if (err)
285  		goto err_free_mem;
286  
287  	err = vmap_pages_range_noflush(start, end, range->pgprot, vm->pages,
288  				       PMD_SHIFT);
289  	if (err)
290  		goto err_free_mem;
291  
292  	err = execmem_cache_add(p, alloc_size);
293  	if (err)
294  		goto err_free_mem;
295  
296  	return 0;
297  
298  err_free_mem:
299  	vfree(p);
300  	return err;
301  }
302  
303  static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
304  {
305  	void *p;
306  	int err;
307  
308  	p = __execmem_cache_alloc(range, size);
309  	if (p)
310  		return p;
311  
312  	err = execmem_cache_populate(range, size);
313  	if (err)
314  		return NULL;
315  
316  	return __execmem_cache_alloc(range, size);
317  }
318  
319  static bool execmem_cache_free(void *ptr)
320  {
321  	struct maple_tree *busy_areas = &execmem_cache.busy_areas;
322  	struct mutex *mutex = &execmem_cache.mutex;
323  	unsigned long addr = (unsigned long)ptr;
324  	MA_STATE(mas, busy_areas, addr, addr);
325  	size_t size;
326  	void *area;
327  
328  	mutex_lock(mutex);
329  	area = mas_walk(&mas);
330  	if (!area) {
331  		mutex_unlock(mutex);
332  		return false;
333  	}
334  	size = mas_range_len(&mas);
335  
336  	mas_store_gfp(&mas, NULL, GFP_KERNEL);
337  	mutex_unlock(mutex);
338  
339  	execmem_fill_trapping_insns(ptr, size, /* writable = */ false);
340  
341  	execmem_cache_add(ptr, size);
342  
343  	schedule_work(&execmem_cache_clean_work);
344  
345  	return true;
346  }
347  #else /* CONFIG_ARCH_HAS_EXECMEM_ROX */
348  static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
349  {
350  	return NULL;
351  }
352  
353  static bool execmem_cache_free(void *ptr)
354  {
355  	return false;
356  }
357  #endif /* CONFIG_ARCH_HAS_EXECMEM_ROX */
358  
359  void *execmem_alloc(enum execmem_type type, size_t size)
360  {
361  	struct execmem_range *range = &execmem_info->ranges[type];
362  	bool use_cache = range->flags & EXECMEM_ROX_CACHE;
363  	unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
364  	pgprot_t pgprot = range->pgprot;
365  	void *p;
366  
367  	if (use_cache)
368  		p = execmem_cache_alloc(range, size);
369  	else
370  		p = execmem_vmalloc(range, size, pgprot, vm_flags);
371  
372  	return kasan_reset_tag(p);
373  }
374  
375  void execmem_free(void *ptr)
376  {
377  	/*
378  	 * This memory may be RO, and freeing RO memory in an interrupt is not
379  	 * supported by vmalloc.
380  	 */
381  	WARN_ON(in_interrupt());
382  
383  	if (!execmem_cache_free(ptr))
384  		vfree(ptr);
385  }
386  
387  void *execmem_update_copy(void *dst, const void *src, size_t size)
388  {
389  	return text_poke_copy(dst, src, size);
390  }
391  
392  bool execmem_is_rox(enum execmem_type type)
393  {
394  	return !!(execmem_info->ranges[type].flags & EXECMEM_ROX_CACHE);
395  }
396  
397  static bool execmem_validate(struct execmem_info *info)
398  {
399  	struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
400  
401  	if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
402  		pr_crit("Invalid parameters for execmem allocator, module loading will fail");
403  		return false;
404  	}
405  
406  	if (!IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX)) {
407  		for (int i = EXECMEM_DEFAULT; i < EXECMEM_TYPE_MAX; i++) {
408  			r = &info->ranges[i];
409  
410  			if (r->flags & EXECMEM_ROX_CACHE) {
411  				pr_warn_once("ROX cache is not supported\n");
412  				r->flags &= ~EXECMEM_ROX_CACHE;
413  			}
414  		}
415  	}
416  
417  	return true;
418  }
419  
420  static void execmem_init_missing(struct execmem_info *info)
421  {
422  	struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
423  
424  	for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
425  		struct execmem_range *r = &info->ranges[i];
426  
427  		if (!r->start) {
428  			if (i == EXECMEM_MODULE_DATA)
429  				r->pgprot = PAGE_KERNEL;
430  			else
431  				r->pgprot = default_range->pgprot;
432  			r->alignment = default_range->alignment;
433  			r->start = default_range->start;
434  			r->end = default_range->end;
435  			r->flags = default_range->flags;
436  			r->fallback_start = default_range->fallback_start;
437  			r->fallback_end = default_range->fallback_end;
438  		}
439  	}
440  }
441  
442  struct execmem_info * __weak execmem_arch_setup(void)
443  {
444  	return NULL;
445  }
446  
447  static void __init __execmem_init(void)
448  {
449  	struct execmem_info *info = execmem_arch_setup();
450  
451  	if (!info) {
452  		info = execmem_info = &default_execmem_info;
453  		info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
454  		info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
455  		info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
456  		info->ranges[EXECMEM_DEFAULT].alignment = 1;
457  	}
458  
459  	if (!execmem_validate(info))
460  		return;
461  
462  	execmem_init_missing(info);
463  
464  	execmem_info = info;
465  }
466  
467  #ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
468  static int __init execmem_late_init(void)
469  {
470  	__execmem_init();
471  	return 0;
472  }
473  core_initcall(execmem_late_init);
474  #else
475  void __init execmem_init(void)
476  {
477  	__execmem_init();
478  }
479  #endif
480