112af2b83SMike Rapoport (IBM) // SPDX-License-Identifier: GPL-2.0
212af2b83SMike Rapoport (IBM) /*
312af2b83SMike Rapoport (IBM) * Copyright (C) 2002 Richard Henderson
412af2b83SMike Rapoport (IBM) * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
512af2b83SMike Rapoport (IBM) * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
612af2b83SMike Rapoport (IBM) * Copyright (C) 2024 Mike Rapoport IBM.
712af2b83SMike Rapoport (IBM) */
812af2b83SMike Rapoport (IBM)
912af2b83SMike Rapoport (IBM) #include <linux/mm.h>
1012af2b83SMike Rapoport (IBM) #include <linux/vmalloc.h>
1112af2b83SMike Rapoport (IBM) #include <linux/execmem.h>
1212af2b83SMike Rapoport (IBM) #include <linux/moduleloader.h>
1312af2b83SMike Rapoport (IBM)
14f6bec26cSMike Rapoport (IBM) static struct execmem_info *execmem_info __ro_after_init;
15*223b5e57SMike Rapoport (IBM) static struct execmem_info default_execmem_info __ro_after_init;
16f6bec26cSMike Rapoport (IBM)
__execmem_alloc(struct execmem_range * range,size_t size)17f6bec26cSMike Rapoport (IBM) static void *__execmem_alloc(struct execmem_range *range, size_t size)
1812af2b83SMike Rapoport (IBM) {
19*223b5e57SMike Rapoport (IBM) bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
20*223b5e57SMike Rapoport (IBM) unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
21*223b5e57SMike Rapoport (IBM) gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
22f6bec26cSMike Rapoport (IBM) unsigned long start = range->start;
23f6bec26cSMike Rapoport (IBM) unsigned long end = range->end;
24f6bec26cSMike Rapoport (IBM) unsigned int align = range->alignment;
25f6bec26cSMike Rapoport (IBM) pgprot_t pgprot = range->pgprot;
26*223b5e57SMike Rapoport (IBM) void *p;
27f6bec26cSMike Rapoport (IBM)
28*223b5e57SMike Rapoport (IBM) if (kasan)
29*223b5e57SMike Rapoport (IBM) vm_flags |= VM_DEFER_KMEMLEAK;
30*223b5e57SMike Rapoport (IBM)
31*223b5e57SMike Rapoport (IBM) p = __vmalloc_node_range(size, align, start, end, gfp_flags,
32*223b5e57SMike Rapoport (IBM) pgprot, vm_flags, NUMA_NO_NODE,
33*223b5e57SMike Rapoport (IBM) __builtin_return_address(0));
34*223b5e57SMike Rapoport (IBM) if (!p && range->fallback_start) {
35*223b5e57SMike Rapoport (IBM) start = range->fallback_start;
36*223b5e57SMike Rapoport (IBM) end = range->fallback_end;
37*223b5e57SMike Rapoport (IBM) p = __vmalloc_node_range(size, align, start, end, gfp_flags,
38*223b5e57SMike Rapoport (IBM) pgprot, vm_flags, NUMA_NO_NODE,
39*223b5e57SMike Rapoport (IBM) __builtin_return_address(0));
40*223b5e57SMike Rapoport (IBM) }
41*223b5e57SMike Rapoport (IBM)
42*223b5e57SMike Rapoport (IBM) if (!p) {
43*223b5e57SMike Rapoport (IBM) pr_warn_ratelimited("execmem: unable to allocate memory\n");
44*223b5e57SMike Rapoport (IBM) return NULL;
45*223b5e57SMike Rapoport (IBM) }
46*223b5e57SMike Rapoport (IBM)
47*223b5e57SMike Rapoport (IBM) if (kasan && (kasan_alloc_module_shadow(p, size, GFP_KERNEL) < 0)) {
48*223b5e57SMike Rapoport (IBM) vfree(p);
49*223b5e57SMike Rapoport (IBM) return NULL;
50*223b5e57SMike Rapoport (IBM) }
51*223b5e57SMike Rapoport (IBM)
52*223b5e57SMike Rapoport (IBM) return kasan_reset_tag(p);
5312af2b83SMike Rapoport (IBM) }
5412af2b83SMike Rapoport (IBM)
execmem_alloc(enum execmem_type type,size_t size)5512af2b83SMike Rapoport (IBM) void *execmem_alloc(enum execmem_type type, size_t size)
5612af2b83SMike Rapoport (IBM) {
57*223b5e57SMike Rapoport (IBM) struct execmem_range *range = &execmem_info->ranges[type];
58f6bec26cSMike Rapoport (IBM)
59f6bec26cSMike Rapoport (IBM) return __execmem_alloc(range, size);
6012af2b83SMike Rapoport (IBM) }
6112af2b83SMike Rapoport (IBM)
execmem_free(void * ptr)6212af2b83SMike Rapoport (IBM) void execmem_free(void *ptr)
6312af2b83SMike Rapoport (IBM) {
6412af2b83SMike Rapoport (IBM) /*
6512af2b83SMike Rapoport (IBM) * This memory may be RO, and freeing RO memory in an interrupt is not
6612af2b83SMike Rapoport (IBM) * supported by vmalloc.
6712af2b83SMike Rapoport (IBM) */
6812af2b83SMike Rapoport (IBM) WARN_ON(in_interrupt());
6912af2b83SMike Rapoport (IBM) vfree(ptr);
7012af2b83SMike Rapoport (IBM) }
71f6bec26cSMike Rapoport (IBM)
execmem_validate(struct execmem_info * info)72f6bec26cSMike Rapoport (IBM) static bool execmem_validate(struct execmem_info *info)
73f6bec26cSMike Rapoport (IBM) {
74f6bec26cSMike Rapoport (IBM) struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
75f6bec26cSMike Rapoport (IBM)
76f6bec26cSMike Rapoport (IBM) if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
77f6bec26cSMike Rapoport (IBM) pr_crit("Invalid parameters for execmem allocator, module loading will fail");
78f6bec26cSMike Rapoport (IBM) return false;
79f6bec26cSMike Rapoport (IBM) }
80f6bec26cSMike Rapoport (IBM)
81f6bec26cSMike Rapoport (IBM) return true;
82f6bec26cSMike Rapoport (IBM) }
83f6bec26cSMike Rapoport (IBM)
execmem_init_missing(struct execmem_info * info)84f6bec26cSMike Rapoport (IBM) static void execmem_init_missing(struct execmem_info *info)
85f6bec26cSMike Rapoport (IBM) {
86f6bec26cSMike Rapoport (IBM) struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
87f6bec26cSMike Rapoport (IBM)
88f6bec26cSMike Rapoport (IBM) for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
89f6bec26cSMike Rapoport (IBM) struct execmem_range *r = &info->ranges[i];
90f6bec26cSMike Rapoport (IBM)
91f6bec26cSMike Rapoport (IBM) if (!r->start) {
92*223b5e57SMike Rapoport (IBM) if (i == EXECMEM_MODULE_DATA)
93*223b5e57SMike Rapoport (IBM) r->pgprot = PAGE_KERNEL;
94*223b5e57SMike Rapoport (IBM) else
95f6bec26cSMike Rapoport (IBM) r->pgprot = default_range->pgprot;
96f6bec26cSMike Rapoport (IBM) r->alignment = default_range->alignment;
97f6bec26cSMike Rapoport (IBM) r->start = default_range->start;
98f6bec26cSMike Rapoport (IBM) r->end = default_range->end;
99*223b5e57SMike Rapoport (IBM) r->flags = default_range->flags;
100*223b5e57SMike Rapoport (IBM) r->fallback_start = default_range->fallback_start;
101*223b5e57SMike Rapoport (IBM) r->fallback_end = default_range->fallback_end;
102f6bec26cSMike Rapoport (IBM) }
103f6bec26cSMike Rapoport (IBM) }
104f6bec26cSMike Rapoport (IBM) }
105f6bec26cSMike Rapoport (IBM)
execmem_arch_setup(void)106f6bec26cSMike Rapoport (IBM) struct execmem_info * __weak execmem_arch_setup(void)
107f6bec26cSMike Rapoport (IBM) {
108f6bec26cSMike Rapoport (IBM) return NULL;
109f6bec26cSMike Rapoport (IBM) }
110f6bec26cSMike Rapoport (IBM)
__execmem_init(void)111*223b5e57SMike Rapoport (IBM) static void __init __execmem_init(void)
112f6bec26cSMike Rapoport (IBM) {
113f6bec26cSMike Rapoport (IBM) struct execmem_info *info = execmem_arch_setup();
114f6bec26cSMike Rapoport (IBM)
115*223b5e57SMike Rapoport (IBM) if (!info) {
116*223b5e57SMike Rapoport (IBM) info = execmem_info = &default_execmem_info;
117*223b5e57SMike Rapoport (IBM) info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
118*223b5e57SMike Rapoport (IBM) info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
119*223b5e57SMike Rapoport (IBM) info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
120*223b5e57SMike Rapoport (IBM) info->ranges[EXECMEM_DEFAULT].alignment = 1;
121*223b5e57SMike Rapoport (IBM) }
122*223b5e57SMike Rapoport (IBM)
123*223b5e57SMike Rapoport (IBM) if (!execmem_validate(info))
124f6bec26cSMike Rapoport (IBM) return;
125f6bec26cSMike Rapoport (IBM)
126f6bec26cSMike Rapoport (IBM) execmem_init_missing(info);
127f6bec26cSMike Rapoport (IBM)
128f6bec26cSMike Rapoport (IBM) execmem_info = info;
129f6bec26cSMike Rapoport (IBM) }
130*223b5e57SMike Rapoport (IBM)
131*223b5e57SMike Rapoport (IBM) #ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
execmem_late_init(void)132*223b5e57SMike Rapoport (IBM) static int __init execmem_late_init(void)
133*223b5e57SMike Rapoport (IBM) {
134*223b5e57SMike Rapoport (IBM) __execmem_init();
135*223b5e57SMike Rapoport (IBM) return 0;
136*223b5e57SMike Rapoport (IBM) }
137*223b5e57SMike Rapoport (IBM) core_initcall(execmem_late_init);
138*223b5e57SMike Rapoport (IBM) #else
execmem_init(void)139*223b5e57SMike Rapoport (IBM) void __init execmem_init(void)
140*223b5e57SMike Rapoport (IBM) {
141*223b5e57SMike Rapoport (IBM) __execmem_init();
142*223b5e57SMike Rapoport (IBM) }
143*223b5e57SMike Rapoport (IBM) #endif
144