1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* Module internals
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
7 */
8
9 #include <linux/elf.h>
10 #include <linux/compiler.h>
11 #include <linux/module.h>
12 #include <linux/mutex.h>
13 #include <linux/rculist.h>
14 #include <linux/rcupdate.h>
15 #include <linux/mm.h>
16
17 #ifndef ARCH_SHF_SMALL
18 #define ARCH_SHF_SMALL 0
19 #endif
20
21 /*
22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
23 * section. This leaves 28 bits for offset on 32-bit systems, which is
24 * about 256 MiB (WARN_ON_ONCE if we exceed that).
25 */
26
27 #define SH_ENTSIZE_TYPE_BITS 4
28 #define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
29 #define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
30 #define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
31
32 /* Maximum number of characters written by module_flags() */
33 #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
34
35 struct kernel_symbol {
36 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
37 int value_offset;
38 int name_offset;
39 int namespace_offset;
40 #else
41 unsigned long value;
42 const char *name;
43 const char *namespace;
44 #endif
45 };
46
47 extern struct mutex module_mutex;
48 extern struct list_head modules;
49
50 extern const struct module_attribute *const modinfo_attrs[];
51 extern const size_t modinfo_attrs_count;
52
53 /* Provided by the linker */
54 extern const struct kernel_symbol __start___ksymtab[];
55 extern const struct kernel_symbol __stop___ksymtab[];
56 extern const struct kernel_symbol __start___ksymtab_gpl[];
57 extern const struct kernel_symbol __stop___ksymtab_gpl[];
58 extern const u32 __start___kcrctab[];
59 extern const u32 __start___kcrctab_gpl[];
60
61 #define KMOD_PATH_LEN 256
62 extern char modprobe_path[];
63
64 struct load_info {
65 const char *name;
66 /* pointer to module in temporary copy, freed at end of load_module() */
67 struct module *mod;
68 Elf_Ehdr *hdr;
69 unsigned long len;
70 Elf_Shdr *sechdrs;
71 char *secstrings, *strtab;
72 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
73 bool sig_ok;
74 #ifdef CONFIG_KALLSYMS
75 unsigned long mod_kallsyms_init_off;
76 #endif
77 #ifdef CONFIG_MODULE_DECOMPRESS
78 #ifdef CONFIG_MODULE_STATS
79 unsigned long compressed_len;
80 #endif
81 struct page **pages;
82 unsigned int max_pages;
83 unsigned int used_pages;
84 #endif
85 struct {
86 unsigned int sym;
87 unsigned int str;
88 unsigned int mod;
89 unsigned int vers;
90 unsigned int info;
91 unsigned int pcpu;
92 unsigned int vers_ext_crc;
93 unsigned int vers_ext_name;
94 } index;
95 };
96
97 enum mod_license {
98 NOT_GPL_ONLY,
99 GPL_ONLY,
100 };
101
102 struct find_symbol_arg {
103 /* Input */
104 const char *name;
105 bool gplok;
106 bool warn;
107
108 /* Output */
109 struct module *owner;
110 const u32 *crc;
111 const struct kernel_symbol *sym;
112 enum mod_license license;
113 };
114
115 /* modules using other modules */
116 struct module_use {
117 struct list_head source_list;
118 struct list_head target_list;
119 struct module *source, *target;
120 };
121
122 int mod_verify_sig(const void *mod, struct load_info *info);
123 int try_to_force_load(struct module *mod, const char *reason);
124 bool find_symbol(struct find_symbol_arg *fsa);
125 struct module *find_module_all(const char *name, size_t len, bool even_unformed);
126 int cmp_name(const void *name, const void *sym);
127 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
128 Elf_Shdr *sechdr, unsigned int section);
129 char *module_flags(struct module *mod, char *buf, bool show_state);
130 size_t module_flags_taint(unsigned long taints, char *buf);
131
132 char *module_next_tag_pair(char *string, unsigned long *secsize);
133
134 #define for_each_modinfo_entry(entry, info, name) \
135 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
136
kernel_symbol_value(const struct kernel_symbol * sym)137 static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
138 {
139 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
140 return (unsigned long)offset_to_ptr(&sym->value_offset);
141 #else
142 return sym->value;
143 #endif
144 }
145
146 #ifdef CONFIG_LIVEPATCH
147 int copy_module_elf(struct module *mod, struct load_info *info);
148 void free_module_elf(struct module *mod);
149 #else /* !CONFIG_LIVEPATCH */
copy_module_elf(struct module * mod,struct load_info * info)150 static inline int copy_module_elf(struct module *mod, struct load_info *info)
151 {
152 return 0;
153 }
154
free_module_elf(struct module * mod)155 static inline void free_module_elf(struct module *mod) { }
156 #endif /* CONFIG_LIVEPATCH */
157
set_livepatch_module(struct module * mod)158 static inline bool set_livepatch_module(struct module *mod)
159 {
160 #ifdef CONFIG_LIVEPATCH
161 mod->klp = true;
162 return true;
163 #else
164 return false;
165 #endif
166 }
167
168 /**
169 * enum fail_dup_mod_reason - state at which a duplicate module was detected
170 *
171 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
172 * we've determined that another module with the same name is already loaded
173 * or being processed on our &modules list. This happens on early_mod_check()
174 * right before layout_and_allocate(). The kernel would have already
175 * vmalloc()'d space for the entire module through finit_module(). If
176 * decompression was used two vmap() spaces were used. These failures can
177 * happen when userspace has not seen the module present on the kernel and
178 * tries to load the module multiple times at same time.
179 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
180 * checks and the kernel determines that the module was unique and because
181 * of this allocated yet another private kernel copy of the module space in
182 * layout_and_allocate() but after this determined in add_unformed_module()
183 * that another module with the same name is already loaded or being processed.
184 * These failures should be mitigated as much as possible and are indicative
185 * of really fast races in loading modules. Without module decompression
186 * they waste twice as much vmap space. With module decompression three
187 * times the module's size vmap space is wasted.
188 */
189 enum fail_dup_mod_reason {
190 FAIL_DUP_MOD_BECOMING = 0,
191 FAIL_DUP_MOD_LOAD,
192 };
193
194 #ifdef CONFIG_MODULE_DEBUGFS
195 extern struct dentry *mod_debugfs_root;
196 #endif
197
198 #ifdef CONFIG_MODULE_STATS
199
200 #define mod_stat_add_long(count, var) atomic_long_add(count, var)
201 #define mod_stat_inc(name) atomic_inc(name)
202
203 extern atomic_long_t total_mod_size;
204 extern atomic_long_t total_text_size;
205 extern atomic_long_t invalid_kread_bytes;
206 extern atomic_long_t invalid_decompress_bytes;
207
208 extern atomic_t modcount;
209 extern atomic_t failed_kreads;
210 extern atomic_t failed_decompress;
211 struct mod_fail_load {
212 struct list_head list;
213 char name[MODULE_NAME_LEN];
214 atomic_long_t count;
215 unsigned long dup_fail_mask;
216 };
217
218 int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
219 void mod_stat_bump_invalid(struct load_info *info, int flags);
220 void mod_stat_bump_becoming(struct load_info *info, int flags);
221
222 #else
223
224 #define mod_stat_add_long(name, var)
225 #define mod_stat_inc(name)
226
try_add_failed_module(const char * name,enum fail_dup_mod_reason reason)227 static inline int try_add_failed_module(const char *name,
228 enum fail_dup_mod_reason reason)
229 {
230 return 0;
231 }
232
mod_stat_bump_invalid(struct load_info * info,int flags)233 static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
234 {
235 }
236
mod_stat_bump_becoming(struct load_info * info,int flags)237 static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
238 {
239 }
240
241 #endif /* CONFIG_MODULE_STATS */
242
243 #ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
244 bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
245 void kmod_dup_request_announce(char *module_name, int ret);
246 #else
kmod_dup_request_exists_wait(char * module_name,bool wait,int * dup_ret)247 static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
248 {
249 return false;
250 }
251
kmod_dup_request_announce(char * module_name,int ret)252 static inline void kmod_dup_request_announce(char *module_name, int ret)
253 {
254 }
255 #endif
256
257 #ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
258 struct mod_unload_taint {
259 struct list_head list;
260 char name[MODULE_NAME_LEN];
261 unsigned long taints;
262 u64 count;
263 };
264
265 int try_add_tainted_module(struct module *mod);
266 void print_unloaded_tainted_modules(void);
267 #else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
try_add_tainted_module(struct module * mod)268 static inline int try_add_tainted_module(struct module *mod)
269 {
270 return 0;
271 }
272
print_unloaded_tainted_modules(void)273 static inline void print_unloaded_tainted_modules(void)
274 {
275 }
276 #endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
277
278 #ifdef CONFIG_MODULE_DECOMPRESS
279 int module_decompress(struct load_info *info, const void *buf, size_t size);
280 void module_decompress_cleanup(struct load_info *info);
281 #else
module_decompress(struct load_info * info,const void * buf,size_t size)282 static inline int module_decompress(struct load_info *info,
283 const void *buf, size_t size)
284 {
285 return -EOPNOTSUPP;
286 }
287
module_decompress_cleanup(struct load_info * info)288 static inline void module_decompress_cleanup(struct load_info *info)
289 {
290 }
291 #endif
292
293 struct mod_tree_root {
294 #ifdef CONFIG_MODULES_TREE_LOOKUP
295 struct latch_tree_root root;
296 #endif
297 unsigned long addr_min;
298 unsigned long addr_max;
299 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
300 unsigned long data_addr_min;
301 unsigned long data_addr_max;
302 #endif
303 };
304
305 extern struct mod_tree_root mod_tree;
306
307 #ifdef CONFIG_MODULES_TREE_LOOKUP
308 void mod_tree_insert(struct module *mod);
309 void mod_tree_remove_init(struct module *mod);
310 void mod_tree_remove(struct module *mod);
311 struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
312 #else /* !CONFIG_MODULES_TREE_LOOKUP */
313
mod_tree_insert(struct module * mod)314 static inline void mod_tree_insert(struct module *mod) { }
mod_tree_remove_init(struct module * mod)315 static inline void mod_tree_remove_init(struct module *mod) { }
mod_tree_remove(struct module * mod)316 static inline void mod_tree_remove(struct module *mod) { }
mod_find(unsigned long addr,struct mod_tree_root * tree)317 static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
318 {
319 struct module *mod;
320
321 list_for_each_entry_rcu(mod, &modules, list,
322 lockdep_is_held(&module_mutex)) {
323 if (within_module(addr, mod))
324 return mod;
325 }
326
327 return NULL;
328 }
329 #endif /* CONFIG_MODULES_TREE_LOOKUP */
330
331 int module_enable_rodata_ro(const struct module *mod);
332 int module_enable_rodata_ro_after_init(const struct module *mod);
333 int module_enable_data_nx(const struct module *mod);
334 int module_enable_text_rox(const struct module *mod);
335 int module_enforce_rwx_sections(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
336 const char *secstrings,
337 const struct module *mod);
338 void module_mark_ro_after_init(const Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
339 const char *secstrings);
340
341 #ifdef CONFIG_MODULE_SIG
342 int module_sig_check(struct load_info *info, int flags);
343 #else /* !CONFIG_MODULE_SIG */
module_sig_check(struct load_info * info,int flags)344 static inline int module_sig_check(struct load_info *info, int flags)
345 {
346 return 0;
347 }
348 #endif /* !CONFIG_MODULE_SIG */
349
350 #ifdef CONFIG_DEBUG_KMEMLEAK
351 void kmemleak_load_module(const struct module *mod, const struct load_info *info);
352 #else /* !CONFIG_DEBUG_KMEMLEAK */
kmemleak_load_module(const struct module * mod,const struct load_info * info)353 static inline void kmemleak_load_module(const struct module *mod,
354 const struct load_info *info) { }
355 #endif /* CONFIG_DEBUG_KMEMLEAK */
356
357 #ifdef CONFIG_KALLSYMS
358 void init_build_id(struct module *mod, const struct load_info *info);
359 void layout_symtab(struct module *mod, struct load_info *info);
360 void add_kallsyms(struct module *mod, const struct load_info *info);
361
sect_empty(const Elf_Shdr * sect)362 static inline bool sect_empty(const Elf_Shdr *sect)
363 {
364 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
365 }
366 #else /* !CONFIG_KALLSYMS */
init_build_id(struct module * mod,const struct load_info * info)367 static inline void init_build_id(struct module *mod, const struct load_info *info) { }
layout_symtab(struct module * mod,struct load_info * info)368 static inline void layout_symtab(struct module *mod, struct load_info *info) { }
add_kallsyms(struct module * mod,const struct load_info * info)369 static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
370 #endif /* CONFIG_KALLSYMS */
371
372 #ifdef CONFIG_SYSFS
373 int mod_sysfs_setup(struct module *mod, const struct load_info *info,
374 struct kernel_param *kparam, unsigned int num_params);
375 void mod_sysfs_teardown(struct module *mod);
376 void init_param_lock(struct module *mod);
377 #else /* !CONFIG_SYSFS */
mod_sysfs_setup(struct module * mod,const struct load_info * info,struct kernel_param * kparam,unsigned int num_params)378 static inline int mod_sysfs_setup(struct module *mod,
379 const struct load_info *info,
380 struct kernel_param *kparam,
381 unsigned int num_params)
382 {
383 return 0;
384 }
385
mod_sysfs_teardown(struct module * mod)386 static inline void mod_sysfs_teardown(struct module *mod) { }
init_param_lock(struct module * mod)387 static inline void init_param_lock(struct module *mod) { }
388 #endif /* CONFIG_SYSFS */
389
390 #ifdef CONFIG_MODVERSIONS
391 int check_version(const struct load_info *info,
392 const char *symname, struct module *mod, const u32 *crc);
393 void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
394 struct kernel_symbol *ks, struct tracepoint * const *tp);
395 int check_modstruct_version(const struct load_info *info, struct module *mod);
396 int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
397 struct modversion_info_ext {
398 size_t remaining;
399 const u32 *crc;
400 const char *name;
401 };
402 void modversion_ext_start(const struct load_info *info, struct modversion_info_ext *ver);
403 void modversion_ext_advance(struct modversion_info_ext *ver);
404 #define for_each_modversion_info_ext(ver, info) \
405 for (modversion_ext_start(info, &ver); ver.remaining > 0; modversion_ext_advance(&ver))
406 #else /* !CONFIG_MODVERSIONS */
check_version(const struct load_info * info,const char * symname,struct module * mod,const u32 * crc)407 static inline int check_version(const struct load_info *info,
408 const char *symname,
409 struct module *mod,
410 const u32 *crc)
411 {
412 return 1;
413 }
414
check_modstruct_version(const struct load_info * info,struct module * mod)415 static inline int check_modstruct_version(const struct load_info *info,
416 struct module *mod)
417 {
418 return 1;
419 }
420
same_magic(const char * amagic,const char * bmagic,bool has_crcs)421 static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
422 {
423 return strcmp(amagic, bmagic) == 0;
424 }
425 #endif /* CONFIG_MODVERSIONS */
426