1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Module internals 3 * 4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org> 7 */ 8 9 #include <linux/elf.h> 10 #include <linux/compiler.h> 11 #include <linux/module.h> 12 #include <linux/mutex.h> 13 #include <linux/rculist.h> 14 #include <linux/rcupdate.h> 15 #include <linux/mm.h> 16 17 #ifndef ARCH_SHF_SMALL 18 #define ARCH_SHF_SMALL 0 19 #endif 20 21 /* 22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this 23 * section. This leaves 28 bits for offset on 32-bit systems, which is 24 * about 256 MiB (WARN_ON_ONCE if we exceed that). 25 */ 26 27 #define SH_ENTSIZE_TYPE_BITS 4 28 #define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS) 29 #define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1) 30 #define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1) 31 32 /* Maximum number of characters written by module_flags() */ 33 #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) 34 35 extern struct mutex module_mutex; 36 extern struct list_head modules; 37 38 extern struct module_attribute *modinfo_attrs[]; 39 extern size_t modinfo_attrs_count; 40 41 /* Provided by the linker */ 42 extern const struct kernel_symbol __start___ksymtab[]; 43 extern const struct kernel_symbol __stop___ksymtab[]; 44 extern const struct kernel_symbol __start___ksymtab_gpl[]; 45 extern const struct kernel_symbol __stop___ksymtab_gpl[]; 46 extern const s32 __start___kcrctab[]; 47 extern const s32 __start___kcrctab_gpl[]; 48 49 struct load_info { 50 const char *name; 51 /* pointer to module in temporary copy, freed at end of load_module() */ 52 struct module *mod; 53 Elf_Ehdr *hdr; 54 unsigned long len; 55 Elf_Shdr *sechdrs; 56 char *secstrings, *strtab; 57 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; 58 bool sig_ok; 59 #ifdef CONFIG_KALLSYMS 60 unsigned long mod_kallsyms_init_off; 61 #endif 62 #ifdef CONFIG_MODULE_DECOMPRESS 63 #ifdef CONFIG_MODULE_STATS 64 unsigned long compressed_len; 65 #endif 66 struct page **pages; 67 unsigned int max_pages; 68 unsigned int used_pages; 69 #endif 70 struct { 71 unsigned int sym, str, mod, vers, info, pcpu; 72 } index; 73 }; 74 75 enum mod_license { 76 NOT_GPL_ONLY, 77 GPL_ONLY, 78 }; 79 80 struct find_symbol_arg { 81 /* Input */ 82 const char *name; 83 bool gplok; 84 bool warn; 85 86 /* Output */ 87 struct module *owner; 88 const s32 *crc; 89 const struct kernel_symbol *sym; 90 enum mod_license license; 91 }; 92 93 int mod_verify_sig(const void *mod, struct load_info *info); 94 int try_to_force_load(struct module *mod, const char *reason); 95 bool find_symbol(struct find_symbol_arg *fsa); 96 struct module *find_module_all(const char *name, size_t len, bool even_unformed); 97 int cmp_name(const void *name, const void *sym); 98 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, 99 Elf_Shdr *sechdr, unsigned int section); 100 char *module_flags(struct module *mod, char *buf, bool show_state); 101 size_t module_flags_taint(unsigned long taints, char *buf); 102 103 char *module_next_tag_pair(char *string, unsigned long *secsize); 104 105 #define for_each_modinfo_entry(entry, info, name) \ 106 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry)) 107 108 static inline void module_assert_mutex_or_preempt(void) 109 { 110 #ifdef CONFIG_LOCKDEP 111 if (unlikely(!debug_locks)) 112 return; 113 114 WARN_ON_ONCE(!rcu_read_lock_sched_held() && 115 !lockdep_is_held(&module_mutex)); 116 #endif 117 } 118 119 static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym) 120 { 121 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 122 return (unsigned long)offset_to_ptr(&sym->value_offset); 123 #else 124 return sym->value; 125 #endif 126 } 127 128 #ifdef CONFIG_LIVEPATCH 129 int copy_module_elf(struct module *mod, struct load_info *info); 130 void free_module_elf(struct module *mod); 131 #else /* !CONFIG_LIVEPATCH */ 132 static inline int copy_module_elf(struct module *mod, struct load_info *info) 133 { 134 return 0; 135 } 136 137 static inline void free_module_elf(struct module *mod) { } 138 #endif /* CONFIG_LIVEPATCH */ 139 140 static inline bool set_livepatch_module(struct module *mod) 141 { 142 #ifdef CONFIG_LIVEPATCH 143 mod->klp = true; 144 return true; 145 #else 146 return false; 147 #endif 148 } 149 150 /** 151 * enum fail_dup_mod_reason - state at which a duplicate module was detected 152 * 153 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but 154 * we've determined that another module with the same name is already loaded 155 * or being processed on our &modules list. This happens on early_mod_check() 156 * right before layout_and_allocate(). The kernel would have already 157 * vmalloc()'d space for the entire module through finit_module(). If 158 * decompression was used two vmap() spaces were used. These failures can 159 * happen when userspace has not seen the module present on the kernel and 160 * tries to load the module multiple times at same time. 161 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation 162 * checks and the kernel determines that the module was unique and because 163 * of this allocated yet another private kernel copy of the module space in 164 * layout_and_allocate() but after this determined in add_unformed_module() 165 * that another module with the same name is already loaded or being processed. 166 * These failures should be mitigated as much as possible and are indicative 167 * of really fast races in loading modules. Without module decompression 168 * they waste twice as much vmap space. With module decompression three 169 * times the module's size vmap space is wasted. 170 */ 171 enum fail_dup_mod_reason { 172 FAIL_DUP_MOD_BECOMING = 0, 173 FAIL_DUP_MOD_LOAD, 174 }; 175 176 #ifdef CONFIG_MODULE_DEBUGFS 177 extern struct dentry *mod_debugfs_root; 178 #endif 179 180 #ifdef CONFIG_MODULE_STATS 181 182 #define mod_stat_add_long(count, var) atomic_long_add(count, var) 183 #define mod_stat_inc(name) atomic_inc(name) 184 185 extern atomic_long_t total_mod_size; 186 extern atomic_long_t total_text_size; 187 extern atomic_long_t invalid_kread_bytes; 188 extern atomic_long_t invalid_decompress_bytes; 189 190 extern atomic_t modcount; 191 extern atomic_t failed_kreads; 192 extern atomic_t failed_decompress; 193 struct mod_fail_load { 194 struct list_head list; 195 char name[MODULE_NAME_LEN]; 196 atomic_long_t count; 197 unsigned long dup_fail_mask; 198 }; 199 200 int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason); 201 void mod_stat_bump_invalid(struct load_info *info, int flags); 202 void mod_stat_bump_becoming(struct load_info *info, int flags); 203 204 #else 205 206 #define mod_stat_add_long(name, var) 207 #define mod_stat_inc(name) 208 209 static inline int try_add_failed_module(const char *name, 210 enum fail_dup_mod_reason reason) 211 { 212 return 0; 213 } 214 215 static inline void mod_stat_bump_invalid(struct load_info *info, int flags) 216 { 217 } 218 219 static inline void mod_stat_bump_becoming(struct load_info *info, int flags) 220 { 221 } 222 223 #endif /* CONFIG_MODULE_STATS */ 224 225 #ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS 226 bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret); 227 void kmod_dup_request_announce(char *module_name, int ret); 228 #else 229 static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) 230 { 231 return false; 232 } 233 234 static inline void kmod_dup_request_announce(char *module_name, int ret) 235 { 236 } 237 #endif 238 239 #ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING 240 struct mod_unload_taint { 241 struct list_head list; 242 char name[MODULE_NAME_LEN]; 243 unsigned long taints; 244 u64 count; 245 }; 246 247 int try_add_tainted_module(struct module *mod); 248 void print_unloaded_tainted_modules(void); 249 #else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 250 static inline int try_add_tainted_module(struct module *mod) 251 { 252 return 0; 253 } 254 255 static inline void print_unloaded_tainted_modules(void) 256 { 257 } 258 #endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 259 260 #ifdef CONFIG_MODULE_DECOMPRESS 261 int module_decompress(struct load_info *info, const void *buf, size_t size); 262 void module_decompress_cleanup(struct load_info *info); 263 #else 264 static inline int module_decompress(struct load_info *info, 265 const void *buf, size_t size) 266 { 267 return -EOPNOTSUPP; 268 } 269 270 static inline void module_decompress_cleanup(struct load_info *info) 271 { 272 } 273 #endif 274 275 struct mod_tree_root { 276 #ifdef CONFIG_MODULES_TREE_LOOKUP 277 struct latch_tree_root root; 278 #endif 279 unsigned long addr_min; 280 unsigned long addr_max; 281 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 282 unsigned long data_addr_min; 283 unsigned long data_addr_max; 284 #endif 285 }; 286 287 extern struct mod_tree_root mod_tree; 288 289 #ifdef CONFIG_MODULES_TREE_LOOKUP 290 void mod_tree_insert(struct module *mod); 291 void mod_tree_remove_init(struct module *mod); 292 void mod_tree_remove(struct module *mod); 293 struct module *mod_find(unsigned long addr, struct mod_tree_root *tree); 294 #else /* !CONFIG_MODULES_TREE_LOOKUP */ 295 296 static inline void mod_tree_insert(struct module *mod) { } 297 static inline void mod_tree_remove_init(struct module *mod) { } 298 static inline void mod_tree_remove(struct module *mod) { } 299 static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree) 300 { 301 struct module *mod; 302 303 list_for_each_entry_rcu(mod, &modules, list, 304 lockdep_is_held(&module_mutex)) { 305 if (within_module(addr, mod)) 306 return mod; 307 } 308 309 return NULL; 310 } 311 #endif /* CONFIG_MODULES_TREE_LOOKUP */ 312 313 void module_enable_ro(const struct module *mod, bool after_init); 314 void module_enable_nx(const struct module *mod); 315 void module_enable_x(const struct module *mod); 316 int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, 317 char *secstrings, struct module *mod); 318 319 #ifdef CONFIG_MODULE_SIG 320 int module_sig_check(struct load_info *info, int flags); 321 #else /* !CONFIG_MODULE_SIG */ 322 static inline int module_sig_check(struct load_info *info, int flags) 323 { 324 return 0; 325 } 326 #endif /* !CONFIG_MODULE_SIG */ 327 328 #ifdef CONFIG_DEBUG_KMEMLEAK 329 void kmemleak_load_module(const struct module *mod, const struct load_info *info); 330 #else /* !CONFIG_DEBUG_KMEMLEAK */ 331 static inline void kmemleak_load_module(const struct module *mod, 332 const struct load_info *info) { } 333 #endif /* CONFIG_DEBUG_KMEMLEAK */ 334 335 #ifdef CONFIG_KALLSYMS 336 void init_build_id(struct module *mod, const struct load_info *info); 337 void layout_symtab(struct module *mod, struct load_info *info); 338 void add_kallsyms(struct module *mod, const struct load_info *info); 339 340 static inline bool sect_empty(const Elf_Shdr *sect) 341 { 342 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; 343 } 344 #else /* !CONFIG_KALLSYMS */ 345 static inline void init_build_id(struct module *mod, const struct load_info *info) { } 346 static inline void layout_symtab(struct module *mod, struct load_info *info) { } 347 static inline void add_kallsyms(struct module *mod, const struct load_info *info) { } 348 #endif /* CONFIG_KALLSYMS */ 349 350 #ifdef CONFIG_SYSFS 351 int mod_sysfs_setup(struct module *mod, const struct load_info *info, 352 struct kernel_param *kparam, unsigned int num_params); 353 void mod_sysfs_teardown(struct module *mod); 354 void init_param_lock(struct module *mod); 355 #else /* !CONFIG_SYSFS */ 356 static inline int mod_sysfs_setup(struct module *mod, 357 const struct load_info *info, 358 struct kernel_param *kparam, 359 unsigned int num_params) 360 { 361 return 0; 362 } 363 364 static inline void mod_sysfs_teardown(struct module *mod) { } 365 static inline void init_param_lock(struct module *mod) { } 366 #endif /* CONFIG_SYSFS */ 367 368 #ifdef CONFIG_MODVERSIONS 369 int check_version(const struct load_info *info, 370 const char *symname, struct module *mod, const s32 *crc); 371 void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp, 372 struct kernel_symbol *ks, struct tracepoint * const *tp); 373 int check_modstruct_version(const struct load_info *info, struct module *mod); 374 int same_magic(const char *amagic, const char *bmagic, bool has_crcs); 375 #else /* !CONFIG_MODVERSIONS */ 376 static inline int check_version(const struct load_info *info, 377 const char *symname, 378 struct module *mod, 379 const s32 *crc) 380 { 381 return 1; 382 } 383 384 static inline int check_modstruct_version(const struct load_info *info, 385 struct module *mod) 386 { 387 return 1; 388 } 389 390 static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs) 391 { 392 return strcmp(amagic, bmagic) == 0; 393 } 394 #endif /* CONFIG_MODVERSIONS */ 395