1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Module internals 3 * 4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org> 7 */ 8 9 #include <linux/elf.h> 10 #include <linux/compiler.h> 11 #include <linux/module.h> 12 #include <linux/mutex.h> 13 #include <linux/rculist.h> 14 #include <linux/rcupdate.h> 15 #include <linux/mm.h> 16 17 #ifndef ARCH_SHF_SMALL 18 #define ARCH_SHF_SMALL 0 19 #endif 20 21 /* 22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this 23 * section. This leaves 28 bits for offset on 32-bit systems, which is 24 * about 256 MiB (WARN_ON_ONCE if we exceed that). 25 */ 26 27 #define SH_ENTSIZE_TYPE_BITS 4 28 #define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS) 29 #define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1) 30 #define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1) 31 32 /* Maximum number of characters written by module_flags() */ 33 #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) 34 35 struct kernel_symbol { 36 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 37 int value_offset; 38 int name_offset; 39 int namespace_offset; 40 #else 41 unsigned long value; 42 const char *name; 43 const char *namespace; 44 #endif 45 }; 46 47 extern struct mutex module_mutex; 48 extern struct list_head modules; 49 50 extern struct module_attribute *modinfo_attrs[]; 51 extern size_t modinfo_attrs_count; 52 53 /* Provided by the linker */ 54 extern const struct kernel_symbol __start___ksymtab[]; 55 extern const struct kernel_symbol __stop___ksymtab[]; 56 extern const struct kernel_symbol __start___ksymtab_gpl[]; 57 extern const struct kernel_symbol __stop___ksymtab_gpl[]; 58 extern const s32 __start___kcrctab[]; 59 extern const s32 __start___kcrctab_gpl[]; 60 61 struct load_info { 62 const char *name; 63 /* pointer to module in temporary copy, freed at end of load_module() */ 64 struct module *mod; 65 Elf_Ehdr *hdr; 66 unsigned long len; 67 Elf_Shdr *sechdrs; 68 char *secstrings, *strtab; 69 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; 70 bool sig_ok; 71 #ifdef CONFIG_KALLSYMS 72 unsigned long mod_kallsyms_init_off; 73 #endif 74 #ifdef CONFIG_MODULE_DECOMPRESS 75 #ifdef CONFIG_MODULE_STATS 76 unsigned long compressed_len; 77 #endif 78 struct page **pages; 79 unsigned int max_pages; 80 unsigned int used_pages; 81 #endif 82 struct { 83 unsigned int sym; 84 unsigned int str; 85 unsigned int mod; 86 unsigned int vers; 87 unsigned int info; 88 unsigned int pcpu; 89 } index; 90 }; 91 92 enum mod_license { 93 NOT_GPL_ONLY, 94 GPL_ONLY, 95 }; 96 97 struct find_symbol_arg { 98 /* Input */ 99 const char *name; 100 bool gplok; 101 bool warn; 102 103 /* Output */ 104 struct module *owner; 105 const s32 *crc; 106 const struct kernel_symbol *sym; 107 enum mod_license license; 108 }; 109 110 int mod_verify_sig(const void *mod, struct load_info *info); 111 int try_to_force_load(struct module *mod, const char *reason); 112 bool find_symbol(struct find_symbol_arg *fsa); 113 struct module *find_module_all(const char *name, size_t len, bool even_unformed); 114 int cmp_name(const void *name, const void *sym); 115 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, 116 Elf_Shdr *sechdr, unsigned int section); 117 char *module_flags(struct module *mod, char *buf, bool show_state); 118 size_t module_flags_taint(unsigned long taints, char *buf); 119 120 char *module_next_tag_pair(char *string, unsigned long *secsize); 121 122 #define for_each_modinfo_entry(entry, info, name) \ 123 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry)) 124 125 static inline void module_assert_mutex_or_preempt(void) 126 { 127 #ifdef CONFIG_LOCKDEP 128 if (unlikely(!debug_locks)) 129 return; 130 131 WARN_ON_ONCE(!rcu_read_lock_sched_held() && 132 !lockdep_is_held(&module_mutex)); 133 #endif 134 } 135 136 static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym) 137 { 138 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 139 return (unsigned long)offset_to_ptr(&sym->value_offset); 140 #else 141 return sym->value; 142 #endif 143 } 144 145 #ifdef CONFIG_LIVEPATCH 146 int copy_module_elf(struct module *mod, struct load_info *info); 147 void free_module_elf(struct module *mod); 148 #else /* !CONFIG_LIVEPATCH */ 149 static inline int copy_module_elf(struct module *mod, struct load_info *info) 150 { 151 return 0; 152 } 153 154 static inline void free_module_elf(struct module *mod) { } 155 #endif /* CONFIG_LIVEPATCH */ 156 157 static inline bool set_livepatch_module(struct module *mod) 158 { 159 #ifdef CONFIG_LIVEPATCH 160 mod->klp = true; 161 return true; 162 #else 163 return false; 164 #endif 165 } 166 167 /** 168 * enum fail_dup_mod_reason - state at which a duplicate module was detected 169 * 170 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but 171 * we've determined that another module with the same name is already loaded 172 * or being processed on our &modules list. This happens on early_mod_check() 173 * right before layout_and_allocate(). The kernel would have already 174 * vmalloc()'d space for the entire module through finit_module(). If 175 * decompression was used two vmap() spaces were used. These failures can 176 * happen when userspace has not seen the module present on the kernel and 177 * tries to load the module multiple times at same time. 178 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation 179 * checks and the kernel determines that the module was unique and because 180 * of this allocated yet another private kernel copy of the module space in 181 * layout_and_allocate() but after this determined in add_unformed_module() 182 * that another module with the same name is already loaded or being processed. 183 * These failures should be mitigated as much as possible and are indicative 184 * of really fast races in loading modules. Without module decompression 185 * they waste twice as much vmap space. With module decompression three 186 * times the module's size vmap space is wasted. 187 */ 188 enum fail_dup_mod_reason { 189 FAIL_DUP_MOD_BECOMING = 0, 190 FAIL_DUP_MOD_LOAD, 191 }; 192 193 #ifdef CONFIG_MODULE_DEBUGFS 194 extern struct dentry *mod_debugfs_root; 195 #endif 196 197 #ifdef CONFIG_MODULE_STATS 198 199 #define mod_stat_add_long(count, var) atomic_long_add(count, var) 200 #define mod_stat_inc(name) atomic_inc(name) 201 202 extern atomic_long_t total_mod_size; 203 extern atomic_long_t total_text_size; 204 extern atomic_long_t invalid_kread_bytes; 205 extern atomic_long_t invalid_decompress_bytes; 206 207 extern atomic_t modcount; 208 extern atomic_t failed_kreads; 209 extern atomic_t failed_decompress; 210 struct mod_fail_load { 211 struct list_head list; 212 char name[MODULE_NAME_LEN]; 213 atomic_long_t count; 214 unsigned long dup_fail_mask; 215 }; 216 217 int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason); 218 void mod_stat_bump_invalid(struct load_info *info, int flags); 219 void mod_stat_bump_becoming(struct load_info *info, int flags); 220 221 #else 222 223 #define mod_stat_add_long(name, var) 224 #define mod_stat_inc(name) 225 226 static inline int try_add_failed_module(const char *name, 227 enum fail_dup_mod_reason reason) 228 { 229 return 0; 230 } 231 232 static inline void mod_stat_bump_invalid(struct load_info *info, int flags) 233 { 234 } 235 236 static inline void mod_stat_bump_becoming(struct load_info *info, int flags) 237 { 238 } 239 240 #endif /* CONFIG_MODULE_STATS */ 241 242 #ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS 243 bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret); 244 void kmod_dup_request_announce(char *module_name, int ret); 245 #else 246 static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) 247 { 248 return false; 249 } 250 251 static inline void kmod_dup_request_announce(char *module_name, int ret) 252 { 253 } 254 #endif 255 256 #ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING 257 struct mod_unload_taint { 258 struct list_head list; 259 char name[MODULE_NAME_LEN]; 260 unsigned long taints; 261 u64 count; 262 }; 263 264 int try_add_tainted_module(struct module *mod); 265 void print_unloaded_tainted_modules(void); 266 #else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 267 static inline int try_add_tainted_module(struct module *mod) 268 { 269 return 0; 270 } 271 272 static inline void print_unloaded_tainted_modules(void) 273 { 274 } 275 #endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 276 277 #ifdef CONFIG_MODULE_DECOMPRESS 278 int module_decompress(struct load_info *info, const void *buf, size_t size); 279 void module_decompress_cleanup(struct load_info *info); 280 #else 281 static inline int module_decompress(struct load_info *info, 282 const void *buf, size_t size) 283 { 284 return -EOPNOTSUPP; 285 } 286 287 static inline void module_decompress_cleanup(struct load_info *info) 288 { 289 } 290 #endif 291 292 struct mod_tree_root { 293 #ifdef CONFIG_MODULES_TREE_LOOKUP 294 struct latch_tree_root root; 295 #endif 296 unsigned long addr_min; 297 unsigned long addr_max; 298 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 299 unsigned long data_addr_min; 300 unsigned long data_addr_max; 301 #endif 302 }; 303 304 extern struct mod_tree_root mod_tree; 305 306 #ifdef CONFIG_MODULES_TREE_LOOKUP 307 void mod_tree_insert(struct module *mod); 308 void mod_tree_remove_init(struct module *mod); 309 void mod_tree_remove(struct module *mod); 310 struct module *mod_find(unsigned long addr, struct mod_tree_root *tree); 311 #else /* !CONFIG_MODULES_TREE_LOOKUP */ 312 313 static inline void mod_tree_insert(struct module *mod) { } 314 static inline void mod_tree_remove_init(struct module *mod) { } 315 static inline void mod_tree_remove(struct module *mod) { } 316 static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree) 317 { 318 struct module *mod; 319 320 list_for_each_entry_rcu(mod, &modules, list, 321 lockdep_is_held(&module_mutex)) { 322 if (within_module(addr, mod)) 323 return mod; 324 } 325 326 return NULL; 327 } 328 #endif /* CONFIG_MODULES_TREE_LOOKUP */ 329 330 int module_enable_rodata_ro(const struct module *mod, bool after_init); 331 int module_enable_data_nx(const struct module *mod); 332 int module_enable_text_rox(const struct module *mod); 333 int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, 334 char *secstrings, struct module *mod); 335 336 #ifdef CONFIG_MODULE_SIG 337 int module_sig_check(struct load_info *info, int flags); 338 #else /* !CONFIG_MODULE_SIG */ 339 static inline int module_sig_check(struct load_info *info, int flags) 340 { 341 return 0; 342 } 343 #endif /* !CONFIG_MODULE_SIG */ 344 345 #ifdef CONFIG_DEBUG_KMEMLEAK 346 void kmemleak_load_module(const struct module *mod, const struct load_info *info); 347 #else /* !CONFIG_DEBUG_KMEMLEAK */ 348 static inline void kmemleak_load_module(const struct module *mod, 349 const struct load_info *info) { } 350 #endif /* CONFIG_DEBUG_KMEMLEAK */ 351 352 #ifdef CONFIG_KALLSYMS 353 void init_build_id(struct module *mod, const struct load_info *info); 354 void layout_symtab(struct module *mod, struct load_info *info); 355 void add_kallsyms(struct module *mod, const struct load_info *info); 356 357 static inline bool sect_empty(const Elf_Shdr *sect) 358 { 359 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; 360 } 361 #else /* !CONFIG_KALLSYMS */ 362 static inline void init_build_id(struct module *mod, const struct load_info *info) { } 363 static inline void layout_symtab(struct module *mod, struct load_info *info) { } 364 static inline void add_kallsyms(struct module *mod, const struct load_info *info) { } 365 #endif /* CONFIG_KALLSYMS */ 366 367 #ifdef CONFIG_SYSFS 368 int mod_sysfs_setup(struct module *mod, const struct load_info *info, 369 struct kernel_param *kparam, unsigned int num_params); 370 void mod_sysfs_teardown(struct module *mod); 371 void init_param_lock(struct module *mod); 372 #else /* !CONFIG_SYSFS */ 373 static inline int mod_sysfs_setup(struct module *mod, 374 const struct load_info *info, 375 struct kernel_param *kparam, 376 unsigned int num_params) 377 { 378 return 0; 379 } 380 381 static inline void mod_sysfs_teardown(struct module *mod) { } 382 static inline void init_param_lock(struct module *mod) { } 383 #endif /* CONFIG_SYSFS */ 384 385 #ifdef CONFIG_MODVERSIONS 386 int check_version(const struct load_info *info, 387 const char *symname, struct module *mod, const s32 *crc); 388 void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp, 389 struct kernel_symbol *ks, struct tracepoint * const *tp); 390 int check_modstruct_version(const struct load_info *info, struct module *mod); 391 int same_magic(const char *amagic, const char *bmagic, bool has_crcs); 392 #else /* !CONFIG_MODVERSIONS */ 393 static inline int check_version(const struct load_info *info, 394 const char *symname, 395 struct module *mod, 396 const s32 *crc) 397 { 398 return 1; 399 } 400 401 static inline int check_modstruct_version(const struct load_info *info, 402 struct module *mod) 403 { 404 return 1; 405 } 406 407 static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs) 408 { 409 return strcmp(amagic, bmagic) == 0; 410 } 411 #endif /* CONFIG_MODVERSIONS */ 412