1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* Module internals 3 * 4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org> 7 */ 8 9 #include <linux/elf.h> 10 #include <linux/compiler.h> 11 #include <linux/module.h> 12 #include <linux/mutex.h> 13 #include <linux/rculist.h> 14 #include <linux/rcupdate.h> 15 #include <linux/mm.h> 16 17 #ifndef ARCH_SHF_SMALL 18 #define ARCH_SHF_SMALL 0 19 #endif 20 21 /* 22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this 23 * section. This leaves 28 bits for offset on 32-bit systems, which is 24 * about 256 MiB (WARN_ON_ONCE if we exceed that). 25 */ 26 27 #define SH_ENTSIZE_TYPE_BITS 4 28 #define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS) 29 #define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1) 30 #define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1) 31 32 /* Maximum number of characters written by module_flags() */ 33 #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) 34 35 struct kernel_symbol { 36 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 37 int value_offset; 38 int name_offset; 39 int namespace_offset; 40 #else 41 unsigned long value; 42 const char *name; 43 const char *namespace; 44 #endif 45 }; 46 47 extern struct mutex module_mutex; 48 extern struct list_head modules; 49 50 extern const struct module_attribute *const modinfo_attrs[]; 51 extern const size_t modinfo_attrs_count; 52 53 /* Provided by the linker */ 54 extern const struct kernel_symbol __start___ksymtab[]; 55 extern const struct kernel_symbol __stop___ksymtab[]; 56 extern const struct kernel_symbol __start___ksymtab_gpl[]; 57 extern const struct kernel_symbol __stop___ksymtab_gpl[]; 58 extern const u32 __start___kcrctab[]; 59 extern const u32 __start___kcrctab_gpl[]; 60 61 #define KMOD_PATH_LEN 256 62 extern char modprobe_path[]; 63 64 struct load_info { 65 const char *name; 66 /* pointer to module in temporary copy, freed at end of load_module() */ 67 struct module *mod; 68 Elf_Ehdr *hdr; 69 unsigned long len; 70 Elf_Shdr *sechdrs; 71 char *secstrings, *strtab; 72 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; 73 bool sig_ok; 74 #ifdef CONFIG_KALLSYMS 75 unsigned long mod_kallsyms_init_off; 76 #endif 77 #ifdef CONFIG_MODULE_DECOMPRESS 78 #ifdef CONFIG_MODULE_STATS 79 unsigned long compressed_len; 80 #endif 81 struct page **pages; 82 unsigned int max_pages; 83 unsigned int used_pages; 84 #endif 85 struct { 86 unsigned int sym; 87 unsigned int str; 88 unsigned int mod; 89 unsigned int vers; 90 unsigned int info; 91 unsigned int pcpu; 92 unsigned int vers_ext_crc; 93 unsigned int vers_ext_name; 94 } index; 95 }; 96 97 enum mod_license { 98 NOT_GPL_ONLY, 99 GPL_ONLY, 100 }; 101 102 struct find_symbol_arg { 103 /* Input */ 104 const char *name; 105 bool gplok; 106 bool warn; 107 108 /* Output */ 109 struct module *owner; 110 const u32 *crc; 111 const struct kernel_symbol *sym; 112 enum mod_license license; 113 }; 114 115 int mod_verify_sig(const void *mod, struct load_info *info); 116 int try_to_force_load(struct module *mod, const char *reason); 117 bool find_symbol(struct find_symbol_arg *fsa); 118 struct module *find_module_all(const char *name, size_t len, bool even_unformed); 119 int cmp_name(const void *name, const void *sym); 120 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, 121 Elf_Shdr *sechdr, unsigned int section); 122 char *module_flags(struct module *mod, char *buf, bool show_state); 123 size_t module_flags_taint(unsigned long taints, char *buf); 124 125 char *module_next_tag_pair(char *string, unsigned long *secsize); 126 127 #define for_each_modinfo_entry(entry, info, name) \ 128 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry)) 129 130 static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym) 131 { 132 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS 133 return (unsigned long)offset_to_ptr(&sym->value_offset); 134 #else 135 return sym->value; 136 #endif 137 } 138 139 #ifdef CONFIG_LIVEPATCH 140 int copy_module_elf(struct module *mod, struct load_info *info); 141 void free_module_elf(struct module *mod); 142 #else /* !CONFIG_LIVEPATCH */ 143 static inline int copy_module_elf(struct module *mod, struct load_info *info) 144 { 145 return 0; 146 } 147 148 static inline void free_module_elf(struct module *mod) { } 149 #endif /* CONFIG_LIVEPATCH */ 150 151 static inline bool set_livepatch_module(struct module *mod) 152 { 153 #ifdef CONFIG_LIVEPATCH 154 mod->klp = true; 155 return true; 156 #else 157 return false; 158 #endif 159 } 160 161 /** 162 * enum fail_dup_mod_reason - state at which a duplicate module was detected 163 * 164 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but 165 * we've determined that another module with the same name is already loaded 166 * or being processed on our &modules list. This happens on early_mod_check() 167 * right before layout_and_allocate(). The kernel would have already 168 * vmalloc()'d space for the entire module through finit_module(). If 169 * decompression was used two vmap() spaces were used. These failures can 170 * happen when userspace has not seen the module present on the kernel and 171 * tries to load the module multiple times at same time. 172 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation 173 * checks and the kernel determines that the module was unique and because 174 * of this allocated yet another private kernel copy of the module space in 175 * layout_and_allocate() but after this determined in add_unformed_module() 176 * that another module with the same name is already loaded or being processed. 177 * These failures should be mitigated as much as possible and are indicative 178 * of really fast races in loading modules. Without module decompression 179 * they waste twice as much vmap space. With module decompression three 180 * times the module's size vmap space is wasted. 181 */ 182 enum fail_dup_mod_reason { 183 FAIL_DUP_MOD_BECOMING = 0, 184 FAIL_DUP_MOD_LOAD, 185 }; 186 187 #ifdef CONFIG_MODULE_DEBUGFS 188 extern struct dentry *mod_debugfs_root; 189 #endif 190 191 #ifdef CONFIG_MODULE_STATS 192 193 #define mod_stat_add_long(count, var) atomic_long_add(count, var) 194 #define mod_stat_inc(name) atomic_inc(name) 195 196 extern atomic_long_t total_mod_size; 197 extern atomic_long_t total_text_size; 198 extern atomic_long_t invalid_kread_bytes; 199 extern atomic_long_t invalid_decompress_bytes; 200 201 extern atomic_t modcount; 202 extern atomic_t failed_kreads; 203 extern atomic_t failed_decompress; 204 struct mod_fail_load { 205 struct list_head list; 206 char name[MODULE_NAME_LEN]; 207 atomic_long_t count; 208 unsigned long dup_fail_mask; 209 }; 210 211 int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason); 212 void mod_stat_bump_invalid(struct load_info *info, int flags); 213 void mod_stat_bump_becoming(struct load_info *info, int flags); 214 215 #else 216 217 #define mod_stat_add_long(name, var) 218 #define mod_stat_inc(name) 219 220 static inline int try_add_failed_module(const char *name, 221 enum fail_dup_mod_reason reason) 222 { 223 return 0; 224 } 225 226 static inline void mod_stat_bump_invalid(struct load_info *info, int flags) 227 { 228 } 229 230 static inline void mod_stat_bump_becoming(struct load_info *info, int flags) 231 { 232 } 233 234 #endif /* CONFIG_MODULE_STATS */ 235 236 #ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS 237 bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret); 238 void kmod_dup_request_announce(char *module_name, int ret); 239 #else 240 static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) 241 { 242 return false; 243 } 244 245 static inline void kmod_dup_request_announce(char *module_name, int ret) 246 { 247 } 248 #endif 249 250 #ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING 251 struct mod_unload_taint { 252 struct list_head list; 253 char name[MODULE_NAME_LEN]; 254 unsigned long taints; 255 u64 count; 256 }; 257 258 int try_add_tainted_module(struct module *mod); 259 void print_unloaded_tainted_modules(void); 260 #else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 261 static inline int try_add_tainted_module(struct module *mod) 262 { 263 return 0; 264 } 265 266 static inline void print_unloaded_tainted_modules(void) 267 { 268 } 269 #endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ 270 271 #ifdef CONFIG_MODULE_DECOMPRESS 272 int module_decompress(struct load_info *info, const void *buf, size_t size); 273 void module_decompress_cleanup(struct load_info *info); 274 #else 275 static inline int module_decompress(struct load_info *info, 276 const void *buf, size_t size) 277 { 278 return -EOPNOTSUPP; 279 } 280 281 static inline void module_decompress_cleanup(struct load_info *info) 282 { 283 } 284 #endif 285 286 struct mod_tree_root { 287 #ifdef CONFIG_MODULES_TREE_LOOKUP 288 struct latch_tree_root root; 289 #endif 290 unsigned long addr_min; 291 unsigned long addr_max; 292 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC 293 unsigned long data_addr_min; 294 unsigned long data_addr_max; 295 #endif 296 }; 297 298 extern struct mod_tree_root mod_tree; 299 300 #ifdef CONFIG_MODULES_TREE_LOOKUP 301 void mod_tree_insert(struct module *mod); 302 void mod_tree_remove_init(struct module *mod); 303 void mod_tree_remove(struct module *mod); 304 struct module *mod_find(unsigned long addr, struct mod_tree_root *tree); 305 #else /* !CONFIG_MODULES_TREE_LOOKUP */ 306 307 static inline void mod_tree_insert(struct module *mod) { } 308 static inline void mod_tree_remove_init(struct module *mod) { } 309 static inline void mod_tree_remove(struct module *mod) { } 310 static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree) 311 { 312 struct module *mod; 313 314 list_for_each_entry_rcu(mod, &modules, list, 315 lockdep_is_held(&module_mutex)) { 316 if (within_module(addr, mod)) 317 return mod; 318 } 319 320 return NULL; 321 } 322 #endif /* CONFIG_MODULES_TREE_LOOKUP */ 323 324 int module_enable_rodata_ro(const struct module *mod); 325 int module_enable_rodata_ro_after_init(const struct module *mod); 326 int module_enable_data_nx(const struct module *mod); 327 int module_enable_text_rox(const struct module *mod); 328 int module_enforce_rwx_sections(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, 329 const char *secstrings, 330 const struct module *mod); 331 void module_mark_ro_after_init(const Elf_Ehdr *hdr, Elf_Shdr *sechdrs, 332 const char *secstrings); 333 334 #ifdef CONFIG_MODULE_SIG 335 int module_sig_check(struct load_info *info, int flags); 336 #else /* !CONFIG_MODULE_SIG */ 337 static inline int module_sig_check(struct load_info *info, int flags) 338 { 339 return 0; 340 } 341 #endif /* !CONFIG_MODULE_SIG */ 342 343 #ifdef CONFIG_DEBUG_KMEMLEAK 344 void kmemleak_load_module(const struct module *mod, const struct load_info *info); 345 #else /* !CONFIG_DEBUG_KMEMLEAK */ 346 static inline void kmemleak_load_module(const struct module *mod, 347 const struct load_info *info) { } 348 #endif /* CONFIG_DEBUG_KMEMLEAK */ 349 350 #ifdef CONFIG_KALLSYMS 351 void init_build_id(struct module *mod, const struct load_info *info); 352 void layout_symtab(struct module *mod, struct load_info *info); 353 void add_kallsyms(struct module *mod, const struct load_info *info); 354 355 static inline bool sect_empty(const Elf_Shdr *sect) 356 { 357 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; 358 } 359 #else /* !CONFIG_KALLSYMS */ 360 static inline void init_build_id(struct module *mod, const struct load_info *info) { } 361 static inline void layout_symtab(struct module *mod, struct load_info *info) { } 362 static inline void add_kallsyms(struct module *mod, const struct load_info *info) { } 363 #endif /* CONFIG_KALLSYMS */ 364 365 #ifdef CONFIG_SYSFS 366 int mod_sysfs_setup(struct module *mod, const struct load_info *info, 367 struct kernel_param *kparam, unsigned int num_params); 368 void mod_sysfs_teardown(struct module *mod); 369 void init_param_lock(struct module *mod); 370 #else /* !CONFIG_SYSFS */ 371 static inline int mod_sysfs_setup(struct module *mod, 372 const struct load_info *info, 373 struct kernel_param *kparam, 374 unsigned int num_params) 375 { 376 return 0; 377 } 378 379 static inline void mod_sysfs_teardown(struct module *mod) { } 380 static inline void init_param_lock(struct module *mod) { } 381 #endif /* CONFIG_SYSFS */ 382 383 #ifdef CONFIG_MODVERSIONS 384 int check_version(const struct load_info *info, 385 const char *symname, struct module *mod, const u32 *crc); 386 void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp, 387 struct kernel_symbol *ks, struct tracepoint * const *tp); 388 int check_modstruct_version(const struct load_info *info, struct module *mod); 389 int same_magic(const char *amagic, const char *bmagic, bool has_crcs); 390 struct modversion_info_ext { 391 size_t remaining; 392 const u32 *crc; 393 const char *name; 394 }; 395 void modversion_ext_start(const struct load_info *info, struct modversion_info_ext *ver); 396 void modversion_ext_advance(struct modversion_info_ext *ver); 397 #define for_each_modversion_info_ext(ver, info) \ 398 for (modversion_ext_start(info, &ver); ver.remaining > 0; modversion_ext_advance(&ver)) 399 #else /* !CONFIG_MODVERSIONS */ 400 static inline int check_version(const struct load_info *info, 401 const char *symname, 402 struct module *mod, 403 const u32 *crc) 404 { 405 return 1; 406 } 407 408 static inline int check_modstruct_version(const struct load_info *info, 409 struct module *mod) 410 { 411 return 1; 412 } 413 414 static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs) 415 { 416 return strcmp(amagic, bmagic) == 0; 417 } 418 #endif /* CONFIG_MODVERSIONS */ 419