1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/kernel/module.c 4 * 5 * Copyright (C) 2002 Russell King. 6 * Modified for nommu by Hyok S. Choi 7 * 8 * Module allocation method suggested by Andi Kleen. 9 */ 10 #include <linux/module.h> 11 #include <linux/moduleloader.h> 12 #include <linux/kernel.h> 13 #include <linux/mm.h> 14 #include <linux/elf.h> 15 #include <linux/vmalloc.h> 16 #include <linux/fs.h> 17 #include <linux/string.h> 18 #include <linux/gfp.h> 19 #include <linux/execmem.h> 20 21 #include <asm/sections.h> 22 #include <asm/smp_plat.h> 23 #include <asm/unwind.h> 24 #include <asm/opcodes.h> 25 26 #ifdef CONFIG_XIP_KERNEL 27 /* 28 * The XIP kernel text is mapped in the module area for modules and 29 * some other stuff to work without any indirect relocations. 30 * MODULES_VADDR is redefined here and not in asm/memory.h to avoid 31 * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. 32 */ 33 #undef MODULES_VADDR 34 #define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) 35 #endif 36 37 #ifdef CONFIG_MMU 38 static struct execmem_info execmem_info __ro_after_init; 39 40 struct execmem_info __init *execmem_arch_setup(void) 41 { 42 unsigned long fallback_start = 0, fallback_end = 0; 43 44 if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) { 45 fallback_start = VMALLOC_START; 46 fallback_end = VMALLOC_END; 47 } 48 49 execmem_info = (struct execmem_info){ 50 .ranges = { 51 [EXECMEM_DEFAULT] = { 52 .start = MODULES_VADDR, 53 .end = MODULES_END, 54 .pgprot = PAGE_KERNEL_EXEC, 55 .alignment = 1, 56 .fallback_start = fallback_start, 57 .fallback_end = fallback_end, 58 }, 59 }, 60 }; 61 62 return &execmem_info; 63 } 64 #endif 65 66 bool module_init_section(const char *name) 67 { 68 return strstarts(name, ".init") || 69 strstarts(name, ".ARM.extab.init") || 70 strstarts(name, ".ARM.exidx.init"); 71 } 72 73 bool module_exit_section(const char *name) 74 { 75 return strstarts(name, ".exit") || 76 strstarts(name, ".ARM.extab.exit") || 77 strstarts(name, ".ARM.exidx.exit"); 78 } 79 80 #ifdef CONFIG_ARM_HAS_GROUP_RELOCS 81 /* 82 * This implements the partitioning algorithm for group relocations as 83 * documented in the ARM AArch32 ELF psABI (IHI 0044). 84 * 85 * A single PC-relative symbol reference is divided in up to 3 add or subtract 86 * operations, where the final one could be incorporated into a load/store 87 * instruction with immediate offset. E.g., 88 * 89 * ADD Rd, PC, #... or ADD Rd, PC, #... 90 * ADD Rd, Rd, #... ADD Rd, Rd, #... 91 * LDR Rd, [Rd, #...] ADD Rd, Rd, #... 92 * 93 * The latter has a guaranteed range of only 16 MiB (3x8 == 24 bits), so it is 94 * of limited use in the kernel. However, the ADD/ADD/LDR combo has a range of 95 * -/+ 256 MiB, (2x8 + 12 == 28 bits), which means it has sufficient range for 96 * any in-kernel symbol reference (unless module PLTs are being used). 97 * 98 * The main advantage of this approach over the typical pattern using a literal 99 * load is that literal loads may miss in the D-cache, and generally lead to 100 * lower cache efficiency for variables that are referenced often from many 101 * different places in the code. 102 */ 103 static u32 get_group_rem(u32 group, u32 *offset) 104 { 105 u32 val = *offset; 106 u32 shift; 107 do { 108 shift = val ? (31 - __fls(val)) & ~1 : 32; 109 *offset = val; 110 if (!val) 111 break; 112 val &= 0xffffff >> shift; 113 } while (group--); 114 return shift; 115 } 116 #endif 117 118 int 119 apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, 120 unsigned int relindex, struct module *module) 121 { 122 Elf32_Shdr *symsec = sechdrs + symindex; 123 Elf32_Shdr *relsec = sechdrs + relindex; 124 Elf32_Shdr *dstsec = sechdrs + relsec->sh_info; 125 Elf32_Rel *rel = (void *)relsec->sh_addr; 126 unsigned int i; 127 128 for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rel); i++, rel++) { 129 unsigned long loc; 130 Elf32_Sym *sym; 131 const char *symname; 132 #ifdef CONFIG_ARM_HAS_GROUP_RELOCS 133 u32 shift, group = 1; 134 #endif 135 s32 offset; 136 u32 tmp; 137 #ifdef CONFIG_THUMB2_KERNEL 138 u32 upper, lower, sign, j1, j2; 139 #endif 140 141 offset = ELF32_R_SYM(rel->r_info); 142 if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) { 143 pr_err("%s: section %u reloc %u: bad relocation sym offset\n", 144 module->name, relindex, i); 145 return -ENOEXEC; 146 } 147 148 sym = ((Elf32_Sym *)symsec->sh_addr) + offset; 149 symname = strtab + sym->st_name; 150 151 if (rel->r_offset < 0 || rel->r_offset > dstsec->sh_size - sizeof(u32)) { 152 pr_err("%s: section %u reloc %u sym '%s': out of bounds relocation, offset %d size %u\n", 153 module->name, relindex, i, symname, 154 rel->r_offset, dstsec->sh_size); 155 return -ENOEXEC; 156 } 157 158 loc = dstsec->sh_addr + rel->r_offset; 159 160 switch (ELF32_R_TYPE(rel->r_info)) { 161 case R_ARM_NONE: 162 /* ignore */ 163 break; 164 165 case R_ARM_ABS32: 166 case R_ARM_TARGET1: 167 *(u32 *)loc += sym->st_value; 168 break; 169 170 case R_ARM_PC24: 171 case R_ARM_CALL: 172 case R_ARM_JUMP24: 173 if (sym->st_value & 3) { 174 pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (ARM -> Thumb)\n", 175 module->name, relindex, i, symname); 176 return -ENOEXEC; 177 } 178 179 offset = __mem_to_opcode_arm(*(u32 *)loc); 180 offset = (offset & 0x00ffffff) << 2; 181 offset = sign_extend32(offset, 25); 182 183 offset += sym->st_value - loc; 184 185 /* 186 * Route through a PLT entry if 'offset' exceeds the 187 * supported range. Note that 'offset + loc + 8' 188 * contains the absolute jump target, i.e., 189 * @sym + addend, corrected for the +8 PC bias. 190 */ 191 if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && 192 (offset <= (s32)0xfe000000 || 193 offset >= (s32)0x02000000)) 194 offset = get_module_plt(module, loc, 195 offset + loc + 8) 196 - loc - 8; 197 198 if (offset <= (s32)0xfe000000 || 199 offset >= (s32)0x02000000) { 200 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", 201 module->name, relindex, i, symname, 202 ELF32_R_TYPE(rel->r_info), loc, 203 sym->st_value); 204 return -ENOEXEC; 205 } 206 207 offset >>= 2; 208 offset &= 0x00ffffff; 209 210 *(u32 *)loc &= __opcode_to_mem_arm(0xff000000); 211 *(u32 *)loc |= __opcode_to_mem_arm(offset); 212 break; 213 214 case R_ARM_V4BX: 215 /* Preserve Rm and the condition code. Alter 216 * other bits to re-code instruction as 217 * MOV PC,Rm. 218 */ 219 *(u32 *)loc &= __opcode_to_mem_arm(0xf000000f); 220 *(u32 *)loc |= __opcode_to_mem_arm(0x01a0f000); 221 break; 222 223 case R_ARM_PREL31: 224 offset = (*(s32 *)loc << 1) >> 1; /* sign extend */ 225 offset += sym->st_value - loc; 226 if (offset >= 0x40000000 || offset < -0x40000000) { 227 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", 228 module->name, relindex, i, symname, 229 ELF32_R_TYPE(rel->r_info), loc, 230 sym->st_value); 231 return -ENOEXEC; 232 } 233 *(u32 *)loc &= 0x80000000; 234 *(u32 *)loc |= offset & 0x7fffffff; 235 break; 236 237 case R_ARM_REL32: 238 *(u32 *)loc += sym->st_value - loc; 239 break; 240 241 case R_ARM_MOVW_ABS_NC: 242 case R_ARM_MOVT_ABS: 243 case R_ARM_MOVW_PREL_NC: 244 case R_ARM_MOVT_PREL: 245 offset = tmp = __mem_to_opcode_arm(*(u32 *)loc); 246 offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff); 247 offset = sign_extend32(offset, 15); 248 249 offset += sym->st_value; 250 if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL || 251 ELF32_R_TYPE(rel->r_info) == R_ARM_MOVW_PREL_NC) 252 offset -= loc; 253 if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_ABS || 254 ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL) 255 offset >>= 16; 256 257 tmp &= 0xfff0f000; 258 tmp |= ((offset & 0xf000) << 4) | 259 (offset & 0x0fff); 260 261 *(u32 *)loc = __opcode_to_mem_arm(tmp); 262 break; 263 264 #ifdef CONFIG_ARM_HAS_GROUP_RELOCS 265 case R_ARM_ALU_PC_G0_NC: 266 group = 0; 267 fallthrough; 268 case R_ARM_ALU_PC_G1_NC: 269 tmp = __mem_to_opcode_arm(*(u32 *)loc); 270 offset = ror32(tmp & 0xff, (tmp & 0xf00) >> 7); 271 if (tmp & BIT(22)) 272 offset = -offset; 273 offset += sym->st_value - loc; 274 if (offset < 0) { 275 offset = -offset; 276 tmp = (tmp & ~BIT(23)) | BIT(22); // SUB opcode 277 } else { 278 tmp = (tmp & ~BIT(22)) | BIT(23); // ADD opcode 279 } 280 281 shift = get_group_rem(group, &offset); 282 if (shift < 24) { 283 offset >>= 24 - shift; 284 offset |= (shift + 8) << 7; 285 } 286 *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); 287 break; 288 289 case R_ARM_LDR_PC_G2: 290 tmp = __mem_to_opcode_arm(*(u32 *)loc); 291 offset = tmp & 0xfff; 292 if (~tmp & BIT(23)) // U bit cleared? 293 offset = -offset; 294 offset += sym->st_value - loc; 295 if (offset < 0) { 296 offset = -offset; 297 tmp &= ~BIT(23); // clear U bit 298 } else { 299 tmp |= BIT(23); // set U bit 300 } 301 get_group_rem(2, &offset); 302 303 if (offset > 0xfff) { 304 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", 305 module->name, relindex, i, symname, 306 ELF32_R_TYPE(rel->r_info), loc, 307 sym->st_value); 308 return -ENOEXEC; 309 } 310 *(u32 *)loc = __opcode_to_mem_arm((tmp & ~0xfff) | offset); 311 break; 312 #endif 313 #ifdef CONFIG_THUMB2_KERNEL 314 case R_ARM_THM_CALL: 315 case R_ARM_THM_JUMP24: 316 /* 317 * For function symbols, only Thumb addresses are 318 * allowed (no interworking). 319 * 320 * For non-function symbols, the destination 321 * has no specific ARM/Thumb disposition, so 322 * the branch is resolved under the assumption 323 * that interworking is not required. 324 */ 325 if (ELF32_ST_TYPE(sym->st_info) == STT_FUNC && 326 !(sym->st_value & 1)) { 327 pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (Thumb -> ARM)\n", 328 module->name, relindex, i, symname); 329 return -ENOEXEC; 330 } 331 332 upper = __mem_to_opcode_thumb16(*(u16 *)loc); 333 lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2)); 334 335 /* 336 * 25 bit signed address range (Thumb-2 BL and B.W 337 * instructions): 338 * S:I1:I2:imm10:imm11:0 339 * where: 340 * S = upper[10] = offset[24] 341 * I1 = ~(J1 ^ S) = offset[23] 342 * I2 = ~(J2 ^ S) = offset[22] 343 * imm10 = upper[9:0] = offset[21:12] 344 * imm11 = lower[10:0] = offset[11:1] 345 * J1 = lower[13] 346 * J2 = lower[11] 347 */ 348 sign = (upper >> 10) & 1; 349 j1 = (lower >> 13) & 1; 350 j2 = (lower >> 11) & 1; 351 offset = (sign << 24) | ((~(j1 ^ sign) & 1) << 23) | 352 ((~(j2 ^ sign) & 1) << 22) | 353 ((upper & 0x03ff) << 12) | 354 ((lower & 0x07ff) << 1); 355 offset = sign_extend32(offset, 24); 356 offset += sym->st_value - loc; 357 358 /* 359 * Route through a PLT entry if 'offset' exceeds the 360 * supported range. 361 */ 362 if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS) && 363 (offset <= (s32)0xff000000 || 364 offset >= (s32)0x01000000)) 365 offset = get_module_plt(module, loc, 366 offset + loc + 4) 367 - loc - 4; 368 369 if (offset <= (s32)0xff000000 || 370 offset >= (s32)0x01000000) { 371 pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", 372 module->name, relindex, i, symname, 373 ELF32_R_TYPE(rel->r_info), loc, 374 sym->st_value); 375 return -ENOEXEC; 376 } 377 378 sign = (offset >> 24) & 1; 379 j1 = sign ^ (~(offset >> 23) & 1); 380 j2 = sign ^ (~(offset >> 22) & 1); 381 upper = (u16)((upper & 0xf800) | (sign << 10) | 382 ((offset >> 12) & 0x03ff)); 383 lower = (u16)((lower & 0xd000) | 384 (j1 << 13) | (j2 << 11) | 385 ((offset >> 1) & 0x07ff)); 386 387 *(u16 *)loc = __opcode_to_mem_thumb16(upper); 388 *(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower); 389 break; 390 391 case R_ARM_THM_MOVW_ABS_NC: 392 case R_ARM_THM_MOVT_ABS: 393 case R_ARM_THM_MOVW_PREL_NC: 394 case R_ARM_THM_MOVT_PREL: 395 upper = __mem_to_opcode_thumb16(*(u16 *)loc); 396 lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2)); 397 398 /* 399 * MOVT/MOVW instructions encoding in Thumb-2: 400 * 401 * i = upper[10] 402 * imm4 = upper[3:0] 403 * imm3 = lower[14:12] 404 * imm8 = lower[7:0] 405 * 406 * imm16 = imm4:i:imm3:imm8 407 */ 408 offset = ((upper & 0x000f) << 12) | 409 ((upper & 0x0400) << 1) | 410 ((lower & 0x7000) >> 4) | (lower & 0x00ff); 411 offset = sign_extend32(offset, 15); 412 offset += sym->st_value; 413 414 if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL || 415 ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVW_PREL_NC) 416 offset -= loc; 417 if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_ABS || 418 ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL) 419 offset >>= 16; 420 421 upper = (u16)((upper & 0xfbf0) | 422 ((offset & 0xf000) >> 12) | 423 ((offset & 0x0800) >> 1)); 424 lower = (u16)((lower & 0x8f00) | 425 ((offset & 0x0700) << 4) | 426 (offset & 0x00ff)); 427 *(u16 *)loc = __opcode_to_mem_thumb16(upper); 428 *(u16 *)(loc + 2) = __opcode_to_mem_thumb16(lower); 429 break; 430 #endif 431 432 default: 433 pr_err("%s: unknown relocation: %u\n", 434 module->name, ELF32_R_TYPE(rel->r_info)); 435 return -ENOEXEC; 436 } 437 } 438 return 0; 439 } 440 441 struct mod_unwind_map { 442 const Elf_Shdr *unw_sec; 443 const Elf_Shdr *txt_sec; 444 }; 445 446 static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr, 447 const Elf_Shdr *sechdrs, const char *name) 448 { 449 const Elf_Shdr *s, *se; 450 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 451 452 for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) 453 if (strcmp(name, secstrs + s->sh_name) == 0) 454 return s; 455 456 return NULL; 457 } 458 459 extern void fixup_pv_table(const void *, unsigned long); 460 extern void fixup_smp(const void *, unsigned long); 461 462 int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, 463 struct module *mod) 464 { 465 const Elf_Shdr *s = NULL; 466 #ifdef CONFIG_ARM_UNWIND 467 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 468 const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum; 469 struct list_head *unwind_list = &mod->arch.unwind_list; 470 471 INIT_LIST_HEAD(unwind_list); 472 mod->arch.init_table = NULL; 473 474 for (s = sechdrs; s < sechdrs_end; s++) { 475 const char *secname = secstrs + s->sh_name; 476 const char *txtname; 477 const Elf_Shdr *txt_sec; 478 479 if (!(s->sh_flags & SHF_ALLOC) || 480 s->sh_type != ELF_SECTION_UNWIND) 481 continue; 482 483 if (!strcmp(".ARM.exidx", secname)) 484 txtname = ".text"; 485 else 486 txtname = secname + strlen(".ARM.exidx"); 487 txt_sec = find_mod_section(hdr, sechdrs, txtname); 488 489 if (txt_sec) { 490 struct unwind_table *table = 491 unwind_table_add(s->sh_addr, 492 s->sh_size, 493 txt_sec->sh_addr, 494 txt_sec->sh_size); 495 496 list_add(&table->mod_list, unwind_list); 497 498 /* save init table for module_arch_freeing_init */ 499 if (strcmp(".ARM.exidx.init.text", secname) == 0) 500 mod->arch.init_table = table; 501 } 502 } 503 #endif 504 #ifdef CONFIG_ARM_PATCH_PHYS_VIRT 505 s = find_mod_section(hdr, sechdrs, ".pv_table"); 506 if (s) 507 fixup_pv_table((void *)s->sh_addr, s->sh_size); 508 #endif 509 s = find_mod_section(hdr, sechdrs, ".alt.smp.init"); 510 if (s && !is_smp()) 511 #ifdef CONFIG_SMP_ON_UP 512 fixup_smp((void *)s->sh_addr, s->sh_size); 513 #else 514 return -EINVAL; 515 #endif 516 return 0; 517 } 518 519 void 520 module_arch_cleanup(struct module *mod) 521 { 522 #ifdef CONFIG_ARM_UNWIND 523 struct unwind_table *tmp; 524 struct unwind_table *n; 525 526 list_for_each_entry_safe(tmp, n, 527 &mod->arch.unwind_list, mod_list) { 528 list_del(&tmp->mod_list); 529 unwind_table_del(tmp); 530 } 531 mod->arch.init_table = NULL; 532 #endif 533 } 534 535 void __weak module_arch_freeing_init(struct module *mod) 536 { 537 #ifdef CONFIG_ARM_UNWIND 538 struct unwind_table *init = mod->arch.init_table; 539 540 if (init) { 541 mod->arch.init_table = NULL; 542 list_del(&init->mod_list); 543 unwind_table_del(init); 544 } 545 #endif 546 } 547