1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Author: Hanlu Li <lihanlu@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 9 #define pr_fmt(fmt) "kmod: " fmt 10 11 #include <linux/moduleloader.h> 12 #include <linux/elf.h> 13 #include <linux/mm.h> 14 #include <linux/numa.h> 15 #include <linux/vmalloc.h> 16 #include <linux/slab.h> 17 #include <linux/fs.h> 18 #include <linux/ftrace.h> 19 #include <linux/string.h> 20 #include <linux/kernel.h> 21 #include <linux/execmem.h> 22 #include <asm/alternative.h> 23 #include <asm/inst.h> 24 #include <asm/unwind.h> 25 26 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 27 { 28 if (*rela_stack_top >= RELA_STACK_DEPTH) 29 return -ENOEXEC; 30 31 rela_stack[(*rela_stack_top)++] = stack_value; 32 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 33 34 return 0; 35 } 36 37 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 38 { 39 if (*rela_stack_top == 0) 40 return -ENOEXEC; 41 42 *stack_value = rela_stack[--(*rela_stack_top)]; 43 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 44 45 return 0; 46 } 47 48 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 49 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 50 { 51 return 0; 52 } 53 54 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 55 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 56 { 57 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 58 return -EINVAL; 59 } 60 61 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 62 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 63 { 64 *location = v; 65 return 0; 66 } 67 68 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 69 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 70 { 71 *(Elf_Addr *)location = v; 72 return 0; 73 } 74 75 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 76 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 77 { 78 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 79 } 80 81 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 82 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 83 { 84 return rela_stack_push(v, rela_stack, rela_stack_top); 85 } 86 87 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 88 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 89 { 90 int err = 0; 91 s64 opr1; 92 93 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 94 if (err) 95 return err; 96 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 97 if (err) 98 return err; 99 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 100 if (err) 101 return err; 102 103 return 0; 104 } 105 106 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, 107 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 108 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 109 { 110 ptrdiff_t offset = (void *)v - (void *)location; 111 112 if (offset >= SZ_128M) 113 v = module_emit_plt_entry(mod, sechdrs, v); 114 115 if (offset < -SZ_128M) 116 v = module_emit_plt_entry(mod, sechdrs, v); 117 118 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 119 } 120 121 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 122 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 123 { 124 int err = 0; 125 s64 opr1, opr2, opr3; 126 127 if (type == R_LARCH_SOP_IF_ELSE) { 128 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 129 if (err) 130 return err; 131 } 132 133 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 134 if (err) 135 return err; 136 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 137 if (err) 138 return err; 139 140 switch (type) { 141 case R_LARCH_SOP_AND: 142 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 143 break; 144 case R_LARCH_SOP_ADD: 145 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 146 break; 147 case R_LARCH_SOP_SUB: 148 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 149 break; 150 case R_LARCH_SOP_SL: 151 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 152 break; 153 case R_LARCH_SOP_SR: 154 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 155 break; 156 case R_LARCH_SOP_IF_ELSE: 157 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 158 break; 159 default: 160 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 161 return -EINVAL; 162 } 163 164 return err; 165 } 166 167 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 168 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 169 { 170 int err = 0; 171 s64 opr1; 172 union loongarch_instruction *insn = (union loongarch_instruction *)location; 173 174 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 175 if (err) 176 return err; 177 178 switch (type) { 179 case R_LARCH_SOP_POP_32_U_10_12: 180 if (!unsigned_imm_check(opr1, 12)) 181 goto overflow; 182 183 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 184 insn->reg2i12_format.immediate = opr1 & 0xfff; 185 return 0; 186 case R_LARCH_SOP_POP_32_S_10_12: 187 if (!signed_imm_check(opr1, 12)) 188 goto overflow; 189 190 insn->reg2i12_format.immediate = opr1 & 0xfff; 191 return 0; 192 case R_LARCH_SOP_POP_32_S_10_16: 193 if (!signed_imm_check(opr1, 16)) 194 goto overflow; 195 196 insn->reg2i16_format.immediate = opr1 & 0xffff; 197 return 0; 198 case R_LARCH_SOP_POP_32_S_10_16_S2: 199 if (opr1 % 4) 200 goto unaligned; 201 202 if (!signed_imm_check(opr1, 18)) 203 goto overflow; 204 205 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff; 206 return 0; 207 case R_LARCH_SOP_POP_32_S_5_20: 208 if (!signed_imm_check(opr1, 20)) 209 goto overflow; 210 211 insn->reg1i20_format.immediate = (opr1) & 0xfffff; 212 return 0; 213 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 214 if (opr1 % 4) 215 goto unaligned; 216 217 if (!signed_imm_check(opr1, 23)) 218 goto overflow; 219 220 opr1 >>= 2; 221 insn->reg1i21_format.immediate_l = opr1 & 0xffff; 222 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f; 223 return 0; 224 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 225 if (opr1 % 4) 226 goto unaligned; 227 228 if (!signed_imm_check(opr1, 28)) 229 goto overflow; 230 231 opr1 >>= 2; 232 insn->reg0i26_format.immediate_l = opr1 & 0xffff; 233 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff; 234 return 0; 235 case R_LARCH_SOP_POP_32_U: 236 if (!unsigned_imm_check(opr1, 32)) 237 goto overflow; 238 239 /* (*(uint32_t *) PC) = opr */ 240 *location = (u32)opr1; 241 return 0; 242 default: 243 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 244 return -EINVAL; 245 } 246 247 overflow: 248 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 249 mod->name, opr1, __func__, type); 250 return -ENOEXEC; 251 252 unaligned: 253 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 254 mod->name, opr1, __func__, type); 255 return -ENOEXEC; 256 } 257 258 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 259 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 260 { 261 switch (type) { 262 case R_LARCH_ADD32: 263 *(s32 *)location += v; 264 return 0; 265 case R_LARCH_ADD64: 266 *(s64 *)location += v; 267 return 0; 268 case R_LARCH_SUB32: 269 *(s32 *)location -= v; 270 return 0; 271 case R_LARCH_SUB64: 272 *(s64 *)location -= v; 273 return 0; 274 default: 275 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 276 return -EINVAL; 277 } 278 } 279 280 static int apply_r_larch_b26(struct module *mod, 281 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 282 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 283 { 284 ptrdiff_t offset = (void *)v - (void *)location; 285 union loongarch_instruction *insn = (union loongarch_instruction *)location; 286 287 if (offset >= SZ_128M) 288 v = module_emit_plt_entry(mod, sechdrs, v); 289 290 if (offset < -SZ_128M) 291 v = module_emit_plt_entry(mod, sechdrs, v); 292 293 offset = (void *)v - (void *)location; 294 295 if (offset & 3) { 296 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 297 mod->name, (long long)offset, type); 298 return -ENOEXEC; 299 } 300 301 if (!signed_imm_check(offset, 28)) { 302 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 303 mod->name, (long long)offset, type); 304 return -ENOEXEC; 305 } 306 307 offset >>= 2; 308 insn->reg0i26_format.immediate_l = offset & 0xffff; 309 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; 310 311 return 0; 312 } 313 314 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 315 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 316 { 317 union loongarch_instruction *insn = (union loongarch_instruction *)location; 318 /* Use s32 for a sign-extension deliberately. */ 319 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 320 (void *)((Elf_Addr)location & ~0xfff); 321 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 322 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 323 324 switch (type) { 325 case R_LARCH_PCALA_LO12: 326 insn->reg2i12_format.immediate = v & 0xfff; 327 break; 328 case R_LARCH_PCALA_HI20: 329 v = offset_hi20 >> 12; 330 insn->reg1i20_format.immediate = v & 0xfffff; 331 break; 332 case R_LARCH_PCALA64_LO20: 333 v = offset_rem >> 32; 334 insn->reg1i20_format.immediate = v & 0xfffff; 335 break; 336 case R_LARCH_PCALA64_HI12: 337 v = offset_rem >> 52; 338 insn->reg2i12_format.immediate = v & 0xfff; 339 break; 340 default: 341 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 342 return -EINVAL; 343 } 344 345 return 0; 346 } 347 348 static int apply_r_larch_got_pc(struct module *mod, 349 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 350 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 351 { 352 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v); 353 354 if (!got) 355 return -EINVAL; 356 357 switch (type) { 358 case R_LARCH_GOT_PC_LO12: 359 type = R_LARCH_PCALA_LO12; 360 break; 361 case R_LARCH_GOT_PC_HI20: 362 type = R_LARCH_PCALA_HI20; 363 break; 364 default: 365 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 366 return -EINVAL; 367 } 368 369 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 370 } 371 372 static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v, 373 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 374 { 375 ptrdiff_t offset = (void *)v - (void *)location; 376 377 *(u32 *)location = offset; 378 return 0; 379 } 380 381 static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v, 382 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 383 { 384 ptrdiff_t offset = (void *)v - (void *)location; 385 386 *(u64 *)location = offset; 387 return 0; 388 } 389 390 /* 391 * reloc_handlers_rela() - Apply a particular relocation to a module 392 * @mod: the module to apply the reloc to 393 * @location: the address at which the reloc is to be applied 394 * @v: the value of the reloc, with addend for RELA-style 395 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 396 * @rela_stac_top: where the stack operation(pop/push) applies to 397 * 398 * Return: 0 upon success, else -ERRNO 399 */ 400 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 401 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 402 403 /* The handlers for known reloc types */ 404 static reloc_rela_handler reloc_rela_handlers[] = { 405 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error, 406 407 [R_LARCH_NONE] = apply_r_larch_none, 408 [R_LARCH_32] = apply_r_larch_32, 409 [R_LARCH_64] = apply_r_larch_64, 410 [R_LARCH_MARK_LA] = apply_r_larch_none, 411 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 412 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 413 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 414 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 415 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 416 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 417 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 418 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 419 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel, 420 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel, 421 }; 422 423 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 424 unsigned int symindex, unsigned int relsec, 425 struct module *mod) 426 { 427 int i, err; 428 unsigned int type; 429 s64 rela_stack[RELA_STACK_DEPTH]; 430 size_t rela_stack_top = 0; 431 reloc_rela_handler handler; 432 void *location; 433 Elf_Addr v; 434 Elf_Sym *sym; 435 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 436 437 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 438 sechdrs[relsec].sh_info); 439 440 rela_stack_top = 0; 441 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 442 /* This is where to make the change */ 443 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 444 /* This is the symbol it is referring to */ 445 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 446 if (IS_ERR_VALUE(sym->st_value)) { 447 /* Ignore unresolved weak symbol */ 448 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 449 continue; 450 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 451 return -ENOENT; 452 } 453 454 type = ELF_R_TYPE(rel[i].r_info); 455 456 if (type < ARRAY_SIZE(reloc_rela_handlers)) 457 handler = reloc_rela_handlers[type]; 458 else 459 handler = NULL; 460 461 if (!handler) { 462 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 463 return -EINVAL; 464 } 465 466 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 467 (int)ELF_R_TYPE(rel[i].r_info), 468 sym->st_value, rel[i].r_addend, (u64)location); 469 470 v = sym->st_value + rel[i].r_addend; 471 switch (type) { 472 case R_LARCH_B26: 473 err = apply_r_larch_b26(mod, sechdrs, location, 474 v, rela_stack, &rela_stack_top, type); 475 break; 476 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12: 477 err = apply_r_larch_got_pc(mod, sechdrs, location, 478 v, rela_stack, &rela_stack_top, type); 479 break; 480 case R_LARCH_SOP_PUSH_PLT_PCREL: 481 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location, 482 v, rela_stack, &rela_stack_top, type); 483 break; 484 default: 485 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 486 } 487 if (err) 488 return err; 489 } 490 491 return 0; 492 } 493 494 static struct execmem_info execmem_info __ro_after_init; 495 496 struct execmem_info __init *execmem_arch_setup(void) 497 { 498 execmem_info = (struct execmem_info){ 499 .ranges = { 500 [EXECMEM_DEFAULT] = { 501 .start = MODULES_VADDR, 502 .end = MODULES_END, 503 .pgprot = PAGE_KERNEL, 504 .alignment = 1, 505 }, 506 }, 507 }; 508 509 return &execmem_info; 510 } 511 512 static void module_init_ftrace_plt(const Elf_Ehdr *hdr, 513 const Elf_Shdr *sechdrs, struct module *mod) 514 { 515 #ifdef CONFIG_DYNAMIC_FTRACE 516 struct plt_entry *ftrace_plts; 517 518 ftrace_plts = (void *)sechdrs->sh_addr; 519 520 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR); 521 522 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) 523 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR); 524 525 mod->arch.ftrace_trampolines = ftrace_plts; 526 #endif 527 } 528 529 int module_finalize(const Elf_Ehdr *hdr, 530 const Elf_Shdr *sechdrs, struct module *mod) 531 { 532 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 533 const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL; 534 535 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 536 if (!strcmp(".altinstructions", secstrs + s->sh_name)) 537 alt = s; 538 if (!strcmp(".orc_unwind", secstrs + s->sh_name)) 539 orc = s; 540 if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name)) 541 orc_ip = s; 542 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name)) 543 ftrace = s; 544 } 545 546 if (alt) 547 apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size); 548 549 if (orc && orc_ip) 550 unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size); 551 552 if (ftrace) 553 module_init_ftrace_plt(hdr, ftrace, mod); 554 555 return 0; 556 } 557