1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Author: Hanlu Li <lihanlu@loongson.cn> 4 * Huacai Chen <chenhuacai@loongson.cn> 5 * 6 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 7 */ 8 9 #define pr_fmt(fmt) "kmod: " fmt 10 11 #include <linux/moduleloader.h> 12 #include <linux/elf.h> 13 #include <linux/mm.h> 14 #include <linux/numa.h> 15 #include <linux/vmalloc.h> 16 #include <linux/slab.h> 17 #include <linux/fs.h> 18 #include <linux/ftrace.h> 19 #include <linux/string.h> 20 #include <linux/kernel.h> 21 #include <asm/alternative.h> 22 #include <asm/inst.h> 23 #include <asm/unwind.h> 24 25 static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 26 { 27 if (*rela_stack_top >= RELA_STACK_DEPTH) 28 return -ENOEXEC; 29 30 rela_stack[(*rela_stack_top)++] = stack_value; 31 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 32 33 return 0; 34 } 35 36 static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 37 { 38 if (*rela_stack_top == 0) 39 return -ENOEXEC; 40 41 *stack_value = rela_stack[--(*rela_stack_top)]; 42 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 43 44 return 0; 45 } 46 47 static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 48 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 49 { 50 return 0; 51 } 52 53 static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 54 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 55 { 56 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 57 return -EINVAL; 58 } 59 60 static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 61 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 62 { 63 *location = v; 64 return 0; 65 } 66 67 static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 68 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 69 { 70 *(Elf_Addr *)location = v; 71 return 0; 72 } 73 74 static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 75 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 76 { 77 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 78 } 79 80 static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 81 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 82 { 83 return rela_stack_push(v, rela_stack, rela_stack_top); 84 } 85 86 static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 87 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 88 { 89 int err = 0; 90 s64 opr1; 91 92 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 93 if (err) 94 return err; 95 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 96 if (err) 97 return err; 98 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 99 if (err) 100 return err; 101 102 return 0; 103 } 104 105 static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, 106 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 107 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 108 { 109 ptrdiff_t offset = (void *)v - (void *)location; 110 111 if (offset >= SZ_128M) 112 v = module_emit_plt_entry(mod, sechdrs, v); 113 114 if (offset < -SZ_128M) 115 v = module_emit_plt_entry(mod, sechdrs, v); 116 117 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 118 } 119 120 static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 121 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 122 { 123 int err = 0; 124 s64 opr1, opr2, opr3; 125 126 if (type == R_LARCH_SOP_IF_ELSE) { 127 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 128 if (err) 129 return err; 130 } 131 132 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 133 if (err) 134 return err; 135 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 136 if (err) 137 return err; 138 139 switch (type) { 140 case R_LARCH_SOP_AND: 141 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 142 break; 143 case R_LARCH_SOP_ADD: 144 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 145 break; 146 case R_LARCH_SOP_SUB: 147 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 148 break; 149 case R_LARCH_SOP_SL: 150 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 151 break; 152 case R_LARCH_SOP_SR: 153 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 154 break; 155 case R_LARCH_SOP_IF_ELSE: 156 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 157 break; 158 default: 159 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 160 return -EINVAL; 161 } 162 163 return err; 164 } 165 166 static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 167 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 168 { 169 int err = 0; 170 s64 opr1; 171 union loongarch_instruction *insn = (union loongarch_instruction *)location; 172 173 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 174 if (err) 175 return err; 176 177 switch (type) { 178 case R_LARCH_SOP_POP_32_U_10_12: 179 if (!unsigned_imm_check(opr1, 12)) 180 goto overflow; 181 182 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 183 insn->reg2i12_format.immediate = opr1 & 0xfff; 184 return 0; 185 case R_LARCH_SOP_POP_32_S_10_12: 186 if (!signed_imm_check(opr1, 12)) 187 goto overflow; 188 189 insn->reg2i12_format.immediate = opr1 & 0xfff; 190 return 0; 191 case R_LARCH_SOP_POP_32_S_10_16: 192 if (!signed_imm_check(opr1, 16)) 193 goto overflow; 194 195 insn->reg2i16_format.immediate = opr1 & 0xffff; 196 return 0; 197 case R_LARCH_SOP_POP_32_S_10_16_S2: 198 if (opr1 % 4) 199 goto unaligned; 200 201 if (!signed_imm_check(opr1, 18)) 202 goto overflow; 203 204 insn->reg2i16_format.immediate = (opr1 >> 2) & 0xffff; 205 return 0; 206 case R_LARCH_SOP_POP_32_S_5_20: 207 if (!signed_imm_check(opr1, 20)) 208 goto overflow; 209 210 insn->reg1i20_format.immediate = (opr1) & 0xfffff; 211 return 0; 212 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 213 if (opr1 % 4) 214 goto unaligned; 215 216 if (!signed_imm_check(opr1, 23)) 217 goto overflow; 218 219 opr1 >>= 2; 220 insn->reg1i21_format.immediate_l = opr1 & 0xffff; 221 insn->reg1i21_format.immediate_h = (opr1 >> 16) & 0x1f; 222 return 0; 223 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 224 if (opr1 % 4) 225 goto unaligned; 226 227 if (!signed_imm_check(opr1, 28)) 228 goto overflow; 229 230 opr1 >>= 2; 231 insn->reg0i26_format.immediate_l = opr1 & 0xffff; 232 insn->reg0i26_format.immediate_h = (opr1 >> 16) & 0x3ff; 233 return 0; 234 case R_LARCH_SOP_POP_32_U: 235 if (!unsigned_imm_check(opr1, 32)) 236 goto overflow; 237 238 /* (*(uint32_t *) PC) = opr */ 239 *location = (u32)opr1; 240 return 0; 241 default: 242 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 243 return -EINVAL; 244 } 245 246 overflow: 247 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 248 mod->name, opr1, __func__, type); 249 return -ENOEXEC; 250 251 unaligned: 252 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 253 mod->name, opr1, __func__, type); 254 return -ENOEXEC; 255 } 256 257 static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 258 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 259 { 260 switch (type) { 261 case R_LARCH_ADD32: 262 *(s32 *)location += v; 263 return 0; 264 case R_LARCH_ADD64: 265 *(s64 *)location += v; 266 return 0; 267 case R_LARCH_SUB32: 268 *(s32 *)location -= v; 269 return 0; 270 case R_LARCH_SUB64: 271 *(s64 *)location -= v; 272 return 0; 273 default: 274 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 275 return -EINVAL; 276 } 277 } 278 279 static int apply_r_larch_b26(struct module *mod, 280 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 281 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 282 { 283 ptrdiff_t offset = (void *)v - (void *)location; 284 union loongarch_instruction *insn = (union loongarch_instruction *)location; 285 286 if (offset >= SZ_128M) 287 v = module_emit_plt_entry(mod, sechdrs, v); 288 289 if (offset < -SZ_128M) 290 v = module_emit_plt_entry(mod, sechdrs, v); 291 292 offset = (void *)v - (void *)location; 293 294 if (offset & 3) { 295 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 296 mod->name, (long long)offset, type); 297 return -ENOEXEC; 298 } 299 300 if (!signed_imm_check(offset, 28)) { 301 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 302 mod->name, (long long)offset, type); 303 return -ENOEXEC; 304 } 305 306 offset >>= 2; 307 insn->reg0i26_format.immediate_l = offset & 0xffff; 308 insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; 309 310 return 0; 311 } 312 313 static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 314 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 315 { 316 union loongarch_instruction *insn = (union loongarch_instruction *)location; 317 /* Use s32 for a sign-extension deliberately. */ 318 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 319 (void *)((Elf_Addr)location & ~0xfff); 320 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 321 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 322 323 switch (type) { 324 case R_LARCH_PCALA_LO12: 325 insn->reg2i12_format.immediate = v & 0xfff; 326 break; 327 case R_LARCH_PCALA_HI20: 328 v = offset_hi20 >> 12; 329 insn->reg1i20_format.immediate = v & 0xfffff; 330 break; 331 case R_LARCH_PCALA64_LO20: 332 v = offset_rem >> 32; 333 insn->reg1i20_format.immediate = v & 0xfffff; 334 break; 335 case R_LARCH_PCALA64_HI12: 336 v = offset_rem >> 52; 337 insn->reg2i12_format.immediate = v & 0xfff; 338 break; 339 default: 340 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 341 return -EINVAL; 342 } 343 344 return 0; 345 } 346 347 static int apply_r_larch_got_pc(struct module *mod, 348 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 349 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 350 { 351 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v); 352 353 if (!got) 354 return -EINVAL; 355 356 switch (type) { 357 case R_LARCH_GOT_PC_LO12: 358 type = R_LARCH_PCALA_LO12; 359 break; 360 case R_LARCH_GOT_PC_HI20: 361 type = R_LARCH_PCALA_HI20; 362 break; 363 default: 364 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 365 return -EINVAL; 366 } 367 368 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 369 } 370 371 static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v, 372 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 373 { 374 ptrdiff_t offset = (void *)v - (void *)location; 375 376 *(u32 *)location = offset; 377 return 0; 378 } 379 380 static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v, 381 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 382 { 383 ptrdiff_t offset = (void *)v - (void *)location; 384 385 *(u64 *)location = offset; 386 return 0; 387 } 388 389 /* 390 * reloc_handlers_rela() - Apply a particular relocation to a module 391 * @mod: the module to apply the reloc to 392 * @location: the address at which the reloc is to be applied 393 * @v: the value of the reloc, with addend for RELA-style 394 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 395 * @rela_stac_top: where the stack operation(pop/push) applies to 396 * 397 * Return: 0 upon success, else -ERRNO 398 */ 399 typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 400 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 401 402 /* The handlers for known reloc types */ 403 static reloc_rela_handler reloc_rela_handlers[] = { 404 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error, 405 406 [R_LARCH_NONE] = apply_r_larch_none, 407 [R_LARCH_32] = apply_r_larch_32, 408 [R_LARCH_64] = apply_r_larch_64, 409 [R_LARCH_MARK_LA] = apply_r_larch_none, 410 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 411 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 412 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 413 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 414 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 415 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 416 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 417 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 418 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel, 419 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel, 420 }; 421 422 int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 423 unsigned int symindex, unsigned int relsec, 424 struct module *mod) 425 { 426 int i, err; 427 unsigned int type; 428 s64 rela_stack[RELA_STACK_DEPTH]; 429 size_t rela_stack_top = 0; 430 reloc_rela_handler handler; 431 void *location; 432 Elf_Addr v; 433 Elf_Sym *sym; 434 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 435 436 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 437 sechdrs[relsec].sh_info); 438 439 rela_stack_top = 0; 440 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 441 /* This is where to make the change */ 442 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 443 /* This is the symbol it is referring to */ 444 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 445 if (IS_ERR_VALUE(sym->st_value)) { 446 /* Ignore unresolved weak symbol */ 447 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 448 continue; 449 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 450 return -ENOENT; 451 } 452 453 type = ELF_R_TYPE(rel[i].r_info); 454 455 if (type < ARRAY_SIZE(reloc_rela_handlers)) 456 handler = reloc_rela_handlers[type]; 457 else 458 handler = NULL; 459 460 if (!handler) { 461 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 462 return -EINVAL; 463 } 464 465 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 466 (int)ELF_R_TYPE(rel[i].r_info), 467 sym->st_value, rel[i].r_addend, (u64)location); 468 469 v = sym->st_value + rel[i].r_addend; 470 switch (type) { 471 case R_LARCH_B26: 472 err = apply_r_larch_b26(mod, sechdrs, location, 473 v, rela_stack, &rela_stack_top, type); 474 break; 475 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12: 476 err = apply_r_larch_got_pc(mod, sechdrs, location, 477 v, rela_stack, &rela_stack_top, type); 478 break; 479 case R_LARCH_SOP_PUSH_PLT_PCREL: 480 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location, 481 v, rela_stack, &rela_stack_top, type); 482 break; 483 default: 484 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 485 } 486 if (err) 487 return err; 488 } 489 490 return 0; 491 } 492 493 static void module_init_ftrace_plt(const Elf_Ehdr *hdr, 494 const Elf_Shdr *sechdrs, struct module *mod) 495 { 496 #ifdef CONFIG_DYNAMIC_FTRACE 497 struct plt_entry *ftrace_plts; 498 499 ftrace_plts = (void *)sechdrs->sh_addr; 500 501 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR); 502 503 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) 504 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR); 505 506 mod->arch.ftrace_trampolines = ftrace_plts; 507 #endif 508 } 509 510 int module_finalize(const Elf_Ehdr *hdr, 511 const Elf_Shdr *sechdrs, struct module *mod) 512 { 513 const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 514 const Elf_Shdr *s, *alt = NULL, *orc = NULL, *orc_ip = NULL, *ftrace = NULL; 515 516 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 517 if (!strcmp(".altinstructions", secstrs + s->sh_name)) 518 alt = s; 519 if (!strcmp(".orc_unwind", secstrs + s->sh_name)) 520 orc = s; 521 if (!strcmp(".orc_unwind_ip", secstrs + s->sh_name)) 522 orc_ip = s; 523 if (!strcmp(".ftrace_trampoline", secstrs + s->sh_name)) 524 ftrace = s; 525 } 526 527 if (alt) 528 apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size); 529 530 if (orc && orc_ip) 531 unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size); 532 533 if (ftrace) 534 module_init_ftrace_plt(hdr, ftrace, mod); 535 536 return 0; 537 } 538