1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Kernel module help for PPC64. 3 Copyright (C) 2001, 2003 Rusty Russell IBM Corporation. 4 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/module.h> 10 #include <linux/elf.h> 11 #include <linux/moduleloader.h> 12 #include <linux/err.h> 13 #include <linux/vmalloc.h> 14 #include <linux/ftrace.h> 15 #include <linux/bug.h> 16 #include <linux/uaccess.h> 17 #include <linux/kernel.h> 18 #include <asm/module.h> 19 #include <asm/firmware.h> 20 #include <asm/text-patching.h> 21 #include <linux/sort.h> 22 #include <asm/setup.h> 23 #include <asm/sections.h> 24 #include <asm/inst.h> 25 26 /* FIXME: We don't do .init separately. To do this, we'd need to have 27 a separate r2 value in the init and core section, and stub between 28 them, too. 29 30 Using a magic allocator which places modules within 32MB solves 31 this, and makes other things simpler. Anton? 32 --RR. */ 33 34 bool module_elf_check_arch(Elf_Ehdr *hdr) 35 { 36 unsigned long abi_level = hdr->e_flags & 0x3; 37 38 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2)) 39 return abi_level == 2; 40 else 41 return abi_level < 2; 42 } 43 44 #ifdef CONFIG_PPC64_ELF_ABI_V2 45 46 static func_desc_t func_desc(unsigned long addr) 47 { 48 func_desc_t desc = { 49 .addr = addr, 50 }; 51 52 return desc; 53 } 54 55 /* PowerPC64 specific values for the Elf64_Sym st_other field. */ 56 #define STO_PPC64_LOCAL_BIT 5 57 #define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT) 58 #define PPC64_LOCAL_ENTRY_OFFSET(other) \ 59 (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2) 60 61 static unsigned int local_entry_offset(const Elf64_Sym *sym) 62 { 63 /* sym->st_other indicates offset to local entry point 64 * (otherwise it will assume r12 is the address of the start 65 * of function and try to derive r2 from it). */ 66 return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other); 67 } 68 #else 69 70 static func_desc_t func_desc(unsigned long addr) 71 { 72 return *(struct func_desc *)addr; 73 } 74 static unsigned int local_entry_offset(const Elf64_Sym *sym) 75 { 76 return 0; 77 } 78 79 void *dereference_module_function_descriptor(struct module *mod, void *ptr) 80 { 81 if (ptr < (void *)mod->arch.start_opd || 82 ptr >= (void *)mod->arch.end_opd) 83 return ptr; 84 85 return dereference_function_descriptor(ptr); 86 } 87 #endif 88 89 static unsigned long func_addr(unsigned long addr) 90 { 91 return func_desc(addr).addr; 92 } 93 94 static unsigned long stub_func_addr(func_desc_t func) 95 { 96 return func.addr; 97 } 98 99 #define STUB_MAGIC 0x73747562 /* stub */ 100 101 /* Like PPC32, we need little trampolines to do > 24-bit jumps (into 102 the kernel itself). But on PPC64, these need to be used for every 103 jump, actually, to reset r2 (TOC+0x8000). */ 104 struct ppc64_stub_entry { 105 /* 106 * 28 byte jump instruction sequence (7 instructions) that can 107 * hold ppc64_stub_insns or stub_insns. Must be 8-byte aligned 108 * with PCREL kernels that use prefix instructions in the stub. 109 */ 110 u32 jump[7]; 111 /* Used by ftrace to identify stubs */ 112 u32 magic; 113 /* Data for the above code */ 114 func_desc_t funcdata; 115 } __aligned(8); 116 117 struct ppc64_got_entry { 118 u64 addr; 119 }; 120 121 /* 122 * PPC64 uses 24 bit jumps, but we need to jump into other modules or 123 * the kernel which may be further. So we jump to a stub. 124 * 125 * Target address and TOC are loaded from function descriptor in the 126 * ppc64_stub_entry. 127 * 128 * r12 is used to generate the target address, which is required for the 129 * ELFv2 global entry point calling convention. 130 * 131 * TOC handling: 132 * - PCREL does not have a TOC. 133 * - ELFv2 non-PCREL just has to save r2, the callee is responsible for 134 * setting its own TOC pointer at the global entry address. 135 * - ELFv1 must load the new TOC pointer from the function descriptor. 136 */ 137 static u32 ppc64_stub_insns[] = { 138 #ifdef CONFIG_PPC_KERNEL_PCREL 139 /* pld r12,addr */ 140 PPC_PREFIX_8LS | __PPC_PRFX_R(1), 141 PPC_INST_PLD | ___PPC_RT(_R12), 142 #else 143 PPC_RAW_ADDIS(_R11, _R2, 0), 144 PPC_RAW_ADDI(_R11, _R11, 0), 145 /* Save current r2 value in magic place on the stack. */ 146 PPC_RAW_STD(_R2, _R1, R2_STACK_OFFSET), 147 PPC_RAW_LD(_R12, _R11, 32), 148 #ifdef CONFIG_PPC64_ELF_ABI_V1 149 /* Set up new r2 from function descriptor */ 150 PPC_RAW_LD(_R2, _R11, 40), 151 #endif 152 #endif 153 PPC_RAW_MTCTR(_R12), 154 PPC_RAW_BCTR(), 155 }; 156 157 /* 158 * Count how many different r_type relocations (different symbol, 159 * different addend). 160 */ 161 static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num, 162 unsigned long r_type) 163 { 164 unsigned int i, r_info, r_addend, _count_relocs; 165 166 /* FIXME: Only count external ones --RR */ 167 _count_relocs = 0; 168 r_info = 0; 169 r_addend = 0; 170 for (i = 0; i < num; i++) 171 /* Only count r_type relocs, others don't need stubs */ 172 if (ELF64_R_TYPE(rela[i].r_info) == r_type && 173 (r_info != ELF64_R_SYM(rela[i].r_info) || 174 r_addend != rela[i].r_addend)) { 175 _count_relocs++; 176 r_info = ELF64_R_SYM(rela[i].r_info); 177 r_addend = rela[i].r_addend; 178 } 179 180 return _count_relocs; 181 } 182 183 static int relacmp(const void *_x, const void *_y) 184 { 185 const Elf64_Rela *x, *y; 186 187 y = (Elf64_Rela *)_x; 188 x = (Elf64_Rela *)_y; 189 190 /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to 191 * make the comparison cheaper/faster. It won't affect the sorting or 192 * the counting algorithms' performance 193 */ 194 if (x->r_info < y->r_info) 195 return -1; 196 else if (x->r_info > y->r_info) 197 return 1; 198 else if (x->r_addend < y->r_addend) 199 return -1; 200 else if (x->r_addend > y->r_addend) 201 return 1; 202 else 203 return 0; 204 } 205 206 /* Get size of potential trampolines required. */ 207 static unsigned long get_stubs_size(const Elf64_Ehdr *hdr, 208 const Elf64_Shdr *sechdrs, 209 char *secstrings, 210 struct module *me) 211 { 212 /* One extra reloc so it's always 0-addr terminated */ 213 unsigned long relocs = 1; 214 unsigned i; 215 216 /* Every relocated section... */ 217 for (i = 1; i < hdr->e_shnum; i++) { 218 if (sechdrs[i].sh_type == SHT_RELA) { 219 pr_debug("Found relocations in section %u\n", i); 220 pr_debug("Ptr: %p. Number: %Lu\n", 221 (void *)sechdrs[i].sh_addr, 222 sechdrs[i].sh_size / sizeof(Elf64_Rela)); 223 224 /* Sort the relocation information based on a symbol and 225 * addend key. This is a stable O(n*log n) complexity 226 * algorithm but it will reduce the complexity of 227 * count_relocs() to linear complexity O(n) 228 */ 229 sort((void *)sechdrs[i].sh_addr, 230 sechdrs[i].sh_size / sizeof(Elf64_Rela), 231 sizeof(Elf64_Rela), relacmp, NULL); 232 233 relocs += count_relocs((void *)sechdrs[i].sh_addr, 234 sechdrs[i].sh_size 235 / sizeof(Elf64_Rela), 236 R_PPC_REL24); 237 #ifdef CONFIG_PPC_KERNEL_PCREL 238 relocs += count_relocs((void *)sechdrs[i].sh_addr, 239 sechdrs[i].sh_size 240 / sizeof(Elf64_Rela), 241 R_PPC64_REL24_NOTOC); 242 #endif 243 } 244 } 245 246 /* stubs for ftrace_caller and ftrace_regs_caller */ 247 relocs += IS_ENABLED(CONFIG_DYNAMIC_FTRACE) + IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS); 248 249 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 250 /* stubs for the function tracer */ 251 for (i = 1; i < hdr->e_shnum; i++) { 252 if (!strcmp(secstrings + sechdrs[i].sh_name, "__patchable_function_entries")) { 253 me->arch.ool_stub_count = sechdrs[i].sh_size / sizeof(unsigned long); 254 me->arch.ool_stub_index = 0; 255 relocs += roundup(me->arch.ool_stub_count * sizeof(struct ftrace_ool_stub), 256 sizeof(struct ppc64_stub_entry)) / 257 sizeof(struct ppc64_stub_entry); 258 break; 259 } 260 } 261 if (i == hdr->e_shnum) { 262 pr_err("%s: doesn't contain __patchable_function_entries.\n", me->name); 263 return -ENOEXEC; 264 } 265 #endif 266 267 pr_debug("Looks like a total of %lu stubs, max\n", relocs); 268 return relocs * sizeof(struct ppc64_stub_entry); 269 } 270 271 #ifdef CONFIG_PPC_KERNEL_PCREL 272 static int count_pcpu_relocs(const Elf64_Shdr *sechdrs, 273 const Elf64_Rela *rela, unsigned int num, 274 unsigned int symindex, unsigned int pcpu) 275 { 276 unsigned int i, r_info, r_addend, _count_relocs; 277 278 _count_relocs = 0; 279 r_info = 0; 280 r_addend = 0; 281 282 for (i = 0; i < num; i++) { 283 Elf64_Sym *sym; 284 285 /* This is the symbol it is referring to */ 286 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 287 + ELF64_R_SYM(rela[i].r_info); 288 289 if (sym->st_shndx == pcpu && 290 (r_info != ELF64_R_SYM(rela[i].r_info) || 291 r_addend != rela[i].r_addend)) { 292 _count_relocs++; 293 r_info = ELF64_R_SYM(rela[i].r_info); 294 r_addend = rela[i].r_addend; 295 } 296 } 297 298 return _count_relocs; 299 } 300 301 /* Get size of potential GOT required. */ 302 static unsigned long get_got_size(const Elf64_Ehdr *hdr, 303 const Elf64_Shdr *sechdrs, 304 struct module *me) 305 { 306 /* One extra reloc so it's always 0-addr terminated */ 307 unsigned long relocs = 1; 308 unsigned int i, symindex = 0; 309 310 for (i = 1; i < hdr->e_shnum; i++) { 311 if (sechdrs[i].sh_type == SHT_SYMTAB) { 312 symindex = i; 313 break; 314 } 315 } 316 WARN_ON_ONCE(!symindex); 317 318 /* Every relocated section... */ 319 for (i = 1; i < hdr->e_shnum; i++) { 320 if (sechdrs[i].sh_type == SHT_RELA) { 321 pr_debug("Found relocations in section %u\n", i); 322 pr_debug("Ptr: %p. Number: %llu\n", (void *)sechdrs[i].sh_addr, 323 sechdrs[i].sh_size / sizeof(Elf64_Rela)); 324 325 /* 326 * Sort the relocation information based on a symbol and 327 * addend key. This is a stable O(n*log n) complexity 328 * algorithm but it will reduce the complexity of 329 * count_relocs() to linear complexity O(n) 330 */ 331 sort((void *)sechdrs[i].sh_addr, 332 sechdrs[i].sh_size / sizeof(Elf64_Rela), 333 sizeof(Elf64_Rela), relacmp, NULL); 334 335 relocs += count_relocs((void *)sechdrs[i].sh_addr, 336 sechdrs[i].sh_size 337 / sizeof(Elf64_Rela), 338 R_PPC64_GOT_PCREL34); 339 340 /* 341 * Percpu data access typically gets linked with 342 * REL34 relocations, but the percpu section gets 343 * moved at load time and requires that to be 344 * converted to GOT linkage. 345 */ 346 if (IS_ENABLED(CONFIG_SMP) && symindex) 347 relocs += count_pcpu_relocs(sechdrs, 348 (void *)sechdrs[i].sh_addr, 349 sechdrs[i].sh_size 350 / sizeof(Elf64_Rela), 351 symindex, me->arch.pcpu_section); 352 } 353 } 354 355 pr_debug("Looks like a total of %lu GOT entries, max\n", relocs); 356 return relocs * sizeof(struct ppc64_got_entry); 357 } 358 #else /* CONFIG_PPC_KERNEL_PCREL */ 359 360 /* Still needed for ELFv2, for .TOC. */ 361 static void dedotify_versions(struct modversion_info *vers, 362 unsigned long size) 363 { 364 struct modversion_info *end; 365 366 for (end = (void *)vers + size; vers < end; vers++) 367 if (vers->name[0] == '.') { 368 memmove(vers->name, vers->name+1, strlen(vers->name)); 369 } 370 } 371 372 /* Same as normal versions, remove a leading dot if present. */ 373 static void dedotify_ext_version_names(char *str_seq, unsigned long size) 374 { 375 unsigned long out = 0; 376 unsigned long in; 377 char last = '\0'; 378 379 for (in = 0; in < size; in++) { 380 /* Skip one leading dot */ 381 if (last == '\0' && str_seq[in] == '.') 382 in++; 383 last = str_seq[in]; 384 str_seq[out++] = last; 385 } 386 /* Zero the trailing portion of the names table for robustness */ 387 memset(&str_seq[out], 0, size - out); 388 } 389 390 /* 391 * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC. 392 * seem to be defined (value set later). 393 */ 394 static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab) 395 { 396 unsigned int i; 397 398 for (i = 1; i < numsyms; i++) { 399 if (syms[i].st_shndx == SHN_UNDEF) { 400 char *name = strtab + syms[i].st_name; 401 if (name[0] == '.') { 402 if (strcmp(name+1, "TOC.") == 0) 403 syms[i].st_shndx = SHN_ABS; 404 syms[i].st_name++; 405 } 406 } 407 } 408 } 409 410 static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs, 411 const char *strtab, 412 unsigned int symindex) 413 { 414 unsigned int i, numsyms; 415 Elf64_Sym *syms; 416 417 syms = (Elf64_Sym *)sechdrs[symindex].sh_addr; 418 numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym); 419 420 for (i = 1; i < numsyms; i++) { 421 if (syms[i].st_shndx == SHN_ABS 422 && strcmp(strtab + syms[i].st_name, "TOC.") == 0) 423 return &syms[i]; 424 } 425 return NULL; 426 } 427 #endif /* CONFIG_PPC_KERNEL_PCREL */ 428 429 bool module_init_section(const char *name) 430 { 431 /* We don't handle .init for the moment: always return false. */ 432 return false; 433 } 434 435 int module_frob_arch_sections(Elf64_Ehdr *hdr, 436 Elf64_Shdr *sechdrs, 437 char *secstrings, 438 struct module *me) 439 { 440 unsigned int i; 441 442 /* Find .toc and .stubs sections, symtab and strtab */ 443 for (i = 1; i < hdr->e_shnum; i++) { 444 if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0) 445 me->arch.stubs_section = i; 446 #ifdef CONFIG_PPC_KERNEL_PCREL 447 else if (strcmp(secstrings + sechdrs[i].sh_name, ".data..percpu") == 0) 448 me->arch.pcpu_section = i; 449 else if (strcmp(secstrings + sechdrs[i].sh_name, ".mygot") == 0) { 450 me->arch.got_section = i; 451 if (sechdrs[i].sh_addralign < 8) 452 sechdrs[i].sh_addralign = 8; 453 } 454 #else 455 else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) { 456 me->arch.toc_section = i; 457 if (sechdrs[i].sh_addralign < 8) 458 sechdrs[i].sh_addralign = 8; 459 } else if (strcmp(secstrings + sechdrs[i].sh_name, "__versions") == 0) 460 dedotify_versions((void *)hdr + sechdrs[i].sh_offset, 461 sechdrs[i].sh_size); 462 else if (strcmp(secstrings + sechdrs[i].sh_name, "__version_ext_names") == 0) 463 dedotify_ext_version_names((void *)hdr + sechdrs[i].sh_offset, 464 sechdrs[i].sh_size); 465 466 if (sechdrs[i].sh_type == SHT_SYMTAB) 467 dedotify((void *)hdr + sechdrs[i].sh_offset, 468 sechdrs[i].sh_size / sizeof(Elf64_Sym), 469 (void *)hdr 470 + sechdrs[sechdrs[i].sh_link].sh_offset); 471 #endif 472 } 473 474 if (!me->arch.stubs_section) { 475 pr_err("%s: doesn't contain .stubs.\n", me->name); 476 return -ENOEXEC; 477 } 478 479 #ifdef CONFIG_PPC_KERNEL_PCREL 480 if (!me->arch.got_section) { 481 pr_err("%s: doesn't contain .mygot.\n", me->name); 482 return -ENOEXEC; 483 } 484 485 /* Override the got size */ 486 sechdrs[me->arch.got_section].sh_size = get_got_size(hdr, sechdrs, me); 487 #else 488 /* If we don't have a .toc, just use .stubs. We need to set r2 489 to some reasonable value in case the module calls out to 490 other functions via a stub, or if a function pointer escapes 491 the module by some means. */ 492 if (!me->arch.toc_section) 493 me->arch.toc_section = me->arch.stubs_section; 494 #endif 495 496 /* Override the stubs size */ 497 sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs, secstrings, me); 498 499 return 0; 500 } 501 502 #if defined(CONFIG_MPROFILE_KERNEL) || defined(CONFIG_ARCH_USING_PATCHABLE_FUNCTION_ENTRY) 503 504 static u32 stub_insns[] = { 505 #ifdef CONFIG_PPC_KERNEL_PCREL 506 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)), 507 PPC_RAW_NOP(), /* align the prefix insn */ 508 /* paddi r12,r12,addr */ 509 PPC_PREFIX_MLS | __PPC_PRFX_R(0), 510 PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12), 511 PPC_RAW_MTCTR(_R12), 512 PPC_RAW_BCTR(), 513 #else 514 PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)), 515 PPC_RAW_ADDIS(_R12, _R12, 0), 516 PPC_RAW_ADDI(_R12, _R12, 0), 517 PPC_RAW_MTCTR(_R12), 518 PPC_RAW_BCTR(), 519 #endif 520 }; 521 522 /* 523 * For mprofile-kernel we use a special stub for ftrace_caller() because we 524 * can't rely on r2 containing this module's TOC when we enter the stub. 525 * 526 * That can happen if the function calling us didn't need to use the toc. In 527 * that case it won't have setup r2, and the r2 value will be either the 528 * kernel's toc, or possibly another modules toc. 529 * 530 * To deal with that this stub uses the kernel toc, which is always accessible 531 * via the paca (in r13). The target (ftrace_caller()) is responsible for 532 * saving and restoring the toc before returning. 533 */ 534 static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, 535 unsigned long addr, 536 struct module *me) 537 { 538 long reladdr; 539 540 if ((unsigned long)entry->jump % 8 != 0) { 541 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name); 542 return 0; 543 } 544 545 BUILD_BUG_ON(sizeof(stub_insns) > sizeof(entry->jump)); 546 memcpy(entry->jump, stub_insns, sizeof(stub_insns)); 547 548 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 549 /* Stub uses address relative to kernel base (from the paca) */ 550 reladdr = addr - local_paca->kernelbase; 551 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) { 552 pr_err("%s: Address of %ps out of range of 34-bit relative address.\n", 553 me->name, (void *)addr); 554 return 0; 555 } 556 557 entry->jump[2] |= IMM_H18(reladdr); 558 entry->jump[3] |= IMM_L(reladdr); 559 } else { 560 /* Stub uses address relative to kernel toc (from the paca) */ 561 reladdr = addr - kernel_toc_addr(); 562 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 563 pr_err("%s: Address of %ps out of range of kernel_toc.\n", 564 me->name, (void *)addr); 565 return 0; 566 } 567 568 entry->jump[1] |= PPC_HA(reladdr); 569 entry->jump[2] |= PPC_LO(reladdr); 570 } 571 572 /* Even though we don't use funcdata in the stub, it's needed elsewhere. */ 573 entry->funcdata = func_desc(addr); 574 entry->magic = STUB_MAGIC; 575 576 return 1; 577 } 578 579 static bool is_mprofile_ftrace_call(const char *name) 580 { 581 if (!strcmp("_mcount", name)) 582 return true; 583 #ifdef CONFIG_DYNAMIC_FTRACE 584 if (!strcmp("ftrace_caller", name)) 585 return true; 586 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 587 if (!strcmp("ftrace_regs_caller", name)) 588 return true; 589 #endif 590 #endif 591 592 return false; 593 } 594 #else 595 static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, 596 unsigned long addr, 597 struct module *me) 598 { 599 return 0; 600 } 601 602 static bool is_mprofile_ftrace_call(const char *name) 603 { 604 return false; 605 } 606 #endif 607 608 /* 609 * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the 610 * value maximum span in an instruction which uses a signed offset). Round down 611 * to a 256 byte boundary for the odd case where we are setting up r2 without a 612 * .toc section. 613 */ 614 static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me) 615 { 616 #ifndef CONFIG_PPC_KERNEL_PCREL 617 return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000; 618 #else 619 return -1; 620 #endif 621 } 622 623 /* Patch stub to reference function and correct r2 value. */ 624 static inline int create_stub(const Elf64_Shdr *sechdrs, 625 struct ppc64_stub_entry *entry, 626 unsigned long addr, 627 struct module *me, 628 const char *name) 629 { 630 long reladdr; 631 func_desc_t desc; 632 int i; 633 634 if (is_mprofile_ftrace_call(name)) 635 return create_ftrace_stub(entry, addr, me); 636 637 if ((unsigned long)entry->jump % 8 != 0) { 638 pr_err("%s: Address of stub entry is not 8-byte aligned\n", me->name); 639 return 0; 640 } 641 642 BUILD_BUG_ON(sizeof(ppc64_stub_insns) > sizeof(entry->jump)); 643 for (i = 0; i < ARRAY_SIZE(ppc64_stub_insns); i++) { 644 if (patch_instruction(&entry->jump[i], 645 ppc_inst(ppc64_stub_insns[i]))) 646 return 0; 647 } 648 649 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) { 650 /* Stub uses address relative to itself! */ 651 reladdr = 0 + offsetof(struct ppc64_stub_entry, funcdata); 652 BUILD_BUG_ON(reladdr != 32); 653 if (reladdr > 0x1FFFFFFFFL || reladdr < -0x200000000L) { 654 pr_err("%s: Address of %p out of range of 34-bit relative address.\n", 655 me->name, (void *)reladdr); 656 return 0; 657 } 658 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr); 659 660 /* May not even need this if we're relative to 0 */ 661 if (patch_instruction(&entry->jump[0], 662 ppc_inst_prefix(entry->jump[0] | IMM_H18(reladdr), 663 entry->jump[1] | IMM_L(reladdr)))) 664 return 0; 665 666 } else { 667 /* Stub uses address relative to r2. */ 668 reladdr = (unsigned long)entry - my_r2(sechdrs, me); 669 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 670 pr_err("%s: Address %p of stub out of range of %p.\n", 671 me->name, (void *)reladdr, (void *)my_r2); 672 return 0; 673 } 674 pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr); 675 676 if (patch_instruction(&entry->jump[0], 677 ppc_inst(entry->jump[0] | PPC_HA(reladdr)))) 678 return 0; 679 680 if (patch_instruction(&entry->jump[1], 681 ppc_inst(entry->jump[1] | PPC_LO(reladdr)))) 682 return 0; 683 } 684 685 // func_desc_t is 8 bytes if ABIv2, else 16 bytes 686 desc = func_desc(addr); 687 for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) { 688 if (patch_u32(((u32 *)&entry->funcdata) + i, ((u32 *)&desc)[i])) 689 return 0; 690 } 691 692 if (patch_u32(&entry->magic, STUB_MAGIC)) 693 return 0; 694 695 return 1; 696 } 697 698 /* Create stub to jump to function described in this OPD/ptr: we need the 699 stub to set up the TOC ptr (r2) for the function. */ 700 static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs, 701 unsigned long addr, 702 struct module *me, 703 const char *name) 704 { 705 struct ppc64_stub_entry *stubs; 706 unsigned int i, num_stubs; 707 708 num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs); 709 710 /* Find this stub, or if that fails, the next avail. entry */ 711 stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr; 712 for (i = 0; stub_func_addr(stubs[i].funcdata); i++) { 713 if (WARN_ON(i >= num_stubs)) 714 return 0; 715 716 if (stub_func_addr(stubs[i].funcdata) == func_addr(addr)) 717 return (unsigned long)&stubs[i]; 718 } 719 720 if (!create_stub(sechdrs, &stubs[i], addr, me, name)) 721 return 0; 722 723 return (unsigned long)&stubs[i]; 724 } 725 726 #ifdef CONFIG_PPC_KERNEL_PCREL 727 /* Create GOT to load the location described in this ptr */ 728 static unsigned long got_for_addr(const Elf64_Shdr *sechdrs, 729 unsigned long addr, 730 struct module *me, 731 const char *name) 732 { 733 struct ppc64_got_entry *got; 734 unsigned int i, num_got; 735 736 if (!IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 737 return addr; 738 739 num_got = sechdrs[me->arch.got_section].sh_size / sizeof(*got); 740 741 /* Find this stub, or if that fails, the next avail. entry */ 742 got = (void *)sechdrs[me->arch.got_section].sh_addr; 743 for (i = 0; got[i].addr; i++) { 744 if (WARN_ON(i >= num_got)) 745 return 0; 746 747 if (got[i].addr == addr) 748 return (unsigned long)&got[i]; 749 } 750 751 got[i].addr = addr; 752 753 return (unsigned long)&got[i]; 754 } 755 #endif 756 757 /* We expect a noop next: if it is, replace it with instruction to 758 restore r2. */ 759 static int restore_r2(const char *name, u32 *instruction, struct module *me) 760 { 761 u32 *prev_insn = instruction - 1; 762 u32 insn_val = *instruction; 763 764 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 765 return 0; 766 767 if (is_mprofile_ftrace_call(name)) 768 return 0; 769 770 /* 771 * Make sure the branch isn't a sibling call. Sibling calls aren't 772 * "link" branches and they don't return, so they don't need the r2 773 * restore afterwards. 774 */ 775 if (!instr_is_relative_link_branch(ppc_inst(*prev_insn))) 776 return 0; 777 778 /* 779 * For livepatch, the restore r2 instruction might have already been 780 * written previously, if the referenced symbol is in a previously 781 * unloaded module which is now being loaded again. In that case, skip 782 * the warning and the instruction write. 783 */ 784 if (insn_val == PPC_INST_LD_TOC) 785 return 0; 786 787 if (insn_val != PPC_RAW_NOP()) { 788 pr_err("%s: Expected nop after call, got %08x at %pS\n", 789 me->name, insn_val, instruction); 790 return -ENOEXEC; 791 } 792 793 /* ld r2,R2_STACK_OFFSET(r1) */ 794 return patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC)); 795 } 796 797 int apply_relocate_add(Elf64_Shdr *sechdrs, 798 const char *strtab, 799 unsigned int symindex, 800 unsigned int relsec, 801 struct module *me) 802 { 803 unsigned int i; 804 Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr; 805 Elf64_Sym *sym; 806 unsigned long *location; 807 unsigned long value; 808 809 pr_debug("Applying ADD relocate section %u to %u\n", relsec, 810 sechdrs[relsec].sh_info); 811 812 #ifndef CONFIG_PPC_KERNEL_PCREL 813 /* First time we're called, we can fix up .TOC. */ 814 if (!me->arch.toc_fixed) { 815 sym = find_dot_toc(sechdrs, strtab, symindex); 816 /* It's theoretically possible that a module doesn't want a 817 * .TOC. so don't fail it just for that. */ 818 if (sym) 819 sym->st_value = my_r2(sechdrs, me); 820 me->arch.toc_fixed = true; 821 } 822 #endif 823 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) { 824 /* This is where to make the change */ 825 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 826 + rela[i].r_offset; 827 /* This is the symbol it is referring to */ 828 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 829 + ELF64_R_SYM(rela[i].r_info); 830 831 pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n", 832 location, (long)ELF64_R_TYPE(rela[i].r_info), 833 strtab + sym->st_name, (unsigned long)sym->st_value, 834 (long)rela[i].r_addend); 835 836 /* `Everything is relative'. */ 837 value = sym->st_value + rela[i].r_addend; 838 839 switch (ELF64_R_TYPE(rela[i].r_info)) { 840 case R_PPC64_ADDR32: 841 /* Simply set it */ 842 *(u32 *)location = value; 843 break; 844 845 case R_PPC64_ADDR64: 846 /* Simply set it */ 847 *(unsigned long *)location = value; 848 break; 849 850 #ifndef CONFIG_PPC_KERNEL_PCREL 851 case R_PPC64_TOC: 852 *(unsigned long *)location = my_r2(sechdrs, me); 853 break; 854 855 case R_PPC64_TOC16: 856 /* Subtract TOC pointer */ 857 value -= my_r2(sechdrs, me); 858 if (value + 0x8000 > 0xffff) { 859 pr_err("%s: bad TOC16 relocation (0x%lx)\n", 860 me->name, value); 861 return -ENOEXEC; 862 } 863 *((uint16_t *) location) 864 = (*((uint16_t *) location) & ~0xffff) 865 | (value & 0xffff); 866 break; 867 868 case R_PPC64_TOC16_LO: 869 /* Subtract TOC pointer */ 870 value -= my_r2(sechdrs, me); 871 *((uint16_t *) location) 872 = (*((uint16_t *) location) & ~0xffff) 873 | (value & 0xffff); 874 break; 875 876 case R_PPC64_TOC16_DS: 877 /* Subtract TOC pointer */ 878 value -= my_r2(sechdrs, me); 879 if ((value & 3) != 0 || value + 0x8000 > 0xffff) { 880 pr_err("%s: bad TOC16_DS relocation (0x%lx)\n", 881 me->name, value); 882 return -ENOEXEC; 883 } 884 *((uint16_t *) location) 885 = (*((uint16_t *) location) & ~0xfffc) 886 | (value & 0xfffc); 887 break; 888 889 case R_PPC64_TOC16_LO_DS: 890 /* Subtract TOC pointer */ 891 value -= my_r2(sechdrs, me); 892 if ((value & 3) != 0) { 893 pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n", 894 me->name, value); 895 return -ENOEXEC; 896 } 897 *((uint16_t *) location) 898 = (*((uint16_t *) location) & ~0xfffc) 899 | (value & 0xfffc); 900 break; 901 902 case R_PPC64_TOC16_HA: 903 /* Subtract TOC pointer */ 904 value -= my_r2(sechdrs, me); 905 value = ((value + 0x8000) >> 16); 906 *((uint16_t *) location) 907 = (*((uint16_t *) location) & ~0xffff) 908 | (value & 0xffff); 909 break; 910 #endif 911 912 case R_PPC_REL24: 913 #ifdef CONFIG_PPC_KERNEL_PCREL 914 /* PCREL still generates REL24 for mcount */ 915 case R_PPC64_REL24_NOTOC: 916 #endif 917 /* FIXME: Handle weak symbols here --RR */ 918 if (sym->st_shndx == SHN_UNDEF || 919 sym->st_shndx == SHN_LIVEPATCH) { 920 /* External: go via stub */ 921 value = stub_for_addr(sechdrs, value, me, 922 strtab + sym->st_name); 923 if (!value) 924 return -ENOENT; 925 if (restore_r2(strtab + sym->st_name, 926 (u32 *)location + 1, me)) 927 return -ENOEXEC; 928 } else 929 value += local_entry_offset(sym); 930 931 /* Convert value to relative */ 932 value -= (unsigned long)location; 933 if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){ 934 pr_err("%s: REL24 %li out of range!\n", 935 me->name, (long int)value); 936 return -ENOEXEC; 937 } 938 939 /* Only replace bits 2 through 26 */ 940 value = (*(uint32_t *)location & ~PPC_LI_MASK) | PPC_LI(value); 941 942 if (patch_instruction((u32 *)location, ppc_inst(value))) 943 return -EFAULT; 944 945 break; 946 947 case R_PPC64_REL64: 948 /* 64 bits relative (used by features fixups) */ 949 *location = value - (unsigned long)location; 950 break; 951 952 case R_PPC64_REL32: 953 /* 32 bits relative (used by relative exception tables) */ 954 /* Convert value to relative */ 955 value -= (unsigned long)location; 956 if (value + 0x80000000 > 0xffffffff) { 957 pr_err("%s: REL32 %li out of range!\n", 958 me->name, (long int)value); 959 return -ENOEXEC; 960 } 961 *(u32 *)location = value; 962 break; 963 964 #ifdef CONFIG_PPC_KERNEL_PCREL 965 case R_PPC64_PCREL34: { 966 unsigned long absvalue = value; 967 968 /* Convert value to relative */ 969 value -= (unsigned long)location; 970 971 if (value + 0x200000000 > 0x3ffffffff) { 972 if (sym->st_shndx != me->arch.pcpu_section) { 973 pr_err("%s: REL34 %li out of range!\n", 974 me->name, (long)value); 975 return -ENOEXEC; 976 } 977 978 /* 979 * per-cpu section is special cased because 980 * it is moved during loading, so has to be 981 * converted to use GOT. 982 */ 983 value = got_for_addr(sechdrs, absvalue, me, 984 strtab + sym->st_name); 985 if (!value) 986 return -ENOENT; 987 value -= (unsigned long)location; 988 989 /* Turn pla into pld */ 990 if (patch_instruction((u32 *)location, 991 ppc_inst_prefix((*(u32 *)location & ~0x02000000), 992 (*((u32 *)location + 1) & ~0xf8000000) | 0xe4000000))) 993 return -EFAULT; 994 } 995 996 if (patch_instruction((u32 *)location, 997 ppc_inst_prefix((*(u32 *)location & ~0x3ffff) | IMM_H18(value), 998 (*((u32 *)location + 1) & ~0xffff) | IMM_L(value)))) 999 return -EFAULT; 1000 1001 break; 1002 } 1003 1004 #else 1005 case R_PPC64_TOCSAVE: 1006 /* 1007 * Marker reloc indicates we don't have to save r2. 1008 * That would only save us one instruction, so ignore 1009 * it. 1010 */ 1011 break; 1012 #endif 1013 1014 case R_PPC64_ENTRY: 1015 if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) 1016 break; 1017 1018 /* 1019 * Optimize ELFv2 large code model entry point if 1020 * the TOC is within 2GB range of current location. 1021 */ 1022 value = my_r2(sechdrs, me) - (unsigned long)location; 1023 if (value + 0x80008000 > 0xffffffff) 1024 break; 1025 /* 1026 * Check for the large code model prolog sequence: 1027 * ld r2, ...(r12) 1028 * add r2, r2, r12 1029 */ 1030 if ((((uint32_t *)location)[0] & ~0xfffc) != PPC_RAW_LD(_R2, _R12, 0)) 1031 break; 1032 if (((uint32_t *)location)[1] != PPC_RAW_ADD(_R2, _R2, _R12)) 1033 break; 1034 /* 1035 * If found, replace it with: 1036 * addis r2, r12, (.TOC.-func)@ha 1037 * addi r2, r2, (.TOC.-func)@l 1038 */ 1039 ((uint32_t *)location)[0] = PPC_RAW_ADDIS(_R2, _R12, PPC_HA(value)); 1040 ((uint32_t *)location)[1] = PPC_RAW_ADDI(_R2, _R2, PPC_LO(value)); 1041 break; 1042 1043 case R_PPC64_REL16_HA: 1044 /* Subtract location pointer */ 1045 value -= (unsigned long)location; 1046 value = ((value + 0x8000) >> 16); 1047 *((uint16_t *) location) 1048 = (*((uint16_t *) location) & ~0xffff) 1049 | (value & 0xffff); 1050 break; 1051 1052 case R_PPC64_REL16_LO: 1053 /* Subtract location pointer */ 1054 value -= (unsigned long)location; 1055 *((uint16_t *) location) 1056 = (*((uint16_t *) location) & ~0xffff) 1057 | (value & 0xffff); 1058 break; 1059 1060 #ifdef CONFIG_PPC_KERNEL_PCREL 1061 case R_PPC64_GOT_PCREL34: 1062 value = got_for_addr(sechdrs, value, me, 1063 strtab + sym->st_name); 1064 if (!value) 1065 return -ENOENT; 1066 value -= (unsigned long)location; 1067 ((uint32_t *)location)[0] = (((uint32_t *)location)[0] & ~0x3ffff) | 1068 ((value >> 16) & 0x3ffff); 1069 ((uint32_t *)location)[1] = (((uint32_t *)location)[1] & ~0xffff) | 1070 (value & 0xffff); 1071 break; 1072 #endif 1073 1074 default: 1075 pr_err("%s: Unknown ADD relocation: %lu\n", 1076 me->name, 1077 (unsigned long)ELF64_R_TYPE(rela[i].r_info)); 1078 return -ENOEXEC; 1079 } 1080 } 1081 1082 return 0; 1083 } 1084 1085 #ifdef CONFIG_DYNAMIC_FTRACE 1086 int module_trampoline_target(struct module *mod, unsigned long addr, 1087 unsigned long *target) 1088 { 1089 struct ppc64_stub_entry *stub; 1090 func_desc_t funcdata; 1091 u32 magic; 1092 1093 if (!within_module_core(addr, mod)) { 1094 pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name); 1095 return -EFAULT; 1096 } 1097 1098 stub = (struct ppc64_stub_entry *)addr; 1099 1100 if (copy_from_kernel_nofault(&magic, &stub->magic, 1101 sizeof(magic))) { 1102 pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name); 1103 return -EFAULT; 1104 } 1105 1106 if (magic != STUB_MAGIC) { 1107 pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name); 1108 return -EFAULT; 1109 } 1110 1111 if (copy_from_kernel_nofault(&funcdata, &stub->funcdata, 1112 sizeof(funcdata))) { 1113 pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name); 1114 return -EFAULT; 1115 } 1116 1117 *target = stub_func_addr(funcdata); 1118 1119 return 0; 1120 } 1121 1122 static int setup_ftrace_ool_stubs(const Elf64_Shdr *sechdrs, unsigned long addr, struct module *me) 1123 { 1124 #ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE 1125 unsigned int i, total_stubs, num_stubs; 1126 struct ppc64_stub_entry *stub; 1127 1128 total_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stub); 1129 num_stubs = roundup(me->arch.ool_stub_count * sizeof(struct ftrace_ool_stub), 1130 sizeof(struct ppc64_stub_entry)) / sizeof(struct ppc64_stub_entry); 1131 1132 /* Find the next available entry */ 1133 stub = (void *)sechdrs[me->arch.stubs_section].sh_addr; 1134 for (i = 0; stub_func_addr(stub[i].funcdata); i++) 1135 if (WARN_ON(i >= total_stubs)) 1136 return -1; 1137 1138 if (WARN_ON(i + num_stubs > total_stubs)) 1139 return -1; 1140 1141 stub += i; 1142 me->arch.ool_stubs = (struct ftrace_ool_stub *)stub; 1143 1144 /* reserve stubs */ 1145 for (i = 0; i < num_stubs; i++) 1146 if (patch_u32((void *)&stub->funcdata, PPC_RAW_NOP())) 1147 return -1; 1148 #endif 1149 1150 return 0; 1151 } 1152 1153 int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs) 1154 { 1155 mod->arch.tramp = stub_for_addr(sechdrs, 1156 (unsigned long)ftrace_caller, 1157 mod, 1158 "ftrace_caller"); 1159 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 1160 mod->arch.tramp_regs = stub_for_addr(sechdrs, 1161 (unsigned long)ftrace_regs_caller, 1162 mod, 1163 "ftrace_regs_caller"); 1164 if (!mod->arch.tramp_regs) 1165 return -ENOENT; 1166 #endif 1167 1168 if (!mod->arch.tramp) 1169 return -ENOENT; 1170 1171 if (setup_ftrace_ool_stubs(sechdrs, mod->arch.tramp, mod)) 1172 return -ENOENT; 1173 1174 return 0; 1175 } 1176 #endif 1177