1a17ae4c3SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0+
21da177e4SLinus Torvalds /*
3a53c8fabSHeiko Carstens * Kernel module help for s390.
41da177e4SLinus Torvalds *
51da177e4SLinus Torvalds * S390 version
6a53c8fabSHeiko Carstens * Copyright IBM Corp. 2002, 2003
71da177e4SLinus Torvalds * Author(s): Arnd Bergmann (arndb@de.ibm.com)
81da177e4SLinus Torvalds * Martin Schwidefsky (schwidefsky@de.ibm.com)
91da177e4SLinus Torvalds *
101da177e4SLinus Torvalds * based on i386 version
111da177e4SLinus Torvalds * Copyright (C) 2001 Rusty Russell.
121da177e4SLinus Torvalds */
131da177e4SLinus Torvalds #include <linux/module.h>
141da177e4SLinus Torvalds #include <linux/elf.h>
151da177e4SLinus Torvalds #include <linux/vmalloc.h>
161da177e4SLinus Torvalds #include <linux/fs.h>
17de5012b4SIlya Leoshkevich #include <linux/ftrace.h>
181da177e4SLinus Torvalds #include <linux/string.h>
191da177e4SLinus Torvalds #include <linux/kernel.h>
20793213a8SVasily Gorbik #include <linux/kasan.h>
212b67fc46SHeiko Carstens #include <linux/moduleloader.h>
22c0007f1aSHeiko Carstens #include <linux/bug.h>
23be242261SPeter Zijlstra #include <linux/memory.h>
24*12af2b83SMike Rapoport (IBM) #include <linux/execmem.h>
25686140a1SVasily Gorbik #include <asm/alternative.h>
26f19fbd5eSMartin Schwidefsky #include <asm/nospec-branch.h>
27f19fbd5eSMartin Schwidefsky #include <asm/facility.h>
28de5012b4SIlya Leoshkevich #include <asm/ftrace.lds.h>
29de5012b4SIlya Leoshkevich #include <asm/set_memory.h>
3034644cc2SHeiko Carstens #include <asm/setup.h>
311da177e4SLinus Torvalds
321da177e4SLinus Torvalds #if 0
331da177e4SLinus Torvalds #define DEBUGP printk
341da177e4SLinus Torvalds #else
351da177e4SLinus Torvalds #define DEBUGP(fmt , ...)
361da177e4SLinus Torvalds #endif
371da177e4SLinus Torvalds
38f3b7e73bSIlya Leoshkevich #define PLT_ENTRY_SIZE 22
391da177e4SLinus Torvalds
40de5012b4SIlya Leoshkevich #ifdef CONFIG_FUNCTION_TRACER
module_arch_cleanup(struct module * mod)41de5012b4SIlya Leoshkevich void module_arch_cleanup(struct module *mod)
42de5012b4SIlya Leoshkevich {
43*12af2b83SMike Rapoport (IBM) execmem_free(mod->arch.trampolines_start);
44de5012b4SIlya Leoshkevich }
45de5012b4SIlya Leoshkevich #endif
46de5012b4SIlya Leoshkevich
module_arch_freeing_init(struct module * mod)47d453cdedSRusty Russell void module_arch_freeing_init(struct module *mod)
481da177e4SLinus Torvalds {
49f31e0960SJessica Yu if (is_livepatch_module(mod) &&
50f31e0960SJessica Yu mod->state == MODULE_STATE_LIVE)
51f31e0960SJessica Yu return;
52f31e0960SJessica Yu
536a03f5f0SChristian Borntraeger vfree(mod->arch.syminfo);
546a03f5f0SChristian Borntraeger mod->arch.syminfo = NULL;
553164a3cbSHendrik Brueckner }
561da177e4SLinus Torvalds
check_rela(Elf_Rela * rela,struct module * me)57083e14c0SMartin Schwidefsky static void check_rela(Elf_Rela *rela, struct module *me)
581da177e4SLinus Torvalds {
591da177e4SLinus Torvalds struct mod_arch_syminfo *info;
601da177e4SLinus Torvalds
611da177e4SLinus Torvalds info = me->arch.syminfo + ELF_R_SYM (rela->r_info);
621da177e4SLinus Torvalds switch (ELF_R_TYPE (rela->r_info)) {
631da177e4SLinus Torvalds case R_390_GOT12: /* 12 bit GOT offset. */
641da177e4SLinus Torvalds case R_390_GOT16: /* 16 bit GOT offset. */
651da177e4SLinus Torvalds case R_390_GOT20: /* 20 bit GOT offset. */
661da177e4SLinus Torvalds case R_390_GOT32: /* 32 bit GOT offset. */
671da177e4SLinus Torvalds case R_390_GOT64: /* 64 bit GOT offset. */
681da177e4SLinus Torvalds case R_390_GOTENT: /* 32 bit PC rel. to GOT entry shifted by 1. */
691da177e4SLinus Torvalds case R_390_GOTPLT12: /* 12 bit offset to jump slot. */
701da177e4SLinus Torvalds case R_390_GOTPLT16: /* 16 bit offset to jump slot. */
711da177e4SLinus Torvalds case R_390_GOTPLT20: /* 20 bit offset to jump slot. */
721da177e4SLinus Torvalds case R_390_GOTPLT32: /* 32 bit offset to jump slot. */
731da177e4SLinus Torvalds case R_390_GOTPLT64: /* 64 bit offset to jump slot. */
741da177e4SLinus Torvalds case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */
751da177e4SLinus Torvalds if (info->got_offset == -1UL) {
761da177e4SLinus Torvalds info->got_offset = me->arch.got_size;
771da177e4SLinus Torvalds me->arch.got_size += sizeof(void*);
781da177e4SLinus Torvalds }
791da177e4SLinus Torvalds break;
801da177e4SLinus Torvalds case R_390_PLT16DBL: /* 16 bit PC rel. PLT shifted by 1. */
811da177e4SLinus Torvalds case R_390_PLT32DBL: /* 32 bit PC rel. PLT shifted by 1. */
821da177e4SLinus Torvalds case R_390_PLT32: /* 32 bit PC relative PLT address. */
831da177e4SLinus Torvalds case R_390_PLT64: /* 64 bit PC relative PLT address. */
841da177e4SLinus Torvalds case R_390_PLTOFF16: /* 16 bit offset from GOT to PLT. */
851da177e4SLinus Torvalds case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */
861da177e4SLinus Torvalds case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */
871da177e4SLinus Torvalds if (info->plt_offset == -1UL) {
881da177e4SLinus Torvalds info->plt_offset = me->arch.plt_size;
891da177e4SLinus Torvalds me->arch.plt_size += PLT_ENTRY_SIZE;
901da177e4SLinus Torvalds }
911da177e4SLinus Torvalds break;
921da177e4SLinus Torvalds case R_390_COPY:
931da177e4SLinus Torvalds case R_390_GLOB_DAT:
941da177e4SLinus Torvalds case R_390_JMP_SLOT:
951da177e4SLinus Torvalds case R_390_RELATIVE:
961da177e4SLinus Torvalds /* Only needed if we want to support loading of
971da177e4SLinus Torvalds modules linked with -shared. */
981da177e4SLinus Torvalds break;
991da177e4SLinus Torvalds }
1001da177e4SLinus Torvalds }
1011da177e4SLinus Torvalds
1021da177e4SLinus Torvalds /*
1031da177e4SLinus Torvalds * Account for GOT and PLT relocations. We can't add sections for
1041da177e4SLinus Torvalds * got and plt but we can increase the core module size.
1051da177e4SLinus Torvalds */
module_frob_arch_sections(Elf_Ehdr * hdr,Elf_Shdr * sechdrs,char * secstrings,struct module * me)106083e14c0SMartin Schwidefsky int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
1071da177e4SLinus Torvalds char *secstrings, struct module *me)
1081da177e4SLinus Torvalds {
1091da177e4SLinus Torvalds Elf_Shdr *symtab;
1101da177e4SLinus Torvalds Elf_Sym *symbols;
1111da177e4SLinus Torvalds Elf_Rela *rela;
1121da177e4SLinus Torvalds char *strings;
1131da177e4SLinus Torvalds int nrela, i, j;
114ac3b4328SSong Liu struct module_memory *mod_mem;
1151da177e4SLinus Torvalds
1161da177e4SLinus Torvalds /* Find symbol table and string table. */
117d2c993d8SHeiko Carstens symtab = NULL;
1181da177e4SLinus Torvalds for (i = 0; i < hdr->e_shnum; i++)
1191da177e4SLinus Torvalds switch (sechdrs[i].sh_type) {
1201da177e4SLinus Torvalds case SHT_SYMTAB:
1211da177e4SLinus Torvalds symtab = sechdrs + i;
1221da177e4SLinus Torvalds break;
1231da177e4SLinus Torvalds }
1241da177e4SLinus Torvalds if (!symtab) {
1251da177e4SLinus Torvalds printk(KERN_ERR "module %s: no symbol table\n", me->name);
1261da177e4SLinus Torvalds return -ENOEXEC;
1271da177e4SLinus Torvalds }
1281da177e4SLinus Torvalds
1291da177e4SLinus Torvalds /* Allocate one syminfo structure per symbol. */
1301da177e4SLinus Torvalds me->arch.nsyms = symtab->sh_size / sizeof(Elf_Sym);
13142bc47b3SKees Cook me->arch.syminfo = vmalloc(array_size(sizeof(struct mod_arch_syminfo),
13242bc47b3SKees Cook me->arch.nsyms));
1331da177e4SLinus Torvalds if (!me->arch.syminfo)
1341da177e4SLinus Torvalds return -ENOMEM;
1351da177e4SLinus Torvalds symbols = (void *) hdr + symtab->sh_offset;
1361da177e4SLinus Torvalds strings = (void *) hdr + sechdrs[symtab->sh_link].sh_offset;
1371da177e4SLinus Torvalds for (i = 0; i < me->arch.nsyms; i++) {
1381da177e4SLinus Torvalds if (symbols[i].st_shndx == SHN_UNDEF &&
1391da177e4SLinus Torvalds strcmp(strings + symbols[i].st_name,
1401da177e4SLinus Torvalds "_GLOBAL_OFFSET_TABLE_") == 0)
1411da177e4SLinus Torvalds /* "Define" it as absolute. */
1421da177e4SLinus Torvalds symbols[i].st_shndx = SHN_ABS;
1431da177e4SLinus Torvalds me->arch.syminfo[i].got_offset = -1UL;
1441da177e4SLinus Torvalds me->arch.syminfo[i].plt_offset = -1UL;
1451da177e4SLinus Torvalds me->arch.syminfo[i].got_initialized = 0;
1461da177e4SLinus Torvalds me->arch.syminfo[i].plt_initialized = 0;
1471da177e4SLinus Torvalds }
1481da177e4SLinus Torvalds
1491da177e4SLinus Torvalds /* Search for got/plt relocations. */
1501da177e4SLinus Torvalds me->arch.got_size = me->arch.plt_size = 0;
1511da177e4SLinus Torvalds for (i = 0; i < hdr->e_shnum; i++) {
1521da177e4SLinus Torvalds if (sechdrs[i].sh_type != SHT_RELA)
1531da177e4SLinus Torvalds continue;
1541da177e4SLinus Torvalds nrela = sechdrs[i].sh_size / sizeof(Elf_Rela);
1551da177e4SLinus Torvalds rela = (void *) hdr + sechdrs[i].sh_offset;
1561da177e4SLinus Torvalds for (j = 0; j < nrela; j++)
1571da177e4SLinus Torvalds check_rela(rela + j, me);
1581da177e4SLinus Torvalds }
1591da177e4SLinus Torvalds
1601da177e4SLinus Torvalds /* Increase core size by size of got & plt and set start
1611da177e4SLinus Torvalds offsets for got and plt. */
162ac3b4328SSong Liu mod_mem = &me->mem[MOD_TEXT];
163ac3b4328SSong Liu mod_mem->size = ALIGN(mod_mem->size, 4);
164ac3b4328SSong Liu me->arch.got_offset = mod_mem->size;
165ac3b4328SSong Liu mod_mem->size += me->arch.got_size;
166ac3b4328SSong Liu me->arch.plt_offset = mod_mem->size;
167f19fbd5eSMartin Schwidefsky if (me->arch.plt_size) {
1686e179d64SMartin Schwidefsky if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable)
169f19fbd5eSMartin Schwidefsky me->arch.plt_size += PLT_ENTRY_SIZE;
170ac3b4328SSong Liu mod_mem->size += me->arch.plt_size;
171f19fbd5eSMartin Schwidefsky }
1721da177e4SLinus Torvalds return 0;
1731da177e4SLinus Torvalds }
1741da177e4SLinus Torvalds
apply_rela_bits(Elf_Addr loc,Elf_Addr val,int sign,int bits,int shift,void * (* write)(void * dest,const void * src,size_t len))175083e14c0SMartin Schwidefsky static int apply_rela_bits(Elf_Addr loc, Elf_Addr val,
176be242261SPeter Zijlstra int sign, int bits, int shift,
177be242261SPeter Zijlstra void *(*write)(void *dest, const void *src, size_t len))
178083e14c0SMartin Schwidefsky {
179083e14c0SMartin Schwidefsky unsigned long umax;
180083e14c0SMartin Schwidefsky long min, max;
181be242261SPeter Zijlstra void *dest = (void *)loc;
182083e14c0SMartin Schwidefsky
183083e14c0SMartin Schwidefsky if (val & ((1UL << shift) - 1))
184083e14c0SMartin Schwidefsky return -ENOEXEC;
185083e14c0SMartin Schwidefsky if (sign) {
186083e14c0SMartin Schwidefsky val = (Elf_Addr)(((long) val) >> shift);
187083e14c0SMartin Schwidefsky min = -(1L << (bits - 1));
188083e14c0SMartin Schwidefsky max = (1L << (bits - 1)) - 1;
189083e14c0SMartin Schwidefsky if ((long) val < min || (long) val > max)
190083e14c0SMartin Schwidefsky return -ENOEXEC;
191083e14c0SMartin Schwidefsky } else {
192083e14c0SMartin Schwidefsky val >>= shift;
193083e14c0SMartin Schwidefsky umax = ((1UL << (bits - 1)) << 1) - 1;
194083e14c0SMartin Schwidefsky if ((unsigned long) val > umax)
195083e14c0SMartin Schwidefsky return -ENOEXEC;
196083e14c0SMartin Schwidefsky }
197083e14c0SMartin Schwidefsky
198be242261SPeter Zijlstra if (bits == 8) {
199be242261SPeter Zijlstra unsigned char tmp = val;
200be242261SPeter Zijlstra write(dest, &tmp, 1);
201be242261SPeter Zijlstra } else if (bits == 12) {
202be242261SPeter Zijlstra unsigned short tmp = (val & 0xfff) |
203083e14c0SMartin Schwidefsky (*(unsigned short *) loc & 0xf000);
204be242261SPeter Zijlstra write(dest, &tmp, 2);
205be242261SPeter Zijlstra } else if (bits == 16) {
206be242261SPeter Zijlstra unsigned short tmp = val;
207be242261SPeter Zijlstra write(dest, &tmp, 2);
208be242261SPeter Zijlstra } else if (bits == 20) {
209be242261SPeter Zijlstra unsigned int tmp = (val & 0xfff) << 16 |
210be242261SPeter Zijlstra (val & 0xff000) >> 4 | (*(unsigned int *) loc & 0xf00000ff);
211be242261SPeter Zijlstra write(dest, &tmp, 4);
212be242261SPeter Zijlstra } else if (bits == 32) {
213be242261SPeter Zijlstra unsigned int tmp = val;
214be242261SPeter Zijlstra write(dest, &tmp, 4);
215be242261SPeter Zijlstra } else if (bits == 64) {
216be242261SPeter Zijlstra unsigned long tmp = val;
217be242261SPeter Zijlstra write(dest, &tmp, 8);
218be242261SPeter Zijlstra }
219083e14c0SMartin Schwidefsky return 0;
220083e14c0SMartin Schwidefsky }
221083e14c0SMartin Schwidefsky
apply_rela(Elf_Rela * rela,Elf_Addr base,Elf_Sym * symtab,const char * strtab,struct module * me,void * (* write)(void * dest,const void * src,size_t len))222083e14c0SMartin Schwidefsky static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab,
223be242261SPeter Zijlstra const char *strtab, struct module *me,
224be242261SPeter Zijlstra void *(*write)(void *dest, const void *src, size_t len))
2251da177e4SLinus Torvalds {
2261da177e4SLinus Torvalds struct mod_arch_syminfo *info;
2271da177e4SLinus Torvalds Elf_Addr loc, val;
2281da177e4SLinus Torvalds int r_type, r_sym;
22972a6b43eSHeiko Carstens int rc = -ENOEXEC;
2301da177e4SLinus Torvalds
2311da177e4SLinus Torvalds /* This is where to make the change */
2321da177e4SLinus Torvalds loc = base + rela->r_offset;
2331da177e4SLinus Torvalds /* This is the symbol it is referring to. Note that all
2341da177e4SLinus Torvalds undefined symbols have been resolved. */
2351da177e4SLinus Torvalds r_sym = ELF_R_SYM(rela->r_info);
2361da177e4SLinus Torvalds r_type = ELF_R_TYPE(rela->r_info);
2371da177e4SLinus Torvalds info = me->arch.syminfo + r_sym;
2381da177e4SLinus Torvalds val = symtab[r_sym].st_value;
2391da177e4SLinus Torvalds
2401da177e4SLinus Torvalds switch (r_type) {
241e80cfc31SHendrik Brueckner case R_390_NONE: /* No relocation. */
242e80cfc31SHendrik Brueckner rc = 0;
243e80cfc31SHendrik Brueckner break;
2441da177e4SLinus Torvalds case R_390_8: /* Direct 8 bit. */
2451da177e4SLinus Torvalds case R_390_12: /* Direct 12 bit. */
2461da177e4SLinus Torvalds case R_390_16: /* Direct 16 bit. */
2471da177e4SLinus Torvalds case R_390_20: /* Direct 20 bit. */
2481da177e4SLinus Torvalds case R_390_32: /* Direct 32 bit. */
2491da177e4SLinus Torvalds case R_390_64: /* Direct 64 bit. */
2501da177e4SLinus Torvalds val += rela->r_addend;
2511da177e4SLinus Torvalds if (r_type == R_390_8)
252be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 8, 0, write);
2531da177e4SLinus Torvalds else if (r_type == R_390_12)
254be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 12, 0, write);
2551da177e4SLinus Torvalds else if (r_type == R_390_16)
256be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 16, 0, write);
2571da177e4SLinus Torvalds else if (r_type == R_390_20)
258be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 20, 0, write);
2591da177e4SLinus Torvalds else if (r_type == R_390_32)
260be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 32, 0, write);
2611da177e4SLinus Torvalds else if (r_type == R_390_64)
262be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 64, 0, write);
2631da177e4SLinus Torvalds break;
2641da177e4SLinus Torvalds case R_390_PC16: /* PC relative 16 bit. */
2651da177e4SLinus Torvalds case R_390_PC16DBL: /* PC relative 16 bit shifted by 1. */
2661da177e4SLinus Torvalds case R_390_PC32DBL: /* PC relative 32 bit shifted by 1. */
2671da177e4SLinus Torvalds case R_390_PC32: /* PC relative 32 bit. */
2681da177e4SLinus Torvalds case R_390_PC64: /* PC relative 64 bit. */
2691da177e4SLinus Torvalds val += rela->r_addend - loc;
2701da177e4SLinus Torvalds if (r_type == R_390_PC16)
271be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 16, 0, write);
2721da177e4SLinus Torvalds else if (r_type == R_390_PC16DBL)
273be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 16, 1, write);
2741da177e4SLinus Torvalds else if (r_type == R_390_PC32DBL)
275be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 32, 1, write);
2761da177e4SLinus Torvalds else if (r_type == R_390_PC32)
277be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 32, 0, write);
2781da177e4SLinus Torvalds else if (r_type == R_390_PC64)
279be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 64, 0, write);
2801da177e4SLinus Torvalds break;
2811da177e4SLinus Torvalds case R_390_GOT12: /* 12 bit GOT offset. */
2821da177e4SLinus Torvalds case R_390_GOT16: /* 16 bit GOT offset. */
2831da177e4SLinus Torvalds case R_390_GOT20: /* 20 bit GOT offset. */
2841da177e4SLinus Torvalds case R_390_GOT32: /* 32 bit GOT offset. */
2851da177e4SLinus Torvalds case R_390_GOT64: /* 64 bit GOT offset. */
2861da177e4SLinus Torvalds case R_390_GOTENT: /* 32 bit PC rel. to GOT entry shifted by 1. */
2871da177e4SLinus Torvalds case R_390_GOTPLT12: /* 12 bit offset to jump slot. */
2881da177e4SLinus Torvalds case R_390_GOTPLT20: /* 20 bit offset to jump slot. */
2891da177e4SLinus Torvalds case R_390_GOTPLT16: /* 16 bit offset to jump slot. */
2901da177e4SLinus Torvalds case R_390_GOTPLT32: /* 32 bit offset to jump slot. */
2911da177e4SLinus Torvalds case R_390_GOTPLT64: /* 64 bit offset to jump slot. */
2921da177e4SLinus Torvalds case R_390_GOTPLTENT: /* 32 bit rel. offset to jump slot >> 1. */
2931da177e4SLinus Torvalds if (info->got_initialized == 0) {
294ac3b4328SSong Liu Elf_Addr *gotent = me->mem[MOD_TEXT].base +
295be242261SPeter Zijlstra me->arch.got_offset +
2961da177e4SLinus Torvalds info->got_offset;
297be242261SPeter Zijlstra
298be242261SPeter Zijlstra write(gotent, &val, sizeof(*gotent));
2991da177e4SLinus Torvalds info->got_initialized = 1;
3001da177e4SLinus Torvalds }
3011da177e4SLinus Torvalds val = info->got_offset + rela->r_addend;
3021da177e4SLinus Torvalds if (r_type == R_390_GOT12 ||
3031da177e4SLinus Torvalds r_type == R_390_GOTPLT12)
304be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 12, 0, write);
3051da177e4SLinus Torvalds else if (r_type == R_390_GOT16 ||
3061da177e4SLinus Torvalds r_type == R_390_GOTPLT16)
307be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 16, 0, write);
3081da177e4SLinus Torvalds else if (r_type == R_390_GOT20 ||
3091da177e4SLinus Torvalds r_type == R_390_GOTPLT20)
310be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 20, 0, write);
3111da177e4SLinus Torvalds else if (r_type == R_390_GOT32 ||
3121da177e4SLinus Torvalds r_type == R_390_GOTPLT32)
313be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 32, 0, write);
3141da177e4SLinus Torvalds else if (r_type == R_390_GOT64 ||
3151da177e4SLinus Torvalds r_type == R_390_GOTPLT64)
316be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 64, 0, write);
317083e14c0SMartin Schwidefsky else if (r_type == R_390_GOTENT ||
318083e14c0SMartin Schwidefsky r_type == R_390_GOTPLTENT) {
31911458e2bSSumanth Korikkar val += (Elf_Addr)me->mem[MOD_TEXT].base +
32011458e2bSSumanth Korikkar me->arch.got_offset - loc;
321be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 32, 1, write);
322083e14c0SMartin Schwidefsky }
3231da177e4SLinus Torvalds break;
3241da177e4SLinus Torvalds case R_390_PLT16DBL: /* 16 bit PC rel. PLT shifted by 1. */
3251da177e4SLinus Torvalds case R_390_PLT32DBL: /* 32 bit PC rel. PLT shifted by 1. */
3261da177e4SLinus Torvalds case R_390_PLT32: /* 32 bit PC relative PLT address. */
3271da177e4SLinus Torvalds case R_390_PLT64: /* 64 bit PC relative PLT address. */
3281da177e4SLinus Torvalds case R_390_PLTOFF16: /* 16 bit offset from GOT to PLT. */
3291da177e4SLinus Torvalds case R_390_PLTOFF32: /* 32 bit offset from GOT to PLT. */
3301da177e4SLinus Torvalds case R_390_PLTOFF64: /* 16 bit offset from GOT to PLT. */
3311da177e4SLinus Torvalds if (info->plt_initialized == 0) {
332f3b7e73bSIlya Leoshkevich unsigned char insn[PLT_ENTRY_SIZE];
333f3b7e73bSIlya Leoshkevich char *plt_base;
334f3b7e73bSIlya Leoshkevich char *ip;
335be242261SPeter Zijlstra
336ac3b4328SSong Liu plt_base = me->mem[MOD_TEXT].base + me->arch.plt_offset;
337f3b7e73bSIlya Leoshkevich ip = plt_base + info->plt_offset;
338f3b7e73bSIlya Leoshkevich *(int *)insn = 0x0d10e310; /* basr 1,0 */
339f3b7e73bSIlya Leoshkevich *(int *)&insn[4] = 0x100c0004; /* lg 1,12(1) */
3406e179d64SMartin Schwidefsky if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_disable) {
341f3b7e73bSIlya Leoshkevich char *jump_r1;
342f3b7e73bSIlya Leoshkevich
343f3b7e73bSIlya Leoshkevich jump_r1 = plt_base + me->arch.plt_size -
344f3b7e73bSIlya Leoshkevich PLT_ENTRY_SIZE;
345f3b7e73bSIlya Leoshkevich /* brcl 0xf,__jump_r1 */
346f3b7e73bSIlya Leoshkevich *(short *)&insn[8] = 0xc0f4;
347f3b7e73bSIlya Leoshkevich *(int *)&insn[10] = (jump_r1 - (ip + 8)) / 2;
348f19fbd5eSMartin Schwidefsky } else {
349f3b7e73bSIlya Leoshkevich *(int *)&insn[8] = 0x07f10000; /* br %r1 */
350f19fbd5eSMartin Schwidefsky }
351f3b7e73bSIlya Leoshkevich *(long *)&insn[14] = val;
352be242261SPeter Zijlstra
353be242261SPeter Zijlstra write(ip, insn, sizeof(insn));
3541da177e4SLinus Torvalds info->plt_initialized = 1;
3551da177e4SLinus Torvalds }
3561da177e4SLinus Torvalds if (r_type == R_390_PLTOFF16 ||
357504665a9SMartin Schwidefsky r_type == R_390_PLTOFF32 ||
358504665a9SMartin Schwidefsky r_type == R_390_PLTOFF64)
3591da177e4SLinus Torvalds val = me->arch.plt_offset - me->arch.got_offset +
3601da177e4SLinus Torvalds info->plt_offset + rela->r_addend;
361504665a9SMartin Schwidefsky else {
362504665a9SMartin Schwidefsky if (!((r_type == R_390_PLT16DBL &&
363504665a9SMartin Schwidefsky val - loc + 0xffffUL < 0x1ffffeUL) ||
364504665a9SMartin Schwidefsky (r_type == R_390_PLT32DBL &&
365504665a9SMartin Schwidefsky val - loc + 0xffffffffULL < 0x1fffffffeULL)))
366ac3b4328SSong Liu val = (Elf_Addr) me->mem[MOD_TEXT].base +
367504665a9SMartin Schwidefsky me->arch.plt_offset +
368504665a9SMartin Schwidefsky info->plt_offset;
369504665a9SMartin Schwidefsky val += rela->r_addend - loc;
370504665a9SMartin Schwidefsky }
3711da177e4SLinus Torvalds if (r_type == R_390_PLT16DBL)
372be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 16, 1, write);
3731da177e4SLinus Torvalds else if (r_type == R_390_PLTOFF16)
374be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 16, 0, write);
3751da177e4SLinus Torvalds else if (r_type == R_390_PLT32DBL)
376be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 32, 1, write);
3771da177e4SLinus Torvalds else if (r_type == R_390_PLT32 ||
3781da177e4SLinus Torvalds r_type == R_390_PLTOFF32)
379be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 32, 0, write);
3801da177e4SLinus Torvalds else if (r_type == R_390_PLT64 ||
3811da177e4SLinus Torvalds r_type == R_390_PLTOFF64)
382be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 64, 0, write);
3831da177e4SLinus Torvalds break;
3841da177e4SLinus Torvalds case R_390_GOTOFF16: /* 16 bit offset to GOT. */
3851da177e4SLinus Torvalds case R_390_GOTOFF32: /* 32 bit offset to GOT. */
3861da177e4SLinus Torvalds case R_390_GOTOFF64: /* 64 bit offset to GOT. */
3871da177e4SLinus Torvalds val = val + rela->r_addend -
388ac3b4328SSong Liu ((Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset);
3891da177e4SLinus Torvalds if (r_type == R_390_GOTOFF16)
390be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 16, 0, write);
3911da177e4SLinus Torvalds else if (r_type == R_390_GOTOFF32)
392be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 32, 0, write);
3931da177e4SLinus Torvalds else if (r_type == R_390_GOTOFF64)
394be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 0, 64, 0, write);
3951da177e4SLinus Torvalds break;
3961da177e4SLinus Torvalds case R_390_GOTPC: /* 32 bit PC relative offset to GOT. */
3971da177e4SLinus Torvalds case R_390_GOTPCDBL: /* 32 bit PC rel. off. to GOT shifted by 1. */
398ac3b4328SSong Liu val = (Elf_Addr) me->mem[MOD_TEXT].base + me->arch.got_offset +
3991da177e4SLinus Torvalds rela->r_addend - loc;
4001da177e4SLinus Torvalds if (r_type == R_390_GOTPC)
401be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 32, 0, write);
4021da177e4SLinus Torvalds else if (r_type == R_390_GOTPCDBL)
403be242261SPeter Zijlstra rc = apply_rela_bits(loc, val, 1, 32, 1, write);
4041da177e4SLinus Torvalds break;
4051da177e4SLinus Torvalds case R_390_COPY:
4061da177e4SLinus Torvalds case R_390_GLOB_DAT: /* Create GOT entry. */
4071da177e4SLinus Torvalds case R_390_JMP_SLOT: /* Create PLT entry. */
4081da177e4SLinus Torvalds case R_390_RELATIVE: /* Adjust by program base. */
4091da177e4SLinus Torvalds /* Only needed if we want to support loading of
4101da177e4SLinus Torvalds modules linked with -shared. */
411083e14c0SMartin Schwidefsky return -ENOEXEC;
4121da177e4SLinus Torvalds default:
413083e14c0SMartin Schwidefsky printk(KERN_ERR "module %s: unknown relocation: %u\n",
4141da177e4SLinus Torvalds me->name, r_type);
4151da177e4SLinus Torvalds return -ENOEXEC;
4161da177e4SLinus Torvalds }
417083e14c0SMartin Schwidefsky if (rc) {
418083e14c0SMartin Schwidefsky printk(KERN_ERR "module %s: relocation error for symbol %s "
419083e14c0SMartin Schwidefsky "(r_type %i, value 0x%lx)\n",
420083e14c0SMartin Schwidefsky me->name, strtab + symtab[r_sym].st_name,
421083e14c0SMartin Schwidefsky r_type, (unsigned long) val);
422083e14c0SMartin Schwidefsky return rc;
423083e14c0SMartin Schwidefsky }
4241da177e4SLinus Torvalds return 0;
4251da177e4SLinus Torvalds }
4261da177e4SLinus Torvalds
__apply_relocate_add(Elf_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me,void * (* write)(void * dest,const void * src,size_t len))427be242261SPeter Zijlstra static int __apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
4281da177e4SLinus Torvalds unsigned int symindex, unsigned int relsec,
429be242261SPeter Zijlstra struct module *me,
430be242261SPeter Zijlstra void *(*write)(void *dest, const void *src, size_t len))
4311da177e4SLinus Torvalds {
4321da177e4SLinus Torvalds Elf_Addr base;
4331da177e4SLinus Torvalds Elf_Sym *symtab;
4341da177e4SLinus Torvalds Elf_Rela *rela;
4351da177e4SLinus Torvalds unsigned long i, n;
4361da177e4SLinus Torvalds int rc;
4371da177e4SLinus Torvalds
4381da177e4SLinus Torvalds DEBUGP("Applying relocate section %u to %u\n",
4391da177e4SLinus Torvalds relsec, sechdrs[relsec].sh_info);
4401da177e4SLinus Torvalds base = sechdrs[sechdrs[relsec].sh_info].sh_addr;
4411da177e4SLinus Torvalds symtab = (Elf_Sym *) sechdrs[symindex].sh_addr;
4421da177e4SLinus Torvalds rela = (Elf_Rela *) sechdrs[relsec].sh_addr;
4431da177e4SLinus Torvalds n = sechdrs[relsec].sh_size / sizeof(Elf_Rela);
4441da177e4SLinus Torvalds
4451da177e4SLinus Torvalds for (i = 0; i < n; i++, rela++) {
446be242261SPeter Zijlstra rc = apply_rela(rela, base, symtab, strtab, me, write);
4471da177e4SLinus Torvalds if (rc)
4481da177e4SLinus Torvalds return rc;
4491da177e4SLinus Torvalds }
4501da177e4SLinus Torvalds return 0;
4511da177e4SLinus Torvalds }
4521da177e4SLinus Torvalds
apply_relocate_add(Elf_Shdr * sechdrs,const char * strtab,unsigned int symindex,unsigned int relsec,struct module * me)453be242261SPeter Zijlstra int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab,
454be242261SPeter Zijlstra unsigned int symindex, unsigned int relsec,
455be242261SPeter Zijlstra struct module *me)
456be242261SPeter Zijlstra {
457be242261SPeter Zijlstra bool early = me->state == MODULE_STATE_UNFORMED;
458be242261SPeter Zijlstra void *(*write)(void *, const void *, size_t) = memcpy;
459be242261SPeter Zijlstra
460be242261SPeter Zijlstra if (!early)
461be242261SPeter Zijlstra write = s390_kernel_write;
462be242261SPeter Zijlstra
463be242261SPeter Zijlstra return __apply_relocate_add(sechdrs, strtab, symindex, relsec, me,
464be242261SPeter Zijlstra write);
465be242261SPeter Zijlstra }
466be242261SPeter Zijlstra
467de5012b4SIlya Leoshkevich #ifdef CONFIG_FUNCTION_TRACER
module_alloc_ftrace_hotpatch_trampolines(struct module * me,const Elf_Shdr * s)468de5012b4SIlya Leoshkevich static int module_alloc_ftrace_hotpatch_trampolines(struct module *me,
469de5012b4SIlya Leoshkevich const Elf_Shdr *s)
470de5012b4SIlya Leoshkevich {
471de5012b4SIlya Leoshkevich char *start, *end;
472de5012b4SIlya Leoshkevich int numpages;
473de5012b4SIlya Leoshkevich size_t size;
474de5012b4SIlya Leoshkevich
475de5012b4SIlya Leoshkevich size = FTRACE_HOTPATCH_TRAMPOLINES_SIZE(s->sh_size);
476de5012b4SIlya Leoshkevich numpages = DIV_ROUND_UP(size, PAGE_SIZE);
477*12af2b83SMike Rapoport (IBM) start = execmem_alloc(EXECMEM_FTRACE, numpages * PAGE_SIZE);
478de5012b4SIlya Leoshkevich if (!start)
479de5012b4SIlya Leoshkevich return -ENOMEM;
4807c7ab788SHeiko Carstens set_memory_rox((unsigned long)start, numpages);
481de5012b4SIlya Leoshkevich end = start + size;
482de5012b4SIlya Leoshkevich
483de5012b4SIlya Leoshkevich me->arch.trampolines_start = (struct ftrace_hotpatch_trampoline *)start;
484de5012b4SIlya Leoshkevich me->arch.trampolines_end = (struct ftrace_hotpatch_trampoline *)end;
485de5012b4SIlya Leoshkevich me->arch.next_trampoline = me->arch.trampolines_start;
486de5012b4SIlya Leoshkevich
487de5012b4SIlya Leoshkevich return 0;
488de5012b4SIlya Leoshkevich }
489de5012b4SIlya Leoshkevich #endif /* CONFIG_FUNCTION_TRACER */
490de5012b4SIlya Leoshkevich
module_finalize(const Elf_Ehdr * hdr,const Elf_Shdr * sechdrs,struct module * me)4911da177e4SLinus Torvalds int module_finalize(const Elf_Ehdr *hdr,
4921da177e4SLinus Torvalds const Elf_Shdr *sechdrs,
4931da177e4SLinus Torvalds struct module *me)
4941da177e4SLinus Torvalds {
495686140a1SVasily Gorbik const Elf_Shdr *s;
496f19fbd5eSMartin Schwidefsky char *secstrings, *secname;
497f19fbd5eSMartin Schwidefsky void *aseg;
498de5012b4SIlya Leoshkevich #ifdef CONFIG_FUNCTION_TRACER
499de5012b4SIlya Leoshkevich int ret;
500de5012b4SIlya Leoshkevich #endif
501f19fbd5eSMartin Schwidefsky
502f19fbd5eSMartin Schwidefsky if (IS_ENABLED(CONFIG_EXPOLINE) &&
5036e179d64SMartin Schwidefsky !nospec_disable && me->arch.plt_size) {
504f19fbd5eSMartin Schwidefsky unsigned int *ij;
505f19fbd5eSMartin Schwidefsky
506ac3b4328SSong Liu ij = me->mem[MOD_TEXT].base + me->arch.plt_offset +
507f19fbd5eSMartin Schwidefsky me->arch.plt_size - PLT_ENTRY_SIZE;
508f19fbd5eSMartin Schwidefsky ij[0] = 0xc6000000; /* exrl %r0,.+10 */
509f19fbd5eSMartin Schwidefsky ij[1] = 0x0005a7f4; /* j . */
510f19fbd5eSMartin Schwidefsky ij[2] = 0x000007f1; /* br %r1 */
511f19fbd5eSMartin Schwidefsky }
512686140a1SVasily Gorbik
513686140a1SVasily Gorbik secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
514686140a1SVasily Gorbik for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
515f19fbd5eSMartin Schwidefsky aseg = (void *) s->sh_addr;
516f19fbd5eSMartin Schwidefsky secname = secstrings + s->sh_name;
517686140a1SVasily Gorbik
518f19fbd5eSMartin Schwidefsky if (!strcmp(".altinstructions", secname))
519f19fbd5eSMartin Schwidefsky /* patch .altinstructions */
520686140a1SVasily Gorbik apply_alternatives(aseg, aseg + s->sh_size);
521f19fbd5eSMartin Schwidefsky
522f19fbd5eSMartin Schwidefsky if (IS_ENABLED(CONFIG_EXPOLINE) &&
523b29cd7c4SVasily Gorbik (str_has_prefix(secname, ".s390_indirect")))
5246e179d64SMartin Schwidefsky nospec_revert(aseg, aseg + s->sh_size);
525f19fbd5eSMartin Schwidefsky
526f19fbd5eSMartin Schwidefsky if (IS_ENABLED(CONFIG_EXPOLINE) &&
527b29cd7c4SVasily Gorbik (str_has_prefix(secname, ".s390_return")))
5286e179d64SMartin Schwidefsky nospec_revert(aseg, aseg + s->sh_size);
529de5012b4SIlya Leoshkevich
530de5012b4SIlya Leoshkevich #ifdef CONFIG_FUNCTION_TRACER
531de5012b4SIlya Leoshkevich if (!strcmp(FTRACE_CALLSITE_SECTION, secname)) {
532de5012b4SIlya Leoshkevich ret = module_alloc_ftrace_hotpatch_trampolines(me, s);
533de5012b4SIlya Leoshkevich if (ret < 0)
534de5012b4SIlya Leoshkevich return ret;
535de5012b4SIlya Leoshkevich }
536de5012b4SIlya Leoshkevich #endif /* CONFIG_FUNCTION_TRACER */
537686140a1SVasily Gorbik }
538686140a1SVasily Gorbik
5395336377dSLinus Torvalds return 0;
5401da177e4SLinus Torvalds }
541