1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Kernel module help for x86. 3 Copyright (C) 2001 Rusty Russell. 4 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/moduleloader.h> 10 #include <linux/elf.h> 11 #include <linux/vmalloc.h> 12 #include <linux/fs.h> 13 #include <linux/string.h> 14 #include <linux/kernel.h> 15 #include <linux/kasan.h> 16 #include <linux/bug.h> 17 #include <linux/mm.h> 18 #include <linux/gfp.h> 19 #include <linux/jump_label.h> 20 #include <linux/random.h> 21 #include <linux/memory.h> 22 23 #include <asm/text-patching.h> 24 #include <asm/page.h> 25 #include <asm/setup.h> 26 #include <asm/unwind.h> 27 28 #if 0 29 #define DEBUGP(fmt, ...) \ 30 printk(KERN_DEBUG fmt, ##__VA_ARGS__) 31 #else 32 #define DEBUGP(fmt, ...) \ 33 do { \ 34 if (0) \ 35 printk(KERN_DEBUG fmt, ##__VA_ARGS__); \ 36 } while (0) 37 #endif 38 39 #ifdef CONFIG_X86_32 40 int apply_relocate(Elf32_Shdr *sechdrs, 41 const char *strtab, 42 unsigned int symindex, 43 unsigned int relsec, 44 struct module *me) 45 { 46 unsigned int i; 47 Elf32_Rel *rel = (void *)sechdrs[relsec].sh_addr; 48 Elf32_Sym *sym; 49 uint32_t *location; 50 51 DEBUGP("Applying relocate section %u to %u\n", 52 relsec, sechdrs[relsec].sh_info); 53 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 54 /* This is where to make the change */ 55 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 56 + rel[i].r_offset; 57 /* This is the symbol it is referring to. Note that all 58 undefined symbols have been resolved. */ 59 sym = (Elf32_Sym *)sechdrs[symindex].sh_addr 60 + ELF32_R_SYM(rel[i].r_info); 61 62 switch (ELF32_R_TYPE(rel[i].r_info)) { 63 case R_386_32: 64 /* We add the value into the location given */ 65 *location += sym->st_value; 66 break; 67 case R_386_PC32: 68 case R_386_PLT32: 69 /* Add the value, subtract its position */ 70 *location += sym->st_value - (uint32_t)location; 71 break; 72 default: 73 pr_err("%s: Unknown relocation: %u\n", 74 me->name, ELF32_R_TYPE(rel[i].r_info)); 75 return -ENOEXEC; 76 } 77 } 78 return 0; 79 } 80 #else /*X86_64*/ 81 static int __write_relocate_add(Elf64_Shdr *sechdrs, 82 const char *strtab, 83 unsigned int symindex, 84 unsigned int relsec, 85 struct module *me, 86 void *(*write)(void *dest, const void *src, size_t len), 87 bool apply) 88 { 89 unsigned int i; 90 Elf64_Rela *rel = (void *)sechdrs[relsec].sh_addr; 91 Elf64_Sym *sym; 92 void *loc; 93 u64 val; 94 u64 zero = 0ULL; 95 96 DEBUGP("%s relocate section %u to %u\n", 97 apply ? "Applying" : "Clearing", 98 relsec, sechdrs[relsec].sh_info); 99 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 100 size_t size; 101 102 /* This is where to make the change */ 103 loc = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr 104 + rel[i].r_offset; 105 106 /* This is the symbol it is referring to. Note that all 107 undefined symbols have been resolved. */ 108 sym = (Elf64_Sym *)sechdrs[symindex].sh_addr 109 + ELF64_R_SYM(rel[i].r_info); 110 111 DEBUGP("type %d st_value %Lx r_addend %Lx loc %Lx\n", 112 (int)ELF64_R_TYPE(rel[i].r_info), 113 sym->st_value, rel[i].r_addend, (u64)loc); 114 115 val = sym->st_value + rel[i].r_addend; 116 117 switch (ELF64_R_TYPE(rel[i].r_info)) { 118 case R_X86_64_NONE: 119 continue; /* nothing to write */ 120 case R_X86_64_64: 121 size = 8; 122 break; 123 case R_X86_64_32: 124 if (val != *(u32 *)&val) 125 goto overflow; 126 size = 4; 127 break; 128 case R_X86_64_32S: 129 if ((s64)val != *(s32 *)&val) 130 goto overflow; 131 size = 4; 132 break; 133 case R_X86_64_PC32: 134 case R_X86_64_PLT32: 135 val -= (u64)loc; 136 size = 4; 137 break; 138 case R_X86_64_PC64: 139 val -= (u64)loc; 140 size = 8; 141 break; 142 default: 143 pr_err("%s: Unknown rela relocation: %llu\n", 144 me->name, ELF64_R_TYPE(rel[i].r_info)); 145 return -ENOEXEC; 146 } 147 148 if (apply) { 149 void *wr_loc = module_writable_address(me, loc); 150 151 if (memcmp(wr_loc, &zero, size)) { 152 pr_err("x86/modules: Invalid relocation target, existing value is nonzero for type %d, loc %p, val %Lx\n", 153 (int)ELF64_R_TYPE(rel[i].r_info), loc, val); 154 return -ENOEXEC; 155 } 156 write(wr_loc, &val, size); 157 } else { 158 if (memcmp(loc, &val, size)) { 159 pr_warn("x86/modules: Invalid relocation target, existing value does not match expected value for type %d, loc %p, val %Lx\n", 160 (int)ELF64_R_TYPE(rel[i].r_info), loc, val); 161 return -ENOEXEC; 162 } 163 /* FIXME: needs care for ROX module allocations */ 164 write(loc, &zero, size); 165 } 166 } 167 return 0; 168 169 overflow: 170 pr_err("overflow in relocation type %d val %Lx\n", 171 (int)ELF64_R_TYPE(rel[i].r_info), val); 172 pr_err("`%s' likely not compiled with -mcmodel=kernel\n", 173 me->name); 174 return -ENOEXEC; 175 } 176 177 static int write_relocate_add(Elf64_Shdr *sechdrs, 178 const char *strtab, 179 unsigned int symindex, 180 unsigned int relsec, 181 struct module *me, 182 bool apply) 183 { 184 int ret; 185 bool early = me->state == MODULE_STATE_UNFORMED; 186 void *(*write)(void *, const void *, size_t) = memcpy; 187 188 if (!early) { 189 write = text_poke; 190 mutex_lock(&text_mutex); 191 } 192 193 ret = __write_relocate_add(sechdrs, strtab, symindex, relsec, me, 194 write, apply); 195 196 if (!early) { 197 text_poke_sync(); 198 mutex_unlock(&text_mutex); 199 } 200 201 return ret; 202 } 203 204 int apply_relocate_add(Elf64_Shdr *sechdrs, 205 const char *strtab, 206 unsigned int symindex, 207 unsigned int relsec, 208 struct module *me) 209 { 210 return write_relocate_add(sechdrs, strtab, symindex, relsec, me, true); 211 } 212 213 #ifdef CONFIG_LIVEPATCH 214 void clear_relocate_add(Elf64_Shdr *sechdrs, 215 const char *strtab, 216 unsigned int symindex, 217 unsigned int relsec, 218 struct module *me) 219 { 220 write_relocate_add(sechdrs, strtab, symindex, relsec, me, false); 221 } 222 #endif 223 224 #endif 225 226 int module_finalize(const Elf_Ehdr *hdr, 227 const Elf_Shdr *sechdrs, 228 struct module *me) 229 { 230 const Elf_Shdr *s, *alt = NULL, 231 *orc = NULL, *orc_ip = NULL, 232 *retpolines = NULL, *returns = NULL, *ibt_endbr = NULL, 233 *calls = NULL, *cfi = NULL; 234 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 235 236 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 237 if (!strcmp(".altinstructions", secstrings + s->sh_name)) 238 alt = s; 239 if (!strcmp(".orc_unwind", secstrings + s->sh_name)) 240 orc = s; 241 if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) 242 orc_ip = s; 243 if (!strcmp(".retpoline_sites", secstrings + s->sh_name)) 244 retpolines = s; 245 if (!strcmp(".return_sites", secstrings + s->sh_name)) 246 returns = s; 247 if (!strcmp(".call_sites", secstrings + s->sh_name)) 248 calls = s; 249 if (!strcmp(".cfi_sites", secstrings + s->sh_name)) 250 cfi = s; 251 if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name)) 252 ibt_endbr = s; 253 } 254 255 if (retpolines || cfi) { 256 void *rseg = NULL, *cseg = NULL; 257 unsigned int rsize = 0, csize = 0; 258 259 if (retpolines) { 260 rseg = (void *)retpolines->sh_addr; 261 rsize = retpolines->sh_size; 262 } 263 264 if (cfi) { 265 cseg = (void *)cfi->sh_addr; 266 csize = cfi->sh_size; 267 } 268 269 apply_fineibt(rseg, rseg + rsize, cseg, cseg + csize, me); 270 } 271 if (retpolines) { 272 void *rseg = (void *)retpolines->sh_addr; 273 apply_retpolines(rseg, rseg + retpolines->sh_size, me); 274 } 275 if (returns) { 276 void *rseg = (void *)returns->sh_addr; 277 apply_returns(rseg, rseg + returns->sh_size, me); 278 } 279 if (alt) { 280 /* patch .altinstructions */ 281 void *aseg = (void *)alt->sh_addr; 282 apply_alternatives(aseg, aseg + alt->sh_size, me); 283 } 284 if (calls || alt) { 285 struct callthunk_sites cs = {}; 286 287 if (calls) { 288 cs.call_start = (void *)calls->sh_addr; 289 cs.call_end = (void *)calls->sh_addr + calls->sh_size; 290 } 291 292 if (alt) { 293 cs.alt_start = (void *)alt->sh_addr; 294 cs.alt_end = (void *)alt->sh_addr + alt->sh_size; 295 } 296 297 callthunks_patch_module_calls(&cs, me); 298 } 299 if (ibt_endbr) { 300 void *iseg = (void *)ibt_endbr->sh_addr; 301 apply_seal_endbr(iseg, iseg + ibt_endbr->sh_size, me); 302 } 303 304 if (orc && orc_ip) 305 unwind_module_init(me, (void *)orc_ip->sh_addr, orc_ip->sh_size, 306 (void *)orc->sh_addr, orc->sh_size); 307 308 return 0; 309 } 310 311 int module_post_finalize(const Elf_Ehdr *hdr, 312 const Elf_Shdr *sechdrs, 313 struct module *me) 314 { 315 const Elf_Shdr *s, *locks = NULL; 316 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 317 318 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 319 if (!strcmp(".smp_locks", secstrings + s->sh_name)) 320 locks = s; 321 } 322 323 if (locks) { 324 void *lseg = (void *)locks->sh_addr; 325 void *text = me->mem[MOD_TEXT].base; 326 void *text_end = text + me->mem[MOD_TEXT].size; 327 alternatives_smp_module_add(me, me->name, 328 lseg, lseg + locks->sh_size, 329 text, text_end); 330 } 331 332 return 0; 333 } 334 335 void module_arch_cleanup(struct module *mod) 336 { 337 alternatives_smp_module_del(mod); 338 } 339