1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1998-2000 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include "opt_ddb.h" 31 #include "opt_gdb.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #ifdef SPARSE_MAPPING 39 #include <sys/mman.h> 40 #endif 41 #include <sys/mutex.h> 42 #include <sys/mount.h> 43 #include <sys/pcpu.h> 44 #include <sys/proc.h> 45 #include <sys/namei.h> 46 #include <sys/fcntl.h> 47 #include <sys/vnode.h> 48 #include <sys/linker.h> 49 #include <sys/sysctl.h> 50 #include <sys/tslog.h> 51 52 #include <machine/elf.h> 53 54 #include <net/vnet.h> 55 56 #include <security/mac/mac_framework.h> 57 58 #include <vm/vm.h> 59 #include <vm/vm_param.h> 60 #ifdef SPARSE_MAPPING 61 #include <vm/vm_object.h> 62 #include <vm/vm_kern.h> 63 #include <vm/vm_extern.h> 64 #endif 65 #include <vm/pmap.h> 66 #include <vm/vm_map.h> 67 68 #include <sys/link_elf.h> 69 70 #include "linker_if.h" 71 72 #define MAXSEGS 4 73 74 typedef struct elf_file { 75 struct linker_file lf; /* Common fields */ 76 int preloaded; /* Was file pre-loaded */ 77 caddr_t address; /* Relocation address */ 78 #ifdef SPARSE_MAPPING 79 vm_object_t object; /* VM object to hold file pages */ 80 #endif 81 Elf_Dyn *dynamic; /* Symbol table etc. */ 82 Elf_Hashelt nbuckets; /* DT_HASH info */ 83 Elf_Hashelt nchains; 84 const Elf_Hashelt *buckets; 85 const Elf_Hashelt *chains; 86 caddr_t hash; 87 caddr_t strtab; /* DT_STRTAB */ 88 int strsz; /* DT_STRSZ */ 89 const Elf_Sym *symtab; /* DT_SYMTAB */ 90 Elf_Addr *got; /* DT_PLTGOT */ 91 const Elf_Rel *pltrel; /* DT_JMPREL */ 92 int pltrelsize; /* DT_PLTRELSZ */ 93 const Elf_Rela *pltrela; /* DT_JMPREL */ 94 int pltrelasize; /* DT_PLTRELSZ */ 95 const Elf_Rel *rel; /* DT_REL */ 96 int relsize; /* DT_RELSZ */ 97 const Elf_Rela *rela; /* DT_RELA */ 98 int relasize; /* DT_RELASZ */ 99 caddr_t modptr; 100 const Elf_Sym *ddbsymtab; /* The symbol table we are using */ 101 long ddbsymcnt; /* Number of symbols */ 102 caddr_t ddbstrtab; /* String table */ 103 long ddbstrcnt; /* number of bytes in string table */ 104 caddr_t symbase; /* malloc'ed symbold base */ 105 caddr_t strbase; /* malloc'ed string base */ 106 caddr_t ctftab; /* CTF table */ 107 long ctfcnt; /* number of bytes in CTF table */ 108 caddr_t ctfoff; /* CTF offset table */ 109 caddr_t typoff; /* Type offset table */ 110 long typlen; /* Number of type entries. */ 111 Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */ 112 Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */ 113 Elf_Addr pcpu_base; /* Relocated pcpu set address. */ 114 #ifdef VIMAGE 115 Elf_Addr vnet_start; /* Pre-relocation vnet set start. */ 116 Elf_Addr vnet_stop; /* Pre-relocation vnet set stop. */ 117 Elf_Addr vnet_base; /* Relocated vnet set address. */ 118 #endif 119 #ifdef GDB 120 struct link_map gdb; /* hooks for gdb */ 121 #endif 122 } *elf_file_t; 123 124 struct elf_set { 125 Elf_Addr es_start; 126 Elf_Addr es_stop; 127 Elf_Addr es_base; 128 TAILQ_ENTRY(elf_set) es_link; 129 }; 130 131 TAILQ_HEAD(elf_set_head, elf_set); 132 133 #include <kern/kern_ctf.c> 134 135 static int link_elf_link_common_finish(linker_file_t); 136 static int link_elf_link_preload(linker_class_t cls, 137 const char *, linker_file_t *); 138 static int link_elf_link_preload_finish(linker_file_t); 139 static int link_elf_load_file(linker_class_t, const char *, 140 linker_file_t *); 141 static int link_elf_lookup_symbol(linker_file_t, const char *, 142 c_linker_sym_t *); 143 static int link_elf_lookup_debug_symbol(linker_file_t, const char *, 144 c_linker_sym_t *); 145 static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, 146 linker_symval_t *); 147 static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t, 148 linker_symval_t*); 149 static int link_elf_search_symbol(linker_file_t, caddr_t, 150 c_linker_sym_t *, long *); 151 152 static void link_elf_unload_file(linker_file_t); 153 static void link_elf_unload_preload(linker_file_t); 154 static int link_elf_lookup_set(linker_file_t, const char *, 155 void ***, void ***, int *); 156 static int link_elf_each_function_name(linker_file_t, 157 int (*)(const char *, void *), void *); 158 static int link_elf_each_function_nameval(linker_file_t, 159 linker_function_nameval_callback_t, void *); 160 static void link_elf_reloc_local(linker_file_t); 161 static long link_elf_symtab_get(linker_file_t, const Elf_Sym **); 162 static long link_elf_strtab_get(linker_file_t, caddr_t *); 163 #ifdef VIMAGE 164 static void link_elf_propagate_vnets(linker_file_t); 165 #endif 166 static int elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *); 167 168 static kobj_method_t link_elf_methods[] = { 169 KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol), 170 KOBJMETHOD(linker_lookup_debug_symbol, link_elf_lookup_debug_symbol), 171 KOBJMETHOD(linker_symbol_values, link_elf_symbol_values), 172 KOBJMETHOD(linker_debug_symbol_values, link_elf_debug_symbol_values), 173 KOBJMETHOD(linker_search_symbol, link_elf_search_symbol), 174 KOBJMETHOD(linker_unload, link_elf_unload_file), 175 KOBJMETHOD(linker_load_file, link_elf_load_file), 176 KOBJMETHOD(linker_link_preload, link_elf_link_preload), 177 KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish), 178 KOBJMETHOD(linker_lookup_set, link_elf_lookup_set), 179 KOBJMETHOD(linker_each_function_name, link_elf_each_function_name), 180 KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval), 181 KOBJMETHOD(linker_ctf_get, link_elf_ctf_get), 182 KOBJMETHOD(linker_symtab_get, link_elf_symtab_get), 183 KOBJMETHOD(linker_strtab_get, link_elf_strtab_get), 184 #ifdef VIMAGE 185 KOBJMETHOD(linker_propagate_vnets, link_elf_propagate_vnets), 186 #endif 187 KOBJMETHOD_END 188 }; 189 190 static struct linker_class link_elf_class = { 191 #if ELF_TARG_CLASS == ELFCLASS32 192 "elf32", 193 #else 194 "elf64", 195 #endif 196 link_elf_methods, sizeof(struct elf_file) 197 }; 198 199 static bool link_elf_leak_locals = true; 200 SYSCTL_BOOL(_debug, OID_AUTO, link_elf_leak_locals, 201 CTLFLAG_RWTUN, &link_elf_leak_locals, 0, 202 "Allow local symbols to participate in global module symbol resolution"); 203 204 typedef int (*elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, 205 const void *data, int type, elf_lookup_fn lookup); 206 207 static int parse_dynamic(elf_file_t); 208 static int relocate_file(elf_file_t); 209 static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, 210 elf_reloc_fn reloc, bool ifuncs); 211 static int link_elf_preload_parse_symbols(elf_file_t); 212 213 static struct elf_set_head set_pcpu_list; 214 #ifdef VIMAGE 215 static struct elf_set_head set_vnet_list; 216 #endif 217 218 static void 219 elf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base) 220 { 221 struct elf_set *set, *iter; 222 223 set = malloc(sizeof(*set), M_LINKER, M_WAITOK); 224 set->es_start = start; 225 set->es_stop = stop; 226 set->es_base = base; 227 228 TAILQ_FOREACH(iter, list, es_link) { 229 KASSERT((set->es_start < iter->es_start && set->es_stop < iter->es_stop) || 230 (set->es_start > iter->es_start && set->es_stop > iter->es_stop), 231 ("linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx", 232 (uintmax_t)set->es_start, (uintmax_t)set->es_stop, 233 (uintmax_t)iter->es_start, (uintmax_t)iter->es_stop)); 234 235 if (iter->es_start > set->es_start) { 236 TAILQ_INSERT_BEFORE(iter, set, es_link); 237 break; 238 } 239 } 240 241 if (iter == NULL) 242 TAILQ_INSERT_TAIL(list, set, es_link); 243 } 244 245 static int 246 elf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base) 247 { 248 struct elf_set *set; 249 250 TAILQ_FOREACH(set, list, es_link) { 251 if (addr < set->es_start) 252 return (0); 253 if (addr < set->es_stop) { 254 *start = set->es_start; 255 *base = set->es_base; 256 return (1); 257 } 258 } 259 260 return (0); 261 } 262 263 static void 264 elf_set_delete(struct elf_set_head *list, Elf_Addr start) 265 { 266 struct elf_set *set; 267 268 TAILQ_FOREACH(set, list, es_link) { 269 if (start < set->es_start) 270 break; 271 if (start == set->es_start) { 272 TAILQ_REMOVE(list, set, es_link); 273 free(set, M_LINKER); 274 return; 275 } 276 } 277 KASSERT(0, ("deleting unknown linker set (start = 0x%jx)", 278 (uintmax_t)start)); 279 } 280 281 #ifdef GDB 282 static void r_debug_state(struct r_debug *, struct link_map *); 283 284 /* 285 * A list of loaded modules for GDB to use for loading symbols. 286 */ 287 struct r_debug r_debug; 288 289 #define GDB_STATE(s) do { \ 290 r_debug.r_state = s; r_debug_state(NULL, NULL); \ 291 } while (0) 292 293 /* 294 * Function for the debugger to set a breakpoint on to gain control. 295 */ 296 static void 297 r_debug_state(struct r_debug *dummy_one __unused, 298 struct link_map *dummy_two __unused) 299 { 300 } 301 302 static void 303 link_elf_add_gdb(struct link_map *l) 304 { 305 struct link_map *prev; 306 307 l->l_next = NULL; 308 309 if (r_debug.r_map == NULL) { 310 /* Add first. */ 311 l->l_prev = NULL; 312 r_debug.r_map = l; 313 } else { 314 /* Append to list. */ 315 for (prev = r_debug.r_map; 316 prev->l_next != NULL; 317 prev = prev->l_next) 318 ; 319 l->l_prev = prev; 320 prev->l_next = l; 321 } 322 } 323 324 static void 325 link_elf_delete_gdb(struct link_map *l) 326 { 327 if (l->l_prev == NULL) { 328 /* Remove first. */ 329 if ((r_debug.r_map = l->l_next) != NULL) 330 l->l_next->l_prev = NULL; 331 } else { 332 /* Remove any but first. */ 333 if ((l->l_prev->l_next = l->l_next) != NULL) 334 l->l_next->l_prev = l->l_prev; 335 } 336 } 337 #endif /* GDB */ 338 339 /* 340 * The kernel symbol table starts here. 341 */ 342 extern struct _dynamic _DYNAMIC; 343 344 static void 345 link_elf_error(const char *filename, const char *s) 346 { 347 if (filename == NULL) 348 printf("kldload: %s\n", s); 349 else 350 printf("kldload: %s: %s\n", filename, s); 351 } 352 353 static void 354 link_elf_invoke_ctors(caddr_t addr, size_t size) 355 { 356 void (**ctor)(void); 357 size_t i, cnt; 358 359 if (addr == NULL || size == 0) 360 return; 361 cnt = size / sizeof(*ctor); 362 ctor = (void *)addr; 363 for (i = 0; i < cnt; i++) { 364 if (ctor[i] != NULL) 365 (*ctor[i])(); 366 } 367 } 368 369 /* 370 * Actions performed after linking/loading both the preloaded kernel and any 371 * modules; whether preloaded or dynamicly loaded. 372 */ 373 static int 374 link_elf_link_common_finish(linker_file_t lf) 375 { 376 #ifdef GDB 377 elf_file_t ef = (elf_file_t)lf; 378 char *newfilename; 379 #endif 380 int error; 381 382 /* Notify MD code that a module is being loaded. */ 383 error = elf_cpu_load_file(lf); 384 if (error != 0) 385 return (error); 386 387 #ifdef GDB 388 GDB_STATE(RT_ADD); 389 ef->gdb.l_addr = lf->address; 390 newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK); 391 strcpy(newfilename, lf->filename); 392 ef->gdb.l_name = newfilename; 393 ef->gdb.l_ld = ef->dynamic; 394 link_elf_add_gdb(&ef->gdb); 395 GDB_STATE(RT_CONSISTENT); 396 #endif 397 398 /* Invoke .ctors */ 399 link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size); 400 return (0); 401 } 402 403 #ifdef RELOCATABLE_KERNEL 404 /* 405 * __startkernel and __endkernel are symbols set up as relocation canaries. 406 * 407 * They are defined in locore to reference linker script symbols at the 408 * beginning and end of the LOAD area. This has the desired side effect of 409 * giving us variables that have relative relocations pointing at them, so 410 * relocation of the kernel object will cause the variables to be updated 411 * automatically by the runtime linker when we initialize. 412 * 413 * There are two main reasons to relocate the kernel: 414 * 1) If the loader needed to load the kernel at an alternate load address. 415 * 2) If the kernel is switching address spaces on machines like POWER9 416 * under Radix where the high bits of the effective address are used to 417 * differentiate between hypervisor, host, guest, and problem state. 418 */ 419 extern vm_offset_t __startkernel, __endkernel; 420 #endif 421 422 static unsigned long kern_relbase = KERNBASE; 423 424 SYSCTL_ULONG(_kern, OID_AUTO, base_address, CTLFLAG_RD, 425 SYSCTL_NULL_ULONG_PTR, KERNBASE, "Kernel base address"); 426 SYSCTL_ULONG(_kern, OID_AUTO, relbase_address, CTLFLAG_RD, 427 &kern_relbase, 0, "Kernel relocated base address"); 428 429 static void 430 link_elf_init(void* arg) 431 { 432 Elf_Dyn *dp; 433 Elf_Addr *ctors_addrp; 434 Elf_Size *ctors_sizep; 435 caddr_t modptr, baseptr, sizeptr; 436 elf_file_t ef; 437 const char *modname; 438 439 linker_add_class(&link_elf_class); 440 441 dp = (Elf_Dyn *)&_DYNAMIC; 442 modname = NULL; 443 modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel"); 444 if (modptr == NULL) 445 modptr = preload_search_by_type("elf kernel"); 446 modname = (char *)preload_search_info(modptr, MODINFO_NAME); 447 if (modname == NULL) 448 modname = "kernel"; 449 linker_kernel_file = linker_make_file(modname, &link_elf_class); 450 if (linker_kernel_file == NULL) 451 panic("%s: Can't create linker structures for kernel", 452 __func__); 453 454 ef = (elf_file_t) linker_kernel_file; 455 ef->preloaded = 1; 456 #ifdef RELOCATABLE_KERNEL 457 /* Compute relative displacement */ 458 ef->address = (caddr_t) (__startkernel - KERNBASE); 459 #else 460 ef->address = 0; 461 #endif 462 #ifdef SPARSE_MAPPING 463 ef->object = NULL; 464 #endif 465 ef->dynamic = dp; 466 467 if (dp != NULL) 468 parse_dynamic(ef); 469 #ifdef RELOCATABLE_KERNEL 470 linker_kernel_file->address = (caddr_t)__startkernel; 471 linker_kernel_file->size = (intptr_t)(__endkernel - __startkernel); 472 kern_relbase = (unsigned long)__startkernel; 473 #else 474 linker_kernel_file->address += KERNBASE; 475 linker_kernel_file->size = -(intptr_t)linker_kernel_file->address; 476 #endif 477 478 if (modptr != NULL) { 479 ef->modptr = modptr; 480 baseptr = preload_search_info(modptr, MODINFO_ADDR); 481 if (baseptr != NULL) 482 linker_kernel_file->address = *(caddr_t *)baseptr; 483 sizeptr = preload_search_info(modptr, MODINFO_SIZE); 484 if (sizeptr != NULL) 485 linker_kernel_file->size = *(size_t *)sizeptr; 486 ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 487 MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 488 ctors_sizep = (Elf_Size *)preload_search_info(modptr, 489 MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 490 if (ctors_addrp != NULL && ctors_sizep != NULL) { 491 linker_kernel_file->ctors_addr = ef->address + 492 *ctors_addrp; 493 linker_kernel_file->ctors_size = *ctors_sizep; 494 } 495 } 496 (void)link_elf_preload_parse_symbols(ef); 497 498 #ifdef GDB 499 r_debug.r_map = NULL; 500 r_debug.r_brk = r_debug_state; 501 r_debug.r_state = RT_CONSISTENT; 502 #endif 503 504 (void)link_elf_link_common_finish(linker_kernel_file); 505 linker_kernel_file->flags |= LINKER_FILE_LINKED; 506 TAILQ_INIT(&set_pcpu_list); 507 #ifdef VIMAGE 508 TAILQ_INIT(&set_vnet_list); 509 vnet_save_init((void *)VNET_START, VNET_STOP - VNET_START); 510 #endif 511 } 512 513 SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, NULL); 514 515 static int 516 link_elf_preload_parse_symbols(elf_file_t ef) 517 { 518 caddr_t pointer; 519 caddr_t ssym, esym, base; 520 caddr_t strtab; 521 int strcnt; 522 Elf_Sym *symtab; 523 int symcnt; 524 525 if (ef->modptr == NULL) 526 return (0); 527 pointer = preload_search_info(ef->modptr, 528 MODINFO_METADATA | MODINFOMD_SSYM); 529 if (pointer == NULL) 530 return (0); 531 ssym = *(caddr_t *)pointer; 532 pointer = preload_search_info(ef->modptr, 533 MODINFO_METADATA | MODINFOMD_ESYM); 534 if (pointer == NULL) 535 return (0); 536 esym = *(caddr_t *)pointer; 537 538 base = ssym; 539 540 symcnt = *(long *)base; 541 base += sizeof(long); 542 symtab = (Elf_Sym *)base; 543 base += roundup(symcnt, sizeof(long)); 544 545 if (base > esym || base < ssym) { 546 printf("Symbols are corrupt!\n"); 547 return (EINVAL); 548 } 549 550 strcnt = *(long *)base; 551 base += sizeof(long); 552 strtab = base; 553 base += roundup(strcnt, sizeof(long)); 554 555 if (base > esym || base < ssym) { 556 printf("Symbols are corrupt!\n"); 557 return (EINVAL); 558 } 559 560 ef->ddbsymtab = symtab; 561 ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 562 ef->ddbstrtab = strtab; 563 ef->ddbstrcnt = strcnt; 564 565 return (0); 566 } 567 568 static int 569 parse_dynamic(elf_file_t ef) 570 { 571 Elf_Dyn *dp; 572 int plttype = DT_REL; 573 574 for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) { 575 switch (dp->d_tag) { 576 case DT_HASH: 577 { 578 /* From src/libexec/rtld-elf/rtld.c */ 579 const Elf_Hashelt *hashtab = (const Elf_Hashelt *) 580 (ef->address + dp->d_un.d_ptr); 581 ef->nbuckets = hashtab[0]; 582 ef->nchains = hashtab[1]; 583 ef->buckets = hashtab + 2; 584 ef->chains = ef->buckets + ef->nbuckets; 585 break; 586 } 587 case DT_STRTAB: 588 ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr); 589 break; 590 case DT_STRSZ: 591 ef->strsz = dp->d_un.d_val; 592 break; 593 case DT_SYMTAB: 594 ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr); 595 break; 596 case DT_SYMENT: 597 if (dp->d_un.d_val != sizeof(Elf_Sym)) 598 return (ENOEXEC); 599 break; 600 case DT_PLTGOT: 601 ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr); 602 break; 603 case DT_REL: 604 ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 605 break; 606 case DT_RELSZ: 607 ef->relsize = dp->d_un.d_val; 608 break; 609 case DT_RELENT: 610 if (dp->d_un.d_val != sizeof(Elf_Rel)) 611 return (ENOEXEC); 612 break; 613 case DT_JMPREL: 614 ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 615 break; 616 case DT_PLTRELSZ: 617 ef->pltrelsize = dp->d_un.d_val; 618 break; 619 case DT_RELA: 620 ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr); 621 break; 622 case DT_RELASZ: 623 ef->relasize = dp->d_un.d_val; 624 break; 625 case DT_RELAENT: 626 if (dp->d_un.d_val != sizeof(Elf_Rela)) 627 return (ENOEXEC); 628 break; 629 case DT_PLTREL: 630 plttype = dp->d_un.d_val; 631 if (plttype != DT_REL && plttype != DT_RELA) 632 return (ENOEXEC); 633 break; 634 #ifdef GDB 635 case DT_DEBUG: 636 dp->d_un.d_ptr = (Elf_Addr)&r_debug; 637 break; 638 #endif 639 } 640 } 641 642 if (plttype == DT_RELA) { 643 ef->pltrela = (const Elf_Rela *)ef->pltrel; 644 ef->pltrel = NULL; 645 ef->pltrelasize = ef->pltrelsize; 646 ef->pltrelsize = 0; 647 } 648 649 ef->ddbsymtab = ef->symtab; 650 ef->ddbsymcnt = ef->nchains; 651 ef->ddbstrtab = ef->strtab; 652 ef->ddbstrcnt = ef->strsz; 653 654 return elf_cpu_parse_dynamic(ef->address, ef->dynamic); 655 } 656 657 #define LS_PADDING 0x90909090 658 static int 659 parse_dpcpu(elf_file_t ef) 660 { 661 int error, size; 662 #if defined(__i386__) 663 uint32_t pad; 664 #endif 665 666 ef->pcpu_start = 0; 667 ef->pcpu_stop = 0; 668 error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start, 669 (void ***)&ef->pcpu_stop, NULL); 670 /* Error just means there is no pcpu set to relocate. */ 671 if (error != 0) 672 return (0); 673 size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start; 674 /* Empty set? */ 675 if (size < 1) 676 return (0); 677 #if defined(__i386__) 678 /* In case we do find __start/stop_set_ symbols double-check. */ 679 if (size < 4) { 680 uprintf("Kernel module '%s' must be recompiled with " 681 "linker script\n", ef->lf.pathname); 682 return (ENOEXEC); 683 } 684 685 /* Padding from linker-script correct? */ 686 pad = *(uint32_t *)((uintptr_t)ef->pcpu_stop - sizeof(pad)); 687 if (pad != LS_PADDING) { 688 uprintf("Kernel module '%s' must be recompiled with " 689 "linker script, invalid padding %#04x (%#04x)\n", 690 ef->lf.pathname, pad, LS_PADDING); 691 return (ENOEXEC); 692 } 693 /* If we only have valid padding, nothing to do. */ 694 if (size == 4) 695 return (0); 696 #endif 697 /* 698 * Allocate space in the primary pcpu area. Copy in our 699 * initialization from the data section and then initialize 700 * all per-cpu storage from that. 701 */ 702 ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size); 703 if (ef->pcpu_base == 0) { 704 printf("%s: pcpu module space is out of space; " 705 "cannot allocate %d for %s\n", 706 __func__, size, ef->lf.pathname); 707 return (ENOSPC); 708 } 709 memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size); 710 dpcpu_copy((void *)ef->pcpu_base, size); 711 elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop, 712 ef->pcpu_base); 713 714 return (0); 715 } 716 717 #ifdef VIMAGE 718 static int 719 parse_vnet(elf_file_t ef) 720 { 721 int error, size; 722 #if defined(__i386__) 723 uint32_t pad; 724 #endif 725 726 ef->vnet_start = 0; 727 ef->vnet_stop = 0; 728 ef->vnet_base = 0; 729 error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start, 730 (void ***)&ef->vnet_stop, NULL); 731 /* Error just means there is no vnet data set to relocate. */ 732 if (error != 0) 733 return (0); 734 size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start; 735 /* Empty set? */ 736 if (size < 1) 737 return (0); 738 #if defined(__i386__) 739 /* In case we do find __start/stop_set_ symbols double-check. */ 740 if (size < 4) { 741 uprintf("Kernel module '%s' must be recompiled with " 742 "linker script\n", ef->lf.pathname); 743 return (ENOEXEC); 744 } 745 746 /* Padding from linker-script correct? */ 747 pad = *(uint32_t *)((uintptr_t)ef->vnet_stop - sizeof(pad)); 748 if (pad != LS_PADDING) { 749 uprintf("Kernel module '%s' must be recompiled with " 750 "linker script, invalid padding %#04x (%#04x)\n", 751 ef->lf.pathname, pad, LS_PADDING); 752 return (ENOEXEC); 753 } 754 /* If we only have valid padding, nothing to do. */ 755 if (size == 4) 756 return (0); 757 #endif 758 /* 759 * Allocate space in the primary vnet area. Copy in our 760 * initialization from the data section and then initialize 761 * all per-vnet storage from that. 762 */ 763 ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size); 764 if (ef->vnet_base == 0) { 765 printf("%s: vnet module space is out of space; " 766 "cannot allocate %d for %s\n", 767 __func__, size, ef->lf.pathname); 768 return (ENOSPC); 769 } 770 memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size); 771 vnet_save_init((void *)ef->vnet_base, size); 772 elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop, 773 ef->vnet_base); 774 775 return (0); 776 } 777 #endif 778 #undef LS_PADDING 779 780 /* 781 * Apply the specified protection to the loadable segments of a preloaded linker 782 * file. 783 */ 784 static int 785 preload_protect(elf_file_t ef, vm_prot_t prot) 786 { 787 #if defined(__aarch64__) || defined(__amd64__) 788 Elf_Ehdr *hdr; 789 Elf_Phdr *phdr, *phlimit; 790 vm_prot_t nprot; 791 int error; 792 793 error = 0; 794 hdr = (Elf_Ehdr *)ef->address; 795 phdr = (Elf_Phdr *)(ef->address + hdr->e_phoff); 796 phlimit = phdr + hdr->e_phnum; 797 for (; phdr < phlimit; phdr++) { 798 if (phdr->p_type != PT_LOAD) 799 continue; 800 801 nprot = prot | VM_PROT_READ; 802 if ((phdr->p_flags & PF_W) != 0) 803 nprot |= VM_PROT_WRITE; 804 if ((phdr->p_flags & PF_X) != 0) 805 nprot |= VM_PROT_EXECUTE; 806 error = pmap_change_prot((vm_offset_t)ef->address + 807 phdr->p_vaddr, round_page(phdr->p_memsz), nprot); 808 if (error != 0) 809 break; 810 } 811 return (error); 812 #else 813 return (0); 814 #endif 815 } 816 817 #ifdef __arm__ 818 /* 819 * Locate the ARM exception/unwind table info for DDB and stack(9) use by 820 * searching for the section header that describes it. There may be no unwind 821 * info, for example in a module containing only data. 822 */ 823 static void 824 link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr, int nhdr) 825 { 826 int i; 827 828 for (i = 0; i < nhdr; i++) { 829 if (shdr[i].sh_type == SHT_ARM_EXIDX) { 830 lf->exidx_addr = shdr[i].sh_addr + lf->address; 831 lf->exidx_size = shdr[i].sh_size; 832 break; 833 } 834 } 835 } 836 837 /* 838 * Locate the section headers metadata in a preloaded module, then use it to 839 * locate the exception/unwind table in the module. The size of the metadata 840 * block is stored in a uint32 word immediately before the data itself, and a 841 * comment in preload_search_info() says it is safe to rely on that. 842 */ 843 static void 844 link_elf_locate_exidx_preload(struct linker_file *lf, caddr_t modptr) 845 { 846 uint32_t *modinfo; 847 Elf_Shdr *shdr; 848 uint32_t nhdr; 849 850 modinfo = (uint32_t *)preload_search_info(modptr, 851 MODINFO_METADATA | MODINFOMD_SHDR); 852 if (modinfo != NULL) { 853 shdr = (Elf_Shdr *)modinfo; 854 nhdr = modinfo[-1] / sizeof(Elf_Shdr); 855 link_elf_locate_exidx(lf, shdr, nhdr); 856 } 857 } 858 859 #endif /* __arm__ */ 860 861 static int 862 link_elf_link_preload(linker_class_t cls, const char *filename, 863 linker_file_t *result) 864 { 865 Elf_Addr *ctors_addrp; 866 Elf_Size *ctors_sizep; 867 caddr_t modptr, baseptr, sizeptr, dynptr; 868 char *type; 869 elf_file_t ef; 870 linker_file_t lf; 871 int error; 872 vm_offset_t dp; 873 874 /* Look to see if we have the file preloaded */ 875 modptr = preload_search_by_name(filename); 876 if (modptr == NULL) 877 return (ENOENT); 878 879 type = (char *)preload_search_info(modptr, MODINFO_TYPE); 880 baseptr = preload_search_info(modptr, MODINFO_ADDR); 881 sizeptr = preload_search_info(modptr, MODINFO_SIZE); 882 dynptr = preload_search_info(modptr, 883 MODINFO_METADATA | MODINFOMD_DYNAMIC); 884 if (type == NULL || 885 (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 && 886 strcmp(type, "elf module") != 0)) 887 return (EFTYPE); 888 if (baseptr == NULL || sizeptr == NULL || dynptr == NULL) 889 return (EINVAL); 890 891 lf = linker_make_file(filename, &link_elf_class); 892 if (lf == NULL) 893 return (ENOMEM); 894 895 ef = (elf_file_t) lf; 896 ef->preloaded = 1; 897 ef->modptr = modptr; 898 ef->address = *(caddr_t *)baseptr; 899 #ifdef SPARSE_MAPPING 900 ef->object = NULL; 901 #endif 902 dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr; 903 ef->dynamic = (Elf_Dyn *)dp; 904 lf->address = ef->address; 905 lf->size = *(size_t *)sizeptr; 906 907 ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 908 MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 909 ctors_sizep = (Elf_Size *)preload_search_info(modptr, 910 MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 911 if (ctors_addrp != NULL && ctors_sizep != NULL) { 912 lf->ctors_addr = ef->address + *ctors_addrp; 913 lf->ctors_size = *ctors_sizep; 914 } 915 916 #ifdef __arm__ 917 link_elf_locate_exidx_preload(lf, modptr); 918 #endif 919 920 error = parse_dynamic(ef); 921 if (error == 0) 922 error = parse_dpcpu(ef); 923 #ifdef VIMAGE 924 if (error == 0) 925 error = parse_vnet(ef); 926 #endif 927 if (error == 0) 928 error = preload_protect(ef, VM_PROT_ALL); 929 if (error != 0) { 930 linker_file_unload(lf, LINKER_UNLOAD_FORCE); 931 return (error); 932 } 933 link_elf_reloc_local(lf); 934 *result = lf; 935 return (0); 936 } 937 938 static int 939 link_elf_link_preload_finish(linker_file_t lf) 940 { 941 elf_file_t ef; 942 int error; 943 944 ef = (elf_file_t) lf; 945 error = relocate_file(ef); 946 if (error == 0) 947 error = preload_protect(ef, VM_PROT_NONE); 948 if (error != 0) 949 return (error); 950 (void)link_elf_preload_parse_symbols(ef); 951 952 return (link_elf_link_common_finish(lf)); 953 } 954 955 static int 956 link_elf_load_file(linker_class_t cls, const char* filename, 957 linker_file_t* result) 958 { 959 struct nameidata nd; 960 struct thread* td = curthread; /* XXX */ 961 Elf_Ehdr *hdr; 962 caddr_t firstpage, segbase; 963 int nbytes, i; 964 Elf_Phdr *phdr; 965 Elf_Phdr *phlimit; 966 Elf_Phdr *segs[MAXSEGS]; 967 int nsegs; 968 Elf_Phdr *phdyn; 969 caddr_t mapbase; 970 size_t mapsize; 971 Elf_Addr base_vaddr; 972 Elf_Addr base_vlimit; 973 int error = 0; 974 ssize_t resid; 975 int flags; 976 elf_file_t ef; 977 linker_file_t lf; 978 Elf_Shdr *shdr; 979 int symtabindex; 980 int symstrindex; 981 int shstrindex; 982 int symcnt; 983 int strcnt; 984 char *shstrs; 985 986 shdr = NULL; 987 lf = NULL; 988 shstrs = NULL; 989 990 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename); 991 flags = FREAD; 992 error = vn_open(&nd, &flags, 0, NULL); 993 if (error != 0) 994 return (error); 995 NDFREE_PNBUF(&nd); 996 if (nd.ni_vp->v_type != VREG) { 997 error = ENOEXEC; 998 firstpage = NULL; 999 goto out; 1000 } 1001 #ifdef MAC 1002 error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp); 1003 if (error != 0) { 1004 firstpage = NULL; 1005 goto out; 1006 } 1007 #endif 1008 1009 /* 1010 * Read the elf header from the file. 1011 */ 1012 firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK); 1013 hdr = (Elf_Ehdr *)firstpage; 1014 error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0, 1015 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1016 &resid, td); 1017 nbytes = PAGE_SIZE - resid; 1018 if (error != 0) 1019 goto out; 1020 1021 if (!IS_ELF(*hdr)) { 1022 error = ENOEXEC; 1023 goto out; 1024 } 1025 1026 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || 1027 hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { 1028 link_elf_error(filename, "Unsupported file layout"); 1029 error = ENOEXEC; 1030 goto out; 1031 } 1032 if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 1033 hdr->e_version != EV_CURRENT) { 1034 link_elf_error(filename, "Unsupported file version"); 1035 error = ENOEXEC; 1036 goto out; 1037 } 1038 if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) { 1039 error = ENOSYS; 1040 goto out; 1041 } 1042 if (hdr->e_machine != ELF_TARG_MACH) { 1043 link_elf_error(filename, "Unsupported machine"); 1044 error = ENOEXEC; 1045 goto out; 1046 } 1047 1048 /* 1049 * We rely on the program header being in the first page. 1050 * This is not strictly required by the ABI specification, but 1051 * it seems to always true in practice. And, it simplifies 1052 * things considerably. 1053 */ 1054 if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) && 1055 (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) && 1056 (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes))) 1057 link_elf_error(filename, "Unreadable program headers"); 1058 1059 /* 1060 * Scan the program header entries, and save key information. 1061 * 1062 * We rely on there being exactly two load segments, text and data, 1063 * in that order. 1064 */ 1065 phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff); 1066 phlimit = phdr + hdr->e_phnum; 1067 nsegs = 0; 1068 phdyn = NULL; 1069 while (phdr < phlimit) { 1070 switch (phdr->p_type) { 1071 case PT_LOAD: 1072 if (nsegs == MAXSEGS) { 1073 link_elf_error(filename, "Too many sections"); 1074 error = ENOEXEC; 1075 goto out; 1076 } 1077 /* 1078 * XXX: We just trust they come in right order ?? 1079 */ 1080 segs[nsegs] = phdr; 1081 ++nsegs; 1082 break; 1083 1084 case PT_DYNAMIC: 1085 phdyn = phdr; 1086 break; 1087 1088 case PT_INTERP: 1089 error = ENOSYS; 1090 goto out; 1091 } 1092 1093 ++phdr; 1094 } 1095 if (phdyn == NULL) { 1096 link_elf_error(filename, "Object is not dynamically-linked"); 1097 error = ENOEXEC; 1098 goto out; 1099 } 1100 if (nsegs == 0) { 1101 link_elf_error(filename, "No sections"); 1102 error = ENOEXEC; 1103 goto out; 1104 } 1105 1106 /* 1107 * Allocate the entire address space of the object, to stake 1108 * out our contiguous region, and to establish the base 1109 * address for relocation. 1110 */ 1111 base_vaddr = trunc_page(segs[0]->p_vaddr); 1112 base_vlimit = round_page(segs[nsegs - 1]->p_vaddr + 1113 segs[nsegs - 1]->p_memsz); 1114 mapsize = base_vlimit - base_vaddr; 1115 1116 lf = linker_make_file(filename, &link_elf_class); 1117 if (lf == NULL) { 1118 error = ENOMEM; 1119 goto out; 1120 } 1121 1122 ef = (elf_file_t) lf; 1123 #ifdef SPARSE_MAPPING 1124 ef->object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL, 1125 0, thread0.td_ucred); 1126 if (ef->object == NULL) { 1127 error = ENOMEM; 1128 goto out; 1129 } 1130 #ifdef __amd64__ 1131 mapbase = (caddr_t)KERNBASE; 1132 #else 1133 mapbase = (caddr_t)vm_map_min(kernel_map); 1134 #endif 1135 /* 1136 * Mapping protections are downgraded after relocation processing. 1137 */ 1138 error = vm_map_find(kernel_map, ef->object, 0, 1139 (vm_offset_t *)&mapbase, mapsize, 0, VMFS_OPTIMAL_SPACE, 1140 VM_PROT_ALL, VM_PROT_ALL, 0); 1141 if (error != 0) { 1142 vm_object_deallocate(ef->object); 1143 ef->object = NULL; 1144 goto out; 1145 } 1146 #else 1147 mapbase = malloc_exec(mapsize, M_LINKER, M_WAITOK); 1148 #endif 1149 ef->address = mapbase; 1150 1151 /* 1152 * Read the text and data sections and zero the bss. 1153 */ 1154 for (i = 0; i < nsegs; i++) { 1155 segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 1156 1157 #ifdef SPARSE_MAPPING 1158 /* 1159 * Consecutive segments may have different mapping permissions, 1160 * so be strict and verify that their mappings do not overlap. 1161 */ 1162 if (((vm_offset_t)segbase & PAGE_MASK) != 0) { 1163 error = EINVAL; 1164 goto out; 1165 } 1166 1167 error = vm_map_wire(kernel_map, 1168 (vm_offset_t)segbase, 1169 (vm_offset_t)segbase + round_page(segs[i]->p_memsz), 1170 VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 1171 if (error != KERN_SUCCESS) { 1172 error = ENOMEM; 1173 goto out; 1174 } 1175 #endif 1176 1177 error = vn_rdwr(UIO_READ, nd.ni_vp, 1178 segbase, segs[i]->p_filesz, segs[i]->p_offset, 1179 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1180 &resid, td); 1181 if (error != 0) 1182 goto out; 1183 bzero(segbase + segs[i]->p_filesz, 1184 segs[i]->p_memsz - segs[i]->p_filesz); 1185 } 1186 1187 ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr); 1188 1189 lf->address = ef->address; 1190 lf->size = mapsize; 1191 1192 error = parse_dynamic(ef); 1193 if (error != 0) 1194 goto out; 1195 error = parse_dpcpu(ef); 1196 if (error != 0) 1197 goto out; 1198 #ifdef VIMAGE 1199 error = parse_vnet(ef); 1200 if (error != 0) 1201 goto out; 1202 #endif 1203 link_elf_reloc_local(lf); 1204 1205 VOP_UNLOCK(nd.ni_vp); 1206 error = linker_load_dependencies(lf); 1207 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); 1208 if (error != 0) 1209 goto out; 1210 error = relocate_file(ef); 1211 if (error != 0) 1212 goto out; 1213 1214 #ifdef SPARSE_MAPPING 1215 /* 1216 * Downgrade permissions on text segment mappings now that relocation 1217 * processing is complete. Restrict permissions on read-only segments. 1218 */ 1219 for (i = 0; i < nsegs; i++) { 1220 vm_prot_t prot; 1221 1222 if (segs[i]->p_type != PT_LOAD) 1223 continue; 1224 1225 prot = VM_PROT_READ; 1226 if ((segs[i]->p_flags & PF_W) != 0) 1227 prot |= VM_PROT_WRITE; 1228 if ((segs[i]->p_flags & PF_X) != 0) 1229 prot |= VM_PROT_EXECUTE; 1230 segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 1231 error = vm_map_protect(kernel_map, 1232 (vm_offset_t)segbase, 1233 (vm_offset_t)segbase + round_page(segs[i]->p_memsz), 1234 prot, 0, VM_MAP_PROTECT_SET_PROT); 1235 if (error != KERN_SUCCESS) { 1236 error = ENOMEM; 1237 goto out; 1238 } 1239 } 1240 #endif 1241 1242 /* 1243 * Try and load the symbol table if it's present. (you can 1244 * strip it!) 1245 */ 1246 nbytes = hdr->e_shnum * hdr->e_shentsize; 1247 if (nbytes == 0 || hdr->e_shoff == 0) 1248 goto nosyms; 1249 shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1250 error = vn_rdwr(UIO_READ, nd.ni_vp, 1251 (caddr_t)shdr, nbytes, hdr->e_shoff, 1252 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1253 &resid, td); 1254 if (error != 0) 1255 goto out; 1256 1257 /* Read section string table */ 1258 shstrindex = hdr->e_shstrndx; 1259 if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB && 1260 shdr[shstrindex].sh_size != 0) { 1261 nbytes = shdr[shstrindex].sh_size; 1262 shstrs = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1263 error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs, nbytes, 1264 shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, 1265 td->td_ucred, NOCRED, &resid, td); 1266 if (error) 1267 goto out; 1268 } 1269 1270 symtabindex = -1; 1271 symstrindex = -1; 1272 for (i = 0; i < hdr->e_shnum; i++) { 1273 if (shdr[i].sh_type == SHT_SYMTAB) { 1274 symtabindex = i; 1275 symstrindex = shdr[i].sh_link; 1276 } else if (shstrs != NULL && shdr[i].sh_name != 0 && 1277 strcmp(shstrs + shdr[i].sh_name, ".ctors") == 0) { 1278 /* Record relocated address and size of .ctors. */ 1279 lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr; 1280 lf->ctors_size = shdr[i].sh_size; 1281 } 1282 } 1283 if (symtabindex < 0 || symstrindex < 0) 1284 goto nosyms; 1285 1286 symcnt = shdr[symtabindex].sh_size; 1287 ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK); 1288 strcnt = shdr[symstrindex].sh_size; 1289 ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK); 1290 1291 error = vn_rdwr(UIO_READ, nd.ni_vp, 1292 ef->symbase, symcnt, shdr[symtabindex].sh_offset, 1293 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1294 &resid, td); 1295 if (error != 0) 1296 goto out; 1297 error = vn_rdwr(UIO_READ, nd.ni_vp, 1298 ef->strbase, strcnt, shdr[symstrindex].sh_offset, 1299 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1300 &resid, td); 1301 if (error != 0) 1302 goto out; 1303 1304 ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 1305 ef->ddbsymtab = (const Elf_Sym *)ef->symbase; 1306 ef->ddbstrcnt = strcnt; 1307 ef->ddbstrtab = ef->strbase; 1308 1309 nosyms: 1310 1311 #ifdef __arm__ 1312 link_elf_locate_exidx(lf, shdr, hdr->e_shnum); 1313 #endif 1314 1315 error = link_elf_link_common_finish(lf); 1316 if (error != 0) 1317 goto out; 1318 1319 *result = lf; 1320 1321 out: 1322 VOP_UNLOCK(nd.ni_vp); 1323 vn_close(nd.ni_vp, FREAD, td->td_ucred, td); 1324 if (error != 0 && lf != NULL) 1325 linker_file_unload(lf, LINKER_UNLOAD_FORCE); 1326 free(shdr, M_LINKER); 1327 free(firstpage, M_LINKER); 1328 free(shstrs, M_LINKER); 1329 1330 return (error); 1331 } 1332 1333 Elf_Addr 1334 elf_relocaddr(linker_file_t lf, Elf_Addr x) 1335 { 1336 elf_file_t ef; 1337 1338 KASSERT(lf->ops->cls == (kobj_class_t)&link_elf_class, 1339 ("elf_relocaddr: unexpected linker file %p", lf)); 1340 1341 ef = (elf_file_t)lf; 1342 if (x >= ef->pcpu_start && x < ef->pcpu_stop) 1343 return ((x - ef->pcpu_start) + ef->pcpu_base); 1344 #ifdef VIMAGE 1345 if (x >= ef->vnet_start && x < ef->vnet_stop) 1346 return ((x - ef->vnet_start) + ef->vnet_base); 1347 #endif 1348 return (x); 1349 } 1350 1351 static void 1352 link_elf_unload_file(linker_file_t file) 1353 { 1354 elf_file_t ef = (elf_file_t) file; 1355 1356 if (ef->pcpu_base != 0) { 1357 dpcpu_free((void *)ef->pcpu_base, 1358 ef->pcpu_stop - ef->pcpu_start); 1359 elf_set_delete(&set_pcpu_list, ef->pcpu_start); 1360 } 1361 #ifdef VIMAGE 1362 if (ef->vnet_base != 0) { 1363 vnet_data_free((void *)ef->vnet_base, 1364 ef->vnet_stop - ef->vnet_start); 1365 elf_set_delete(&set_vnet_list, ef->vnet_start); 1366 } 1367 #endif 1368 #ifdef GDB 1369 if (ef->gdb.l_ld != NULL) { 1370 GDB_STATE(RT_DELETE); 1371 free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER); 1372 link_elf_delete_gdb(&ef->gdb); 1373 GDB_STATE(RT_CONSISTENT); 1374 } 1375 #endif 1376 1377 /* Notify MD code that a module is being unloaded. */ 1378 elf_cpu_unload_file(file); 1379 1380 if (ef->preloaded) { 1381 link_elf_unload_preload(file); 1382 return; 1383 } 1384 1385 #ifdef SPARSE_MAPPING 1386 if (ef->object != NULL) { 1387 vm_map_remove(kernel_map, (vm_offset_t) ef->address, 1388 (vm_offset_t) ef->address 1389 + (ef->object->size << PAGE_SHIFT)); 1390 } 1391 #else 1392 free(ef->address, M_LINKER); 1393 #endif 1394 free(ef->symbase, M_LINKER); 1395 free(ef->strbase, M_LINKER); 1396 free(ef->ctftab, M_LINKER); 1397 free(ef->ctfoff, M_LINKER); 1398 free(ef->typoff, M_LINKER); 1399 } 1400 1401 static void 1402 link_elf_unload_preload(linker_file_t file) 1403 { 1404 1405 if (file->pathname != NULL) 1406 preload_delete_name(file->pathname); 1407 } 1408 1409 static const char * 1410 symbol_name(elf_file_t ef, Elf_Size r_info) 1411 { 1412 const Elf_Sym *ref; 1413 1414 if (ELF_R_SYM(r_info)) { 1415 ref = ef->symtab + ELF_R_SYM(r_info); 1416 return (ef->strtab + ref->st_name); 1417 } 1418 return (NULL); 1419 } 1420 1421 static int 1422 symbol_type(elf_file_t ef, Elf_Size r_info) 1423 { 1424 const Elf_Sym *ref; 1425 1426 if (ELF_R_SYM(r_info)) { 1427 ref = ef->symtab + ELF_R_SYM(r_info); 1428 return (ELF_ST_TYPE(ref->st_info)); 1429 } 1430 return (STT_NOTYPE); 1431 } 1432 1433 static int 1434 relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, 1435 bool ifuncs) 1436 { 1437 const Elf_Rel *rel; 1438 const Elf_Rela *rela; 1439 const char *symname; 1440 1441 TSENTER(); 1442 #define APPLY_RELOCS(iter, tbl, tblsize, type) do { \ 1443 for ((iter) = (tbl); (iter) != NULL && \ 1444 (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \ 1445 if ((symbol_type(ef, (iter)->r_info) == \ 1446 STT_GNU_IFUNC || \ 1447 elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \ 1448 continue; \ 1449 if (reloc(&ef->lf, (Elf_Addr)ef->address, \ 1450 (iter), (type), lookup)) { \ 1451 symname = symbol_name(ef, (iter)->r_info); \ 1452 printf("link_elf: symbol %s undefined\n", \ 1453 symname); \ 1454 return (ENOENT); \ 1455 } \ 1456 } \ 1457 } while (0) 1458 1459 APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL); 1460 TSENTER2("ef->rela"); 1461 APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA); 1462 TSEXIT2("ef->rela"); 1463 APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL); 1464 APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA); 1465 1466 #undef APPLY_RELOCS 1467 1468 TSEXIT(); 1469 return (0); 1470 } 1471 1472 static int 1473 relocate_file(elf_file_t ef) 1474 { 1475 int error; 1476 1477 error = relocate_file1(ef, elf_lookup, elf_reloc, false); 1478 if (error == 0) 1479 error = relocate_file1(ef, elf_lookup, elf_reloc, true); 1480 return (error); 1481 } 1482 1483 /* 1484 * SysV hash function for symbol table lookup. It is specified by the 1485 * System V ABI. 1486 */ 1487 static Elf32_Word 1488 elf_hash(const char *name) 1489 { 1490 const unsigned char *p = (const unsigned char *)name; 1491 Elf32_Word h = 0; 1492 1493 while (*p != '\0') { 1494 h = (h << 4) + *p++; 1495 h ^= (h >> 24) & 0xf0; 1496 } 1497 return (h & 0x0fffffff); 1498 } 1499 1500 static int 1501 link_elf_lookup_symbol1(linker_file_t lf, const char *name, c_linker_sym_t *sym, 1502 bool see_local) 1503 { 1504 elf_file_t ef = (elf_file_t) lf; 1505 unsigned long symnum; 1506 const Elf_Sym* symp; 1507 const char *strp; 1508 Elf32_Word hash; 1509 1510 /* If we don't have a hash, bail. */ 1511 if (ef->buckets == NULL || ef->nbuckets == 0) { 1512 printf("link_elf_lookup_symbol: missing symbol hash table\n"); 1513 return (ENOENT); 1514 } 1515 1516 /* First, search hashed global symbols */ 1517 hash = elf_hash(name); 1518 symnum = ef->buckets[hash % ef->nbuckets]; 1519 1520 while (symnum != STN_UNDEF) { 1521 if (symnum >= ef->nchains) { 1522 printf("%s: corrupt symbol table\n", __func__); 1523 return (ENOENT); 1524 } 1525 1526 symp = ef->symtab + symnum; 1527 if (symp->st_name == 0) { 1528 printf("%s: corrupt symbol table\n", __func__); 1529 return (ENOENT); 1530 } 1531 1532 strp = ef->strtab + symp->st_name; 1533 1534 if (strcmp(name, strp) == 0) { 1535 if (symp->st_shndx != SHN_UNDEF || 1536 (symp->st_value != 0 && 1537 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1538 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1539 if (see_local || 1540 ELF_ST_BIND(symp->st_info) != STB_LOCAL) { 1541 *sym = (c_linker_sym_t) symp; 1542 return (0); 1543 } 1544 } 1545 return (ENOENT); 1546 } 1547 1548 symnum = ef->chains[symnum]; 1549 } 1550 1551 return (ENOENT); 1552 } 1553 1554 static int 1555 link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) 1556 { 1557 if (link_elf_leak_locals) 1558 return (link_elf_lookup_debug_symbol(lf, name, sym)); 1559 return (link_elf_lookup_symbol1(lf, name, sym, false)); 1560 } 1561 1562 static int 1563 link_elf_lookup_debug_symbol(linker_file_t lf, const char *name, 1564 c_linker_sym_t *sym) 1565 { 1566 elf_file_t ef = (elf_file_t)lf; 1567 const Elf_Sym* symp; 1568 const char *strp; 1569 int i; 1570 1571 if (link_elf_lookup_symbol1(lf, name, sym, true) == 0) 1572 return (0); 1573 1574 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1575 strp = ef->ddbstrtab + symp->st_name; 1576 if (strcmp(name, strp) == 0) { 1577 if (symp->st_shndx != SHN_UNDEF || 1578 (symp->st_value != 0 && 1579 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1580 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1581 *sym = (c_linker_sym_t) symp; 1582 return (0); 1583 } 1584 return (ENOENT); 1585 } 1586 } 1587 1588 return (ENOENT); 1589 } 1590 1591 static int 1592 link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, 1593 linker_symval_t *symval, bool see_local) 1594 { 1595 elf_file_t ef; 1596 const Elf_Sym *es; 1597 caddr_t val; 1598 1599 ef = (elf_file_t)lf; 1600 es = (const Elf_Sym *)sym; 1601 if (es >= ef->symtab && es < ef->symtab + ef->nchains) { 1602 if (!see_local && ELF_ST_BIND(es->st_info) == STB_LOCAL) 1603 return (ENOENT); 1604 symval->name = ef->strtab + es->st_name; 1605 val = (caddr_t)ef->address + es->st_value; 1606 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1607 val = ((caddr_t (*)(void))val)(); 1608 symval->value = val; 1609 symval->size = es->st_size; 1610 return (0); 1611 } 1612 return (ENOENT); 1613 } 1614 1615 static int 1616 link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1617 linker_symval_t *symval) 1618 { 1619 if (link_elf_leak_locals) 1620 return (link_elf_debug_symbol_values(lf, sym, symval)); 1621 return (link_elf_symbol_values1(lf, sym, symval, false)); 1622 } 1623 1624 static int 1625 link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1626 linker_symval_t *symval) 1627 { 1628 elf_file_t ef = (elf_file_t)lf; 1629 const Elf_Sym *es = (const Elf_Sym *)sym; 1630 caddr_t val; 1631 1632 if (link_elf_symbol_values1(lf, sym, symval, true) == 0) 1633 return (0); 1634 if (ef->symtab == ef->ddbsymtab) 1635 return (ENOENT); 1636 1637 if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { 1638 symval->name = ef->ddbstrtab + es->st_name; 1639 val = (caddr_t)ef->address + es->st_value; 1640 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1641 val = ((caddr_t (*)(void))val)(); 1642 symval->value = val; 1643 symval->size = es->st_size; 1644 return (0); 1645 } 1646 return (ENOENT); 1647 } 1648 1649 static int 1650 link_elf_search_symbol(linker_file_t lf, caddr_t value, 1651 c_linker_sym_t *sym, long *diffp) 1652 { 1653 elf_file_t ef = (elf_file_t)lf; 1654 u_long off = (uintptr_t)(void *)value; 1655 u_long diff = off; 1656 u_long st_value; 1657 const Elf_Sym *es; 1658 const Elf_Sym *best = NULL; 1659 int i; 1660 1661 for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) { 1662 if (es->st_name == 0) 1663 continue; 1664 st_value = es->st_value + (uintptr_t) (void *) ef->address; 1665 if (off >= st_value) { 1666 if (off - st_value < diff) { 1667 diff = off - st_value; 1668 best = es; 1669 if (diff == 0) 1670 break; 1671 } else if (off - st_value == diff) { 1672 best = es; 1673 } 1674 } 1675 } 1676 if (best == NULL) 1677 *diffp = off; 1678 else 1679 *diffp = diff; 1680 *sym = (c_linker_sym_t) best; 1681 1682 return (0); 1683 } 1684 1685 /* 1686 * Look up a linker set on an ELF system. 1687 */ 1688 static int 1689 link_elf_lookup_set(linker_file_t lf, const char *name, 1690 void ***startp, void ***stopp, int *countp) 1691 { 1692 c_linker_sym_t sym; 1693 linker_symval_t symval; 1694 char *setsym; 1695 void **start, **stop; 1696 int len, error = 0, count; 1697 1698 len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */ 1699 setsym = malloc(len, M_LINKER, M_WAITOK); 1700 1701 /* get address of first entry */ 1702 snprintf(setsym, len, "%s%s", "__start_set_", name); 1703 error = link_elf_lookup_symbol(lf, setsym, &sym); 1704 if (error != 0) 1705 goto out; 1706 link_elf_symbol_values(lf, sym, &symval); 1707 if (symval.value == 0) { 1708 error = ESRCH; 1709 goto out; 1710 } 1711 start = (void **)symval.value; 1712 1713 /* get address of last entry */ 1714 snprintf(setsym, len, "%s%s", "__stop_set_", name); 1715 error = link_elf_lookup_symbol(lf, setsym, &sym); 1716 if (error != 0) 1717 goto out; 1718 link_elf_symbol_values(lf, sym, &symval); 1719 if (symval.value == 0) { 1720 error = ESRCH; 1721 goto out; 1722 } 1723 stop = (void **)symval.value; 1724 1725 /* and the number of entries */ 1726 count = stop - start; 1727 1728 /* and copy out */ 1729 if (startp != NULL) 1730 *startp = start; 1731 if (stopp != NULL) 1732 *stopp = stop; 1733 if (countp != NULL) 1734 *countp = count; 1735 1736 out: 1737 free(setsym, M_LINKER); 1738 return (error); 1739 } 1740 1741 static int 1742 link_elf_each_function_name(linker_file_t file, 1743 int (*callback)(const char *, void *), void *opaque) 1744 { 1745 elf_file_t ef = (elf_file_t)file; 1746 const Elf_Sym *symp; 1747 int i, error; 1748 1749 /* Exhaustive search */ 1750 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1751 if (symp->st_value != 0 && 1752 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1753 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1754 error = callback(ef->ddbstrtab + symp->st_name, opaque); 1755 if (error != 0) 1756 return (error); 1757 } 1758 } 1759 return (0); 1760 } 1761 1762 static int 1763 link_elf_each_function_nameval(linker_file_t file, 1764 linker_function_nameval_callback_t callback, void *opaque) 1765 { 1766 linker_symval_t symval; 1767 elf_file_t ef = (elf_file_t)file; 1768 const Elf_Sym *symp; 1769 int i, error; 1770 1771 /* Exhaustive search */ 1772 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1773 if (symp->st_value != 0 && 1774 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1775 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1776 error = link_elf_debug_symbol_values(file, 1777 (c_linker_sym_t) symp, &symval); 1778 if (error == 0) 1779 error = callback(file, i, &symval, opaque); 1780 if (error != 0) 1781 return (error); 1782 } 1783 } 1784 return (0); 1785 } 1786 1787 const Elf_Sym * 1788 elf_get_sym(linker_file_t lf, Elf_Size symidx) 1789 { 1790 elf_file_t ef = (elf_file_t)lf; 1791 1792 if (symidx >= ef->nchains) 1793 return (NULL); 1794 return (ef->symtab + symidx); 1795 } 1796 1797 const char * 1798 elf_get_symname(linker_file_t lf, Elf_Size symidx) 1799 { 1800 elf_file_t ef = (elf_file_t)lf; 1801 const Elf_Sym *sym; 1802 1803 if (symidx >= ef->nchains) 1804 return (NULL); 1805 sym = ef->symtab + symidx; 1806 return (ef->strtab + sym->st_name); 1807 } 1808 1809 /* 1810 * Symbol lookup function that can be used when the symbol index is known (ie 1811 * in relocations). It uses the symbol index instead of doing a fully fledged 1812 * hash table based lookup when such is valid. For example for local symbols. 1813 * This is not only more efficient, it's also more correct. It's not always 1814 * the case that the symbol can be found through the hash table. 1815 */ 1816 static int 1817 elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) 1818 { 1819 elf_file_t ef = (elf_file_t)lf; 1820 const Elf_Sym *sym; 1821 const char *symbol; 1822 Elf_Addr addr, start, base; 1823 1824 /* Don't even try to lookup the symbol if the index is bogus. */ 1825 if (symidx >= ef->nchains) { 1826 *res = 0; 1827 return (EINVAL); 1828 } 1829 1830 sym = ef->symtab + symidx; 1831 1832 /* 1833 * Don't do a full lookup when the symbol is local. It may even 1834 * fail because it may not be found through the hash table. 1835 */ 1836 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) { 1837 /* Force lookup failure when we have an insanity. */ 1838 if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) { 1839 *res = 0; 1840 return (EINVAL); 1841 } 1842 *res = ((Elf_Addr)ef->address + sym->st_value); 1843 return (0); 1844 } 1845 1846 /* 1847 * XXX we can avoid doing a hash table based lookup for global 1848 * symbols as well. This however is not always valid, so we'll 1849 * just do it the hard way for now. Performance tweaks can 1850 * always be added. 1851 */ 1852 1853 symbol = ef->strtab + sym->st_name; 1854 1855 /* Force a lookup failure if the symbol name is bogus. */ 1856 if (*symbol == 0) { 1857 *res = 0; 1858 return (EINVAL); 1859 } 1860 1861 addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps)); 1862 if (addr == 0 && ELF_ST_BIND(sym->st_info) != STB_WEAK) { 1863 *res = 0; 1864 return (EINVAL); 1865 } 1866 1867 if (elf_set_find(&set_pcpu_list, addr, &start, &base)) 1868 addr = addr - start + base; 1869 #ifdef VIMAGE 1870 else if (elf_set_find(&set_vnet_list, addr, &start, &base)) 1871 addr = addr - start + base; 1872 #endif 1873 *res = addr; 1874 return (0); 1875 } 1876 1877 static void 1878 link_elf_reloc_local(linker_file_t lf) 1879 { 1880 const Elf_Rel *rellim; 1881 const Elf_Rel *rel; 1882 const Elf_Rela *relalim; 1883 const Elf_Rela *rela; 1884 elf_file_t ef = (elf_file_t)lf; 1885 1886 /* Perform relocations without addend if there are any: */ 1887 if ((rel = ef->rel) != NULL) { 1888 rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize); 1889 while (rel < rellim) { 1890 elf_reloc_local(lf, (Elf_Addr)ef->address, rel, 1891 ELF_RELOC_REL, elf_lookup); 1892 rel++; 1893 } 1894 } 1895 1896 /* Perform relocations with addend if there are any: */ 1897 if ((rela = ef->rela) != NULL) { 1898 relalim = (const Elf_Rela *) 1899 ((const char *)ef->rela + ef->relasize); 1900 while (rela < relalim) { 1901 elf_reloc_local(lf, (Elf_Addr)ef->address, rela, 1902 ELF_RELOC_RELA, elf_lookup); 1903 rela++; 1904 } 1905 } 1906 } 1907 1908 static long 1909 link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab) 1910 { 1911 elf_file_t ef = (elf_file_t)lf; 1912 1913 *symtab = ef->ddbsymtab; 1914 1915 if (*symtab == NULL) 1916 return (0); 1917 1918 return (ef->ddbsymcnt); 1919 } 1920 1921 static long 1922 link_elf_strtab_get(linker_file_t lf, caddr_t *strtab) 1923 { 1924 elf_file_t ef = (elf_file_t)lf; 1925 1926 *strtab = ef->ddbstrtab; 1927 1928 if (*strtab == NULL) 1929 return (0); 1930 1931 return (ef->ddbstrcnt); 1932 } 1933 1934 #ifdef VIMAGE 1935 static void 1936 link_elf_propagate_vnets(linker_file_t lf) 1937 { 1938 elf_file_t ef = (elf_file_t)lf; 1939 int size; 1940 1941 if (ef->vnet_base != 0) { 1942 size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start; 1943 vnet_data_copy((void *)ef->vnet_base, size); 1944 } 1945 } 1946 #endif 1947 1948 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) || defined(__powerpc__) 1949 /* 1950 * Use this lookup routine when performing relocations early during boot. 1951 * The generic lookup routine depends on kobj, which is not initialized 1952 * at that point. 1953 */ 1954 static int 1955 elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, 1956 Elf_Addr *res) 1957 { 1958 elf_file_t ef; 1959 const Elf_Sym *symp; 1960 caddr_t val; 1961 1962 ef = (elf_file_t)lf; 1963 symp = ef->symtab + symidx; 1964 if (ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC) { 1965 val = (caddr_t)ef->address + symp->st_value; 1966 *res = ((Elf_Addr (*)(void))val)(); 1967 return (0); 1968 } 1969 return (ENOENT); 1970 } 1971 1972 void 1973 link_elf_ireloc(caddr_t kmdp) 1974 { 1975 struct elf_file eff; 1976 elf_file_t ef; 1977 1978 TSENTER(); 1979 ef = &eff; 1980 1981 bzero_early(ef, sizeof(*ef)); 1982 1983 ef->modptr = kmdp; 1984 ef->dynamic = (Elf_Dyn *)&_DYNAMIC; 1985 1986 #ifdef RELOCATABLE_KERNEL 1987 ef->address = (caddr_t) (__startkernel - KERNBASE); 1988 #else 1989 ef->address = 0; 1990 #endif 1991 parse_dynamic(ef); 1992 1993 link_elf_preload_parse_symbols(ef); 1994 relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); 1995 TSEXIT(); 1996 } 1997 1998 #if defined(__aarch64__) || defined(__amd64__) 1999 void 2000 link_elf_late_ireloc(void) 2001 { 2002 elf_file_t ef; 2003 2004 KASSERT(linker_kernel_file != NULL, 2005 ("link_elf_late_ireloc: No kernel linker file found")); 2006 ef = (elf_file_t)linker_kernel_file; 2007 2008 relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true); 2009 } 2010 #endif 2011 #endif 2012