1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1998-2000 Doug Rabson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include "opt_ddb.h" 33 #include "opt_gdb.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #ifdef SPARSE_MAPPING 41 #include <sys/mman.h> 42 #endif 43 #include <sys/mutex.h> 44 #include <sys/mount.h> 45 #include <sys/pcpu.h> 46 #include <sys/proc.h> 47 #include <sys/namei.h> 48 #include <sys/fcntl.h> 49 #include <sys/vnode.h> 50 #include <sys/linker.h> 51 #include <sys/sysctl.h> 52 #include <sys/tslog.h> 53 54 #include <machine/elf.h> 55 56 #include <net/vnet.h> 57 58 #include <security/mac/mac_framework.h> 59 60 #include <vm/vm.h> 61 #include <vm/vm_param.h> 62 #ifdef SPARSE_MAPPING 63 #include <vm/vm_object.h> 64 #include <vm/vm_kern.h> 65 #include <vm/vm_extern.h> 66 #endif 67 #include <vm/pmap.h> 68 #include <vm/vm_map.h> 69 70 #include <sys/link_elf.h> 71 72 #include "linker_if.h" 73 74 #define MAXSEGS 4 75 76 typedef struct elf_file { 77 struct linker_file lf; /* Common fields */ 78 int preloaded; /* Was file pre-loaded */ 79 caddr_t address; /* Relocation address */ 80 #ifdef SPARSE_MAPPING 81 vm_object_t object; /* VM object to hold file pages */ 82 #endif 83 Elf_Dyn *dynamic; /* Symbol table etc. */ 84 Elf_Hashelt nbuckets; /* DT_HASH info */ 85 Elf_Hashelt nchains; 86 const Elf_Hashelt *buckets; 87 const Elf_Hashelt *chains; 88 caddr_t hash; 89 caddr_t strtab; /* DT_STRTAB */ 90 int strsz; /* DT_STRSZ */ 91 const Elf_Sym *symtab; /* DT_SYMTAB */ 92 Elf_Addr *got; /* DT_PLTGOT */ 93 const Elf_Rel *pltrel; /* DT_JMPREL */ 94 int pltrelsize; /* DT_PLTRELSZ */ 95 const Elf_Rela *pltrela; /* DT_JMPREL */ 96 int pltrelasize; /* DT_PLTRELSZ */ 97 const Elf_Rel *rel; /* DT_REL */ 98 int relsize; /* DT_RELSZ */ 99 const Elf_Rela *rela; /* DT_RELA */ 100 int relasize; /* DT_RELASZ */ 101 caddr_t modptr; 102 const Elf_Sym *ddbsymtab; /* The symbol table we are using */ 103 long ddbsymcnt; /* Number of symbols */ 104 caddr_t ddbstrtab; /* String table */ 105 long ddbstrcnt; /* number of bytes in string table */ 106 caddr_t symbase; /* malloc'ed symbold base */ 107 caddr_t strbase; /* malloc'ed string base */ 108 caddr_t ctftab; /* CTF table */ 109 long ctfcnt; /* number of bytes in CTF table */ 110 caddr_t ctfoff; /* CTF offset table */ 111 caddr_t typoff; /* Type offset table */ 112 long typlen; /* Number of type entries. */ 113 Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */ 114 Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */ 115 Elf_Addr pcpu_base; /* Relocated pcpu set address. */ 116 #ifdef VIMAGE 117 Elf_Addr vnet_start; /* Pre-relocation vnet set start. */ 118 Elf_Addr vnet_stop; /* Pre-relocation vnet set stop. */ 119 Elf_Addr vnet_base; /* Relocated vnet set address. */ 120 #endif 121 #ifdef GDB 122 struct link_map gdb; /* hooks for gdb */ 123 #endif 124 } *elf_file_t; 125 126 struct elf_set { 127 Elf_Addr es_start; 128 Elf_Addr es_stop; 129 Elf_Addr es_base; 130 TAILQ_ENTRY(elf_set) es_link; 131 }; 132 133 TAILQ_HEAD(elf_set_head, elf_set); 134 135 #include <kern/kern_ctf.c> 136 137 static int link_elf_link_common_finish(linker_file_t); 138 static int link_elf_link_preload(linker_class_t cls, 139 const char *, linker_file_t *); 140 static int link_elf_link_preload_finish(linker_file_t); 141 static int link_elf_load_file(linker_class_t, const char *, 142 linker_file_t *); 143 static int link_elf_lookup_symbol(linker_file_t, const char *, 144 c_linker_sym_t *); 145 static int link_elf_lookup_debug_symbol(linker_file_t, const char *, 146 c_linker_sym_t *); 147 static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, 148 linker_symval_t *); 149 static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t, 150 linker_symval_t*); 151 static int link_elf_search_symbol(linker_file_t, caddr_t, 152 c_linker_sym_t *, long *); 153 154 static void link_elf_unload_file(linker_file_t); 155 static void link_elf_unload_preload(linker_file_t); 156 static int link_elf_lookup_set(linker_file_t, const char *, 157 void ***, void ***, int *); 158 static int link_elf_each_function_name(linker_file_t, 159 int (*)(const char *, void *), void *); 160 static int link_elf_each_function_nameval(linker_file_t, 161 linker_function_nameval_callback_t, void *); 162 static void link_elf_reloc_local(linker_file_t); 163 static long link_elf_symtab_get(linker_file_t, const Elf_Sym **); 164 static long link_elf_strtab_get(linker_file_t, caddr_t *); 165 static int elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *); 166 167 static kobj_method_t link_elf_methods[] = { 168 KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol), 169 KOBJMETHOD(linker_lookup_debug_symbol, link_elf_lookup_debug_symbol), 170 KOBJMETHOD(linker_symbol_values, link_elf_symbol_values), 171 KOBJMETHOD(linker_debug_symbol_values, link_elf_debug_symbol_values), 172 KOBJMETHOD(linker_search_symbol, link_elf_search_symbol), 173 KOBJMETHOD(linker_unload, link_elf_unload_file), 174 KOBJMETHOD(linker_load_file, link_elf_load_file), 175 KOBJMETHOD(linker_link_preload, link_elf_link_preload), 176 KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish), 177 KOBJMETHOD(linker_lookup_set, link_elf_lookup_set), 178 KOBJMETHOD(linker_each_function_name, link_elf_each_function_name), 179 KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval), 180 KOBJMETHOD(linker_ctf_get, link_elf_ctf_get), 181 KOBJMETHOD(linker_symtab_get, link_elf_symtab_get), 182 KOBJMETHOD(linker_strtab_get, link_elf_strtab_get), 183 KOBJMETHOD_END 184 }; 185 186 static struct linker_class link_elf_class = { 187 #if ELF_TARG_CLASS == ELFCLASS32 188 "elf32", 189 #else 190 "elf64", 191 #endif 192 link_elf_methods, sizeof(struct elf_file) 193 }; 194 195 static bool link_elf_leak_locals = true; 196 SYSCTL_BOOL(_debug, OID_AUTO, link_elf_leak_locals, 197 CTLFLAG_RWTUN, &link_elf_leak_locals, 0, 198 "Allow local symbols to participate in global module symbol resolution"); 199 200 typedef int (*elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, 201 const void *data, int type, elf_lookup_fn lookup); 202 203 static int parse_dynamic(elf_file_t); 204 static int relocate_file(elf_file_t); 205 static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, 206 elf_reloc_fn reloc, bool ifuncs); 207 static int link_elf_preload_parse_symbols(elf_file_t); 208 209 static struct elf_set_head set_pcpu_list; 210 #ifdef VIMAGE 211 static struct elf_set_head set_vnet_list; 212 #endif 213 214 static void 215 elf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base) 216 { 217 struct elf_set *set, *iter; 218 219 set = malloc(sizeof(*set), M_LINKER, M_WAITOK); 220 set->es_start = start; 221 set->es_stop = stop; 222 set->es_base = base; 223 224 TAILQ_FOREACH(iter, list, es_link) { 225 KASSERT((set->es_start < iter->es_start && set->es_stop < iter->es_stop) || 226 (set->es_start > iter->es_start && set->es_stop > iter->es_stop), 227 ("linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx", 228 (uintmax_t)set->es_start, (uintmax_t)set->es_stop, 229 (uintmax_t)iter->es_start, (uintmax_t)iter->es_stop)); 230 231 if (iter->es_start > set->es_start) { 232 TAILQ_INSERT_BEFORE(iter, set, es_link); 233 break; 234 } 235 } 236 237 if (iter == NULL) 238 TAILQ_INSERT_TAIL(list, set, es_link); 239 } 240 241 static int 242 elf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base) 243 { 244 struct elf_set *set; 245 246 TAILQ_FOREACH(set, list, es_link) { 247 if (addr < set->es_start) 248 return (0); 249 if (addr < set->es_stop) { 250 *start = set->es_start; 251 *base = set->es_base; 252 return (1); 253 } 254 } 255 256 return (0); 257 } 258 259 static void 260 elf_set_delete(struct elf_set_head *list, Elf_Addr start) 261 { 262 struct elf_set *set; 263 264 TAILQ_FOREACH(set, list, es_link) { 265 if (start < set->es_start) 266 break; 267 if (start == set->es_start) { 268 TAILQ_REMOVE(list, set, es_link); 269 free(set, M_LINKER); 270 return; 271 } 272 } 273 KASSERT(0, ("deleting unknown linker set (start = 0x%jx)", 274 (uintmax_t)start)); 275 } 276 277 #ifdef GDB 278 static void r_debug_state(struct r_debug *, struct link_map *); 279 280 /* 281 * A list of loaded modules for GDB to use for loading symbols. 282 */ 283 struct r_debug r_debug; 284 285 #define GDB_STATE(s) do { \ 286 r_debug.r_state = s; r_debug_state(NULL, NULL); \ 287 } while (0) 288 289 /* 290 * Function for the debugger to set a breakpoint on to gain control. 291 */ 292 static void 293 r_debug_state(struct r_debug *dummy_one __unused, 294 struct link_map *dummy_two __unused) 295 { 296 } 297 298 static void 299 link_elf_add_gdb(struct link_map *l) 300 { 301 struct link_map *prev; 302 303 l->l_next = NULL; 304 305 if (r_debug.r_map == NULL) { 306 /* Add first. */ 307 l->l_prev = NULL; 308 r_debug.r_map = l; 309 } else { 310 /* Append to list. */ 311 for (prev = r_debug.r_map; 312 prev->l_next != NULL; 313 prev = prev->l_next) 314 ; 315 l->l_prev = prev; 316 prev->l_next = l; 317 } 318 } 319 320 static void 321 link_elf_delete_gdb(struct link_map *l) 322 { 323 if (l->l_prev == NULL) { 324 /* Remove first. */ 325 if ((r_debug.r_map = l->l_next) != NULL) 326 l->l_next->l_prev = NULL; 327 } else { 328 /* Remove any but first. */ 329 if ((l->l_prev->l_next = l->l_next) != NULL) 330 l->l_next->l_prev = l->l_prev; 331 } 332 } 333 #endif /* GDB */ 334 335 /* 336 * The kernel symbol table starts here. 337 */ 338 extern struct _dynamic _DYNAMIC; 339 340 static void 341 link_elf_error(const char *filename, const char *s) 342 { 343 if (filename == NULL) 344 printf("kldload: %s\n", s); 345 else 346 printf("kldload: %s: %s\n", filename, s); 347 } 348 349 static void 350 link_elf_invoke_ctors(caddr_t addr, size_t size) 351 { 352 void (**ctor)(void); 353 size_t i, cnt; 354 355 if (addr == NULL || size == 0) 356 return; 357 cnt = size / sizeof(*ctor); 358 ctor = (void *)addr; 359 for (i = 0; i < cnt; i++) { 360 if (ctor[i] != NULL) 361 (*ctor[i])(); 362 } 363 } 364 365 /* 366 * Actions performed after linking/loading both the preloaded kernel and any 367 * modules; whether preloaded or dynamicly loaded. 368 */ 369 static int 370 link_elf_link_common_finish(linker_file_t lf) 371 { 372 #ifdef GDB 373 elf_file_t ef = (elf_file_t)lf; 374 char *newfilename; 375 #endif 376 int error; 377 378 /* Notify MD code that a module is being loaded. */ 379 error = elf_cpu_load_file(lf); 380 if (error != 0) 381 return (error); 382 383 #ifdef GDB 384 GDB_STATE(RT_ADD); 385 ef->gdb.l_addr = lf->address; 386 newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK); 387 strcpy(newfilename, lf->filename); 388 ef->gdb.l_name = newfilename; 389 ef->gdb.l_ld = ef->dynamic; 390 link_elf_add_gdb(&ef->gdb); 391 GDB_STATE(RT_CONSISTENT); 392 #endif 393 394 /* Invoke .ctors */ 395 link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size); 396 return (0); 397 } 398 399 #ifdef RELOCATABLE_KERNEL 400 /* 401 * __startkernel and __endkernel are symbols set up as relocation canaries. 402 * 403 * They are defined in locore to reference linker script symbols at the 404 * beginning and end of the LOAD area. This has the desired side effect of 405 * giving us variables that have relative relocations pointing at them, so 406 * relocation of the kernel object will cause the variables to be updated 407 * automatically by the runtime linker when we initialize. 408 * 409 * There are two main reasons to relocate the kernel: 410 * 1) If the loader needed to load the kernel at an alternate load address. 411 * 2) If the kernel is switching address spaces on machines like POWER9 412 * under Radix where the high bits of the effective address are used to 413 * differentiate between hypervisor, host, guest, and problem state. 414 */ 415 extern vm_offset_t __startkernel, __endkernel; 416 #endif 417 418 static unsigned long kern_relbase = KERNBASE; 419 420 SYSCTL_ULONG(_kern, OID_AUTO, base_address, CTLFLAG_RD, 421 SYSCTL_NULL_ULONG_PTR, KERNBASE, "Kernel base address"); 422 SYSCTL_ULONG(_kern, OID_AUTO, relbase_address, CTLFLAG_RD, 423 &kern_relbase, 0, "Kernel relocated base address"); 424 425 static void 426 link_elf_init(void* arg) 427 { 428 Elf_Dyn *dp; 429 Elf_Addr *ctors_addrp; 430 Elf_Size *ctors_sizep; 431 caddr_t modptr, baseptr, sizeptr; 432 elf_file_t ef; 433 const char *modname; 434 435 linker_add_class(&link_elf_class); 436 437 dp = (Elf_Dyn *)&_DYNAMIC; 438 modname = NULL; 439 modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel"); 440 if (modptr == NULL) 441 modptr = preload_search_by_type("elf kernel"); 442 modname = (char *)preload_search_info(modptr, MODINFO_NAME); 443 if (modname == NULL) 444 modname = "kernel"; 445 linker_kernel_file = linker_make_file(modname, &link_elf_class); 446 if (linker_kernel_file == NULL) 447 panic("%s: Can't create linker structures for kernel", 448 __func__); 449 450 ef = (elf_file_t) linker_kernel_file; 451 ef->preloaded = 1; 452 #ifdef RELOCATABLE_KERNEL 453 /* Compute relative displacement */ 454 ef->address = (caddr_t) (__startkernel - KERNBASE); 455 #else 456 ef->address = 0; 457 #endif 458 #ifdef SPARSE_MAPPING 459 ef->object = NULL; 460 #endif 461 ef->dynamic = dp; 462 463 if (dp != NULL) 464 parse_dynamic(ef); 465 #ifdef RELOCATABLE_KERNEL 466 linker_kernel_file->address = (caddr_t)__startkernel; 467 linker_kernel_file->size = (intptr_t)(__endkernel - __startkernel); 468 kern_relbase = (unsigned long)__startkernel; 469 #else 470 linker_kernel_file->address += KERNBASE; 471 linker_kernel_file->size = -(intptr_t)linker_kernel_file->address; 472 #endif 473 474 if (modptr != NULL) { 475 ef->modptr = modptr; 476 baseptr = preload_search_info(modptr, MODINFO_ADDR); 477 if (baseptr != NULL) 478 linker_kernel_file->address = *(caddr_t *)baseptr; 479 sizeptr = preload_search_info(modptr, MODINFO_SIZE); 480 if (sizeptr != NULL) 481 linker_kernel_file->size = *(size_t *)sizeptr; 482 ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 483 MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 484 ctors_sizep = (Elf_Size *)preload_search_info(modptr, 485 MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 486 if (ctors_addrp != NULL && ctors_sizep != NULL) { 487 linker_kernel_file->ctors_addr = ef->address + 488 *ctors_addrp; 489 linker_kernel_file->ctors_size = *ctors_sizep; 490 } 491 } 492 (void)link_elf_preload_parse_symbols(ef); 493 494 #ifdef GDB 495 r_debug.r_map = NULL; 496 r_debug.r_brk = r_debug_state; 497 r_debug.r_state = RT_CONSISTENT; 498 #endif 499 500 (void)link_elf_link_common_finish(linker_kernel_file); 501 linker_kernel_file->flags |= LINKER_FILE_LINKED; 502 TAILQ_INIT(&set_pcpu_list); 503 #ifdef VIMAGE 504 TAILQ_INIT(&set_vnet_list); 505 #endif 506 } 507 508 SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, NULL); 509 510 static int 511 link_elf_preload_parse_symbols(elf_file_t ef) 512 { 513 caddr_t pointer; 514 caddr_t ssym, esym, base; 515 caddr_t strtab; 516 int strcnt; 517 Elf_Sym *symtab; 518 int symcnt; 519 520 if (ef->modptr == NULL) 521 return (0); 522 pointer = preload_search_info(ef->modptr, 523 MODINFO_METADATA | MODINFOMD_SSYM); 524 if (pointer == NULL) 525 return (0); 526 ssym = *(caddr_t *)pointer; 527 pointer = preload_search_info(ef->modptr, 528 MODINFO_METADATA | MODINFOMD_ESYM); 529 if (pointer == NULL) 530 return (0); 531 esym = *(caddr_t *)pointer; 532 533 base = ssym; 534 535 symcnt = *(long *)base; 536 base += sizeof(long); 537 symtab = (Elf_Sym *)base; 538 base += roundup(symcnt, sizeof(long)); 539 540 if (base > esym || base < ssym) { 541 printf("Symbols are corrupt!\n"); 542 return (EINVAL); 543 } 544 545 strcnt = *(long *)base; 546 base += sizeof(long); 547 strtab = base; 548 base += roundup(strcnt, sizeof(long)); 549 550 if (base > esym || base < ssym) { 551 printf("Symbols are corrupt!\n"); 552 return (EINVAL); 553 } 554 555 ef->ddbsymtab = symtab; 556 ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 557 ef->ddbstrtab = strtab; 558 ef->ddbstrcnt = strcnt; 559 560 return (0); 561 } 562 563 static int 564 parse_dynamic(elf_file_t ef) 565 { 566 Elf_Dyn *dp; 567 int plttype = DT_REL; 568 569 for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) { 570 switch (dp->d_tag) { 571 case DT_HASH: 572 { 573 /* From src/libexec/rtld-elf/rtld.c */ 574 const Elf_Hashelt *hashtab = (const Elf_Hashelt *) 575 (ef->address + dp->d_un.d_ptr); 576 ef->nbuckets = hashtab[0]; 577 ef->nchains = hashtab[1]; 578 ef->buckets = hashtab + 2; 579 ef->chains = ef->buckets + ef->nbuckets; 580 break; 581 } 582 case DT_STRTAB: 583 ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr); 584 break; 585 case DT_STRSZ: 586 ef->strsz = dp->d_un.d_val; 587 break; 588 case DT_SYMTAB: 589 ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr); 590 break; 591 case DT_SYMENT: 592 if (dp->d_un.d_val != sizeof(Elf_Sym)) 593 return (ENOEXEC); 594 break; 595 case DT_PLTGOT: 596 ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr); 597 break; 598 case DT_REL: 599 ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 600 break; 601 case DT_RELSZ: 602 ef->relsize = dp->d_un.d_val; 603 break; 604 case DT_RELENT: 605 if (dp->d_un.d_val != sizeof(Elf_Rel)) 606 return (ENOEXEC); 607 break; 608 case DT_JMPREL: 609 ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 610 break; 611 case DT_PLTRELSZ: 612 ef->pltrelsize = dp->d_un.d_val; 613 break; 614 case DT_RELA: 615 ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr); 616 break; 617 case DT_RELASZ: 618 ef->relasize = dp->d_un.d_val; 619 break; 620 case DT_RELAENT: 621 if (dp->d_un.d_val != sizeof(Elf_Rela)) 622 return (ENOEXEC); 623 break; 624 case DT_PLTREL: 625 plttype = dp->d_un.d_val; 626 if (plttype != DT_REL && plttype != DT_RELA) 627 return (ENOEXEC); 628 break; 629 #ifdef GDB 630 case DT_DEBUG: 631 dp->d_un.d_ptr = (Elf_Addr)&r_debug; 632 break; 633 #endif 634 } 635 } 636 637 if (plttype == DT_RELA) { 638 ef->pltrela = (const Elf_Rela *)ef->pltrel; 639 ef->pltrel = NULL; 640 ef->pltrelasize = ef->pltrelsize; 641 ef->pltrelsize = 0; 642 } 643 644 ef->ddbsymtab = ef->symtab; 645 ef->ddbsymcnt = ef->nchains; 646 ef->ddbstrtab = ef->strtab; 647 ef->ddbstrcnt = ef->strsz; 648 649 return elf_cpu_parse_dynamic(ef->address, ef->dynamic); 650 } 651 652 #define LS_PADDING 0x90909090 653 static int 654 parse_dpcpu(elf_file_t ef) 655 { 656 int error, size; 657 #if defined(__i386__) 658 uint32_t pad; 659 #endif 660 661 ef->pcpu_start = 0; 662 ef->pcpu_stop = 0; 663 error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start, 664 (void ***)&ef->pcpu_stop, NULL); 665 /* Error just means there is no pcpu set to relocate. */ 666 if (error != 0) 667 return (0); 668 size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start; 669 /* Empty set? */ 670 if (size < 1) 671 return (0); 672 #if defined(__i386__) 673 /* In case we do find __start/stop_set_ symbols double-check. */ 674 if (size < 4) { 675 uprintf("Kernel module '%s' must be recompiled with " 676 "linker script\n", ef->lf.pathname); 677 return (ENOEXEC); 678 } 679 680 /* Padding from linker-script correct? */ 681 pad = *(uint32_t *)((uintptr_t)ef->pcpu_stop - sizeof(pad)); 682 if (pad != LS_PADDING) { 683 uprintf("Kernel module '%s' must be recompiled with " 684 "linker script, invalid padding %#04x (%#04x)\n", 685 ef->lf.pathname, pad, LS_PADDING); 686 return (ENOEXEC); 687 } 688 /* If we only have valid padding, nothing to do. */ 689 if (size == 4) 690 return (0); 691 #endif 692 /* 693 * Allocate space in the primary pcpu area. Copy in our 694 * initialization from the data section and then initialize 695 * all per-cpu storage from that. 696 */ 697 ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size); 698 if (ef->pcpu_base == 0) { 699 printf("%s: pcpu module space is out of space; " 700 "cannot allocate %d for %s\n", 701 __func__, size, ef->lf.pathname); 702 return (ENOSPC); 703 } 704 memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size); 705 dpcpu_copy((void *)ef->pcpu_base, size); 706 elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop, 707 ef->pcpu_base); 708 709 return (0); 710 } 711 712 #ifdef VIMAGE 713 static int 714 parse_vnet(elf_file_t ef) 715 { 716 int error, size; 717 #if defined(__i386__) 718 uint32_t pad; 719 #endif 720 721 ef->vnet_start = 0; 722 ef->vnet_stop = 0; 723 error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start, 724 (void ***)&ef->vnet_stop, NULL); 725 /* Error just means there is no vnet data set to relocate. */ 726 if (error != 0) 727 return (0); 728 size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start; 729 /* Empty set? */ 730 if (size < 1) 731 return (0); 732 #if defined(__i386__) 733 /* In case we do find __start/stop_set_ symbols double-check. */ 734 if (size < 4) { 735 uprintf("Kernel module '%s' must be recompiled with " 736 "linker script\n", ef->lf.pathname); 737 return (ENOEXEC); 738 } 739 740 /* Padding from linker-script correct? */ 741 pad = *(uint32_t *)((uintptr_t)ef->vnet_stop - sizeof(pad)); 742 if (pad != LS_PADDING) { 743 uprintf("Kernel module '%s' must be recompiled with " 744 "linker script, invalid padding %#04x (%#04x)\n", 745 ef->lf.pathname, pad, LS_PADDING); 746 return (ENOEXEC); 747 } 748 /* If we only have valid padding, nothing to do. */ 749 if (size == 4) 750 return (0); 751 #endif 752 /* 753 * Allocate space in the primary vnet area. Copy in our 754 * initialization from the data section and then initialize 755 * all per-vnet storage from that. 756 */ 757 ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size); 758 if (ef->vnet_base == 0) { 759 printf("%s: vnet module space is out of space; " 760 "cannot allocate %d for %s\n", 761 __func__, size, ef->lf.pathname); 762 return (ENOSPC); 763 } 764 memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size); 765 vnet_data_copy((void *)ef->vnet_base, size); 766 elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop, 767 ef->vnet_base); 768 769 return (0); 770 } 771 #endif 772 #undef LS_PADDING 773 774 /* 775 * Apply the specified protection to the loadable segments of a preloaded linker 776 * file. 777 */ 778 static int 779 preload_protect(elf_file_t ef, vm_prot_t prot) 780 { 781 #if defined(__aarch64__) || defined(__amd64__) 782 Elf_Ehdr *hdr; 783 Elf_Phdr *phdr, *phlimit; 784 vm_prot_t nprot; 785 int error; 786 787 error = 0; 788 hdr = (Elf_Ehdr *)ef->address; 789 phdr = (Elf_Phdr *)(ef->address + hdr->e_phoff); 790 phlimit = phdr + hdr->e_phnum; 791 for (; phdr < phlimit; phdr++) { 792 if (phdr->p_type != PT_LOAD) 793 continue; 794 795 nprot = prot | VM_PROT_READ; 796 if ((phdr->p_flags & PF_W) != 0) 797 nprot |= VM_PROT_WRITE; 798 if ((phdr->p_flags & PF_X) != 0) 799 nprot |= VM_PROT_EXECUTE; 800 error = pmap_change_prot((vm_offset_t)ef->address + 801 phdr->p_vaddr, round_page(phdr->p_memsz), nprot); 802 if (error != 0) 803 break; 804 } 805 return (error); 806 #else 807 return (0); 808 #endif 809 } 810 811 #ifdef __arm__ 812 /* 813 * Locate the ARM exception/unwind table info for DDB and stack(9) use by 814 * searching for the section header that describes it. There may be no unwind 815 * info, for example in a module containing only data. 816 */ 817 static void 818 link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr, int nhdr) 819 { 820 int i; 821 822 for (i = 0; i < nhdr; i++) { 823 if (shdr[i].sh_type == SHT_ARM_EXIDX) { 824 lf->exidx_addr = shdr[i].sh_addr + lf->address; 825 lf->exidx_size = shdr[i].sh_size; 826 break; 827 } 828 } 829 } 830 831 /* 832 * Locate the section headers metadata in a preloaded module, then use it to 833 * locate the exception/unwind table in the module. The size of the metadata 834 * block is stored in a uint32 word immediately before the data itself, and a 835 * comment in preload_search_info() says it is safe to rely on that. 836 */ 837 static void 838 link_elf_locate_exidx_preload(struct linker_file *lf, caddr_t modptr) 839 { 840 uint32_t *modinfo; 841 Elf_Shdr *shdr; 842 uint32_t nhdr; 843 844 modinfo = (uint32_t *)preload_search_info(modptr, 845 MODINFO_METADATA | MODINFOMD_SHDR); 846 if (modinfo != NULL) { 847 shdr = (Elf_Shdr *)modinfo; 848 nhdr = modinfo[-1] / sizeof(Elf_Shdr); 849 link_elf_locate_exidx(lf, shdr, nhdr); 850 } 851 } 852 853 #endif /* __arm__ */ 854 855 static int 856 link_elf_link_preload(linker_class_t cls, const char *filename, 857 linker_file_t *result) 858 { 859 Elf_Addr *ctors_addrp; 860 Elf_Size *ctors_sizep; 861 caddr_t modptr, baseptr, sizeptr, dynptr; 862 char *type; 863 elf_file_t ef; 864 linker_file_t lf; 865 int error; 866 vm_offset_t dp; 867 868 /* Look to see if we have the file preloaded */ 869 modptr = preload_search_by_name(filename); 870 if (modptr == NULL) 871 return (ENOENT); 872 873 type = (char *)preload_search_info(modptr, MODINFO_TYPE); 874 baseptr = preload_search_info(modptr, MODINFO_ADDR); 875 sizeptr = preload_search_info(modptr, MODINFO_SIZE); 876 dynptr = preload_search_info(modptr, 877 MODINFO_METADATA | MODINFOMD_DYNAMIC); 878 if (type == NULL || 879 (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 && 880 strcmp(type, "elf module") != 0)) 881 return (EFTYPE); 882 if (baseptr == NULL || sizeptr == NULL || dynptr == NULL) 883 return (EINVAL); 884 885 lf = linker_make_file(filename, &link_elf_class); 886 if (lf == NULL) 887 return (ENOMEM); 888 889 ef = (elf_file_t) lf; 890 ef->preloaded = 1; 891 ef->modptr = modptr; 892 ef->address = *(caddr_t *)baseptr; 893 #ifdef SPARSE_MAPPING 894 ef->object = NULL; 895 #endif 896 dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr; 897 ef->dynamic = (Elf_Dyn *)dp; 898 lf->address = ef->address; 899 lf->size = *(size_t *)sizeptr; 900 901 ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 902 MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 903 ctors_sizep = (Elf_Size *)preload_search_info(modptr, 904 MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 905 if (ctors_addrp != NULL && ctors_sizep != NULL) { 906 lf->ctors_addr = ef->address + *ctors_addrp; 907 lf->ctors_size = *ctors_sizep; 908 } 909 910 #ifdef __arm__ 911 link_elf_locate_exidx_preload(lf, modptr); 912 #endif 913 914 error = parse_dynamic(ef); 915 if (error == 0) 916 error = parse_dpcpu(ef); 917 #ifdef VIMAGE 918 if (error == 0) 919 error = parse_vnet(ef); 920 #endif 921 if (error == 0) 922 error = preload_protect(ef, VM_PROT_ALL); 923 if (error != 0) { 924 linker_file_unload(lf, LINKER_UNLOAD_FORCE); 925 return (error); 926 } 927 link_elf_reloc_local(lf); 928 *result = lf; 929 return (0); 930 } 931 932 static int 933 link_elf_link_preload_finish(linker_file_t lf) 934 { 935 elf_file_t ef; 936 int error; 937 938 ef = (elf_file_t) lf; 939 error = relocate_file(ef); 940 if (error == 0) 941 error = preload_protect(ef, VM_PROT_NONE); 942 if (error != 0) 943 return (error); 944 (void)link_elf_preload_parse_symbols(ef); 945 946 return (link_elf_link_common_finish(lf)); 947 } 948 949 static int 950 link_elf_load_file(linker_class_t cls, const char* filename, 951 linker_file_t* result) 952 { 953 struct nameidata nd; 954 struct thread* td = curthread; /* XXX */ 955 Elf_Ehdr *hdr; 956 caddr_t firstpage, segbase; 957 int nbytes, i; 958 Elf_Phdr *phdr; 959 Elf_Phdr *phlimit; 960 Elf_Phdr *segs[MAXSEGS]; 961 int nsegs; 962 Elf_Phdr *phdyn; 963 caddr_t mapbase; 964 size_t mapsize; 965 Elf_Addr base_vaddr; 966 Elf_Addr base_vlimit; 967 int error = 0; 968 ssize_t resid; 969 int flags; 970 elf_file_t ef; 971 linker_file_t lf; 972 Elf_Shdr *shdr; 973 int symtabindex; 974 int symstrindex; 975 int shstrindex; 976 int symcnt; 977 int strcnt; 978 char *shstrs; 979 980 shdr = NULL; 981 lf = NULL; 982 shstrs = NULL; 983 984 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename); 985 flags = FREAD; 986 error = vn_open(&nd, &flags, 0, NULL); 987 if (error != 0) 988 return (error); 989 NDFREE_PNBUF(&nd); 990 if (nd.ni_vp->v_type != VREG) { 991 error = ENOEXEC; 992 firstpage = NULL; 993 goto out; 994 } 995 #ifdef MAC 996 error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp); 997 if (error != 0) { 998 firstpage = NULL; 999 goto out; 1000 } 1001 #endif 1002 1003 /* 1004 * Read the elf header from the file. 1005 */ 1006 firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK); 1007 hdr = (Elf_Ehdr *)firstpage; 1008 error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0, 1009 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1010 &resid, td); 1011 nbytes = PAGE_SIZE - resid; 1012 if (error != 0) 1013 goto out; 1014 1015 if (!IS_ELF(*hdr)) { 1016 error = ENOEXEC; 1017 goto out; 1018 } 1019 1020 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || 1021 hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { 1022 link_elf_error(filename, "Unsupported file layout"); 1023 error = ENOEXEC; 1024 goto out; 1025 } 1026 if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 1027 hdr->e_version != EV_CURRENT) { 1028 link_elf_error(filename, "Unsupported file version"); 1029 error = ENOEXEC; 1030 goto out; 1031 } 1032 if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) { 1033 error = ENOSYS; 1034 goto out; 1035 } 1036 if (hdr->e_machine != ELF_TARG_MACH) { 1037 link_elf_error(filename, "Unsupported machine"); 1038 error = ENOEXEC; 1039 goto out; 1040 } 1041 1042 /* 1043 * We rely on the program header being in the first page. 1044 * This is not strictly required by the ABI specification, but 1045 * it seems to always true in practice. And, it simplifies 1046 * things considerably. 1047 */ 1048 if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) && 1049 (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) && 1050 (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes))) 1051 link_elf_error(filename, "Unreadable program headers"); 1052 1053 /* 1054 * Scan the program header entries, and save key information. 1055 * 1056 * We rely on there being exactly two load segments, text and data, 1057 * in that order. 1058 */ 1059 phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff); 1060 phlimit = phdr + hdr->e_phnum; 1061 nsegs = 0; 1062 phdyn = NULL; 1063 while (phdr < phlimit) { 1064 switch (phdr->p_type) { 1065 case PT_LOAD: 1066 if (nsegs == MAXSEGS) { 1067 link_elf_error(filename, "Too many sections"); 1068 error = ENOEXEC; 1069 goto out; 1070 } 1071 /* 1072 * XXX: We just trust they come in right order ?? 1073 */ 1074 segs[nsegs] = phdr; 1075 ++nsegs; 1076 break; 1077 1078 case PT_DYNAMIC: 1079 phdyn = phdr; 1080 break; 1081 1082 case PT_INTERP: 1083 error = ENOSYS; 1084 goto out; 1085 } 1086 1087 ++phdr; 1088 } 1089 if (phdyn == NULL) { 1090 link_elf_error(filename, "Object is not dynamically-linked"); 1091 error = ENOEXEC; 1092 goto out; 1093 } 1094 if (nsegs == 0) { 1095 link_elf_error(filename, "No sections"); 1096 error = ENOEXEC; 1097 goto out; 1098 } 1099 1100 /* 1101 * Allocate the entire address space of the object, to stake 1102 * out our contiguous region, and to establish the base 1103 * address for relocation. 1104 */ 1105 base_vaddr = trunc_page(segs[0]->p_vaddr); 1106 base_vlimit = round_page(segs[nsegs - 1]->p_vaddr + 1107 segs[nsegs - 1]->p_memsz); 1108 mapsize = base_vlimit - base_vaddr; 1109 1110 lf = linker_make_file(filename, &link_elf_class); 1111 if (lf == NULL) { 1112 error = ENOMEM; 1113 goto out; 1114 } 1115 1116 ef = (elf_file_t) lf; 1117 #ifdef SPARSE_MAPPING 1118 ef->object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL, 1119 0, thread0.td_ucred); 1120 if (ef->object == NULL) { 1121 error = ENOMEM; 1122 goto out; 1123 } 1124 #ifdef __amd64__ 1125 mapbase = (caddr_t)KERNBASE; 1126 #else 1127 mapbase = (caddr_t)vm_map_min(kernel_map); 1128 #endif 1129 /* 1130 * Mapping protections are downgraded after relocation processing. 1131 */ 1132 error = vm_map_find(kernel_map, ef->object, 0, 1133 (vm_offset_t *)&mapbase, mapsize, 0, VMFS_OPTIMAL_SPACE, 1134 VM_PROT_ALL, VM_PROT_ALL, 0); 1135 if (error != 0) { 1136 vm_object_deallocate(ef->object); 1137 ef->object = NULL; 1138 goto out; 1139 } 1140 #else 1141 mapbase = malloc_exec(mapsize, M_LINKER, M_WAITOK); 1142 #endif 1143 ef->address = mapbase; 1144 1145 /* 1146 * Read the text and data sections and zero the bss. 1147 */ 1148 for (i = 0; i < nsegs; i++) { 1149 segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 1150 1151 #ifdef SPARSE_MAPPING 1152 /* 1153 * Consecutive segments may have different mapping permissions, 1154 * so be strict and verify that their mappings do not overlap. 1155 */ 1156 if (((vm_offset_t)segbase & PAGE_MASK) != 0) { 1157 error = EINVAL; 1158 goto out; 1159 } 1160 1161 error = vm_map_wire(kernel_map, 1162 (vm_offset_t)segbase, 1163 (vm_offset_t)segbase + round_page(segs[i]->p_memsz), 1164 VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 1165 if (error != KERN_SUCCESS) { 1166 error = ENOMEM; 1167 goto out; 1168 } 1169 #endif 1170 1171 error = vn_rdwr(UIO_READ, nd.ni_vp, 1172 segbase, segs[i]->p_filesz, segs[i]->p_offset, 1173 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1174 &resid, td); 1175 if (error != 0) 1176 goto out; 1177 bzero(segbase + segs[i]->p_filesz, 1178 segs[i]->p_memsz - segs[i]->p_filesz); 1179 } 1180 1181 ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr); 1182 1183 lf->address = ef->address; 1184 lf->size = mapsize; 1185 1186 error = parse_dynamic(ef); 1187 if (error != 0) 1188 goto out; 1189 error = parse_dpcpu(ef); 1190 if (error != 0) 1191 goto out; 1192 #ifdef VIMAGE 1193 error = parse_vnet(ef); 1194 if (error != 0) 1195 goto out; 1196 #endif 1197 link_elf_reloc_local(lf); 1198 1199 VOP_UNLOCK(nd.ni_vp); 1200 error = linker_load_dependencies(lf); 1201 vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); 1202 if (error != 0) 1203 goto out; 1204 error = relocate_file(ef); 1205 if (error != 0) 1206 goto out; 1207 1208 #ifdef SPARSE_MAPPING 1209 /* 1210 * Downgrade permissions on text segment mappings now that relocation 1211 * processing is complete. Restrict permissions on read-only segments. 1212 */ 1213 for (i = 0; i < nsegs; i++) { 1214 vm_prot_t prot; 1215 1216 if (segs[i]->p_type != PT_LOAD) 1217 continue; 1218 1219 prot = VM_PROT_READ; 1220 if ((segs[i]->p_flags & PF_W) != 0) 1221 prot |= VM_PROT_WRITE; 1222 if ((segs[i]->p_flags & PF_X) != 0) 1223 prot |= VM_PROT_EXECUTE; 1224 segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 1225 error = vm_map_protect(kernel_map, 1226 (vm_offset_t)segbase, 1227 (vm_offset_t)segbase + round_page(segs[i]->p_memsz), 1228 prot, 0, VM_MAP_PROTECT_SET_PROT); 1229 if (error != KERN_SUCCESS) { 1230 error = ENOMEM; 1231 goto out; 1232 } 1233 } 1234 #endif 1235 1236 /* 1237 * Try and load the symbol table if it's present. (you can 1238 * strip it!) 1239 */ 1240 nbytes = hdr->e_shnum * hdr->e_shentsize; 1241 if (nbytes == 0 || hdr->e_shoff == 0) 1242 goto nosyms; 1243 shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1244 error = vn_rdwr(UIO_READ, nd.ni_vp, 1245 (caddr_t)shdr, nbytes, hdr->e_shoff, 1246 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1247 &resid, td); 1248 if (error != 0) 1249 goto out; 1250 1251 /* Read section string table */ 1252 shstrindex = hdr->e_shstrndx; 1253 if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB && 1254 shdr[shstrindex].sh_size != 0) { 1255 nbytes = shdr[shstrindex].sh_size; 1256 shstrs = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1257 error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs, nbytes, 1258 shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, 1259 td->td_ucred, NOCRED, &resid, td); 1260 if (error) 1261 goto out; 1262 } 1263 1264 symtabindex = -1; 1265 symstrindex = -1; 1266 for (i = 0; i < hdr->e_shnum; i++) { 1267 if (shdr[i].sh_type == SHT_SYMTAB) { 1268 symtabindex = i; 1269 symstrindex = shdr[i].sh_link; 1270 } else if (shstrs != NULL && shdr[i].sh_name != 0 && 1271 strcmp(shstrs + shdr[i].sh_name, ".ctors") == 0) { 1272 /* Record relocated address and size of .ctors. */ 1273 lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr; 1274 lf->ctors_size = shdr[i].sh_size; 1275 } 1276 } 1277 if (symtabindex < 0 || symstrindex < 0) 1278 goto nosyms; 1279 1280 symcnt = shdr[symtabindex].sh_size; 1281 ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK); 1282 strcnt = shdr[symstrindex].sh_size; 1283 ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK); 1284 1285 error = vn_rdwr(UIO_READ, nd.ni_vp, 1286 ef->symbase, symcnt, shdr[symtabindex].sh_offset, 1287 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1288 &resid, td); 1289 if (error != 0) 1290 goto out; 1291 error = vn_rdwr(UIO_READ, nd.ni_vp, 1292 ef->strbase, strcnt, shdr[symstrindex].sh_offset, 1293 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1294 &resid, td); 1295 if (error != 0) 1296 goto out; 1297 1298 ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 1299 ef->ddbsymtab = (const Elf_Sym *)ef->symbase; 1300 ef->ddbstrcnt = strcnt; 1301 ef->ddbstrtab = ef->strbase; 1302 1303 nosyms: 1304 1305 #ifdef __arm__ 1306 link_elf_locate_exidx(lf, shdr, hdr->e_shnum); 1307 #endif 1308 1309 error = link_elf_link_common_finish(lf); 1310 if (error != 0) 1311 goto out; 1312 1313 *result = lf; 1314 1315 out: 1316 VOP_UNLOCK(nd.ni_vp); 1317 vn_close(nd.ni_vp, FREAD, td->td_ucred, td); 1318 if (error != 0 && lf != NULL) 1319 linker_file_unload(lf, LINKER_UNLOAD_FORCE); 1320 free(shdr, M_LINKER); 1321 free(firstpage, M_LINKER); 1322 free(shstrs, M_LINKER); 1323 1324 return (error); 1325 } 1326 1327 Elf_Addr 1328 elf_relocaddr(linker_file_t lf, Elf_Addr x) 1329 { 1330 elf_file_t ef; 1331 1332 KASSERT(lf->ops->cls == (kobj_class_t)&link_elf_class, 1333 ("elf_relocaddr: unexpected linker file %p", lf)); 1334 1335 ef = (elf_file_t)lf; 1336 if (x >= ef->pcpu_start && x < ef->pcpu_stop) 1337 return ((x - ef->pcpu_start) + ef->pcpu_base); 1338 #ifdef VIMAGE 1339 if (x >= ef->vnet_start && x < ef->vnet_stop) 1340 return ((x - ef->vnet_start) + ef->vnet_base); 1341 #endif 1342 return (x); 1343 } 1344 1345 static void 1346 link_elf_unload_file(linker_file_t file) 1347 { 1348 elf_file_t ef = (elf_file_t) file; 1349 1350 if (ef->pcpu_base != 0) { 1351 dpcpu_free((void *)ef->pcpu_base, 1352 ef->pcpu_stop - ef->pcpu_start); 1353 elf_set_delete(&set_pcpu_list, ef->pcpu_start); 1354 } 1355 #ifdef VIMAGE 1356 if (ef->vnet_base != 0) { 1357 vnet_data_free((void *)ef->vnet_base, 1358 ef->vnet_stop - ef->vnet_start); 1359 elf_set_delete(&set_vnet_list, ef->vnet_start); 1360 } 1361 #endif 1362 #ifdef GDB 1363 if (ef->gdb.l_ld != NULL) { 1364 GDB_STATE(RT_DELETE); 1365 free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER); 1366 link_elf_delete_gdb(&ef->gdb); 1367 GDB_STATE(RT_CONSISTENT); 1368 } 1369 #endif 1370 1371 /* Notify MD code that a module is being unloaded. */ 1372 elf_cpu_unload_file(file); 1373 1374 if (ef->preloaded) { 1375 link_elf_unload_preload(file); 1376 return; 1377 } 1378 1379 #ifdef SPARSE_MAPPING 1380 if (ef->object != NULL) { 1381 vm_map_remove(kernel_map, (vm_offset_t) ef->address, 1382 (vm_offset_t) ef->address 1383 + (ef->object->size << PAGE_SHIFT)); 1384 } 1385 #else 1386 free(ef->address, M_LINKER); 1387 #endif 1388 free(ef->symbase, M_LINKER); 1389 free(ef->strbase, M_LINKER); 1390 free(ef->ctftab, M_LINKER); 1391 free(ef->ctfoff, M_LINKER); 1392 free(ef->typoff, M_LINKER); 1393 } 1394 1395 static void 1396 link_elf_unload_preload(linker_file_t file) 1397 { 1398 1399 if (file->pathname != NULL) 1400 preload_delete_name(file->pathname); 1401 } 1402 1403 static const char * 1404 symbol_name(elf_file_t ef, Elf_Size r_info) 1405 { 1406 const Elf_Sym *ref; 1407 1408 if (ELF_R_SYM(r_info)) { 1409 ref = ef->symtab + ELF_R_SYM(r_info); 1410 return (ef->strtab + ref->st_name); 1411 } 1412 return (NULL); 1413 } 1414 1415 static int 1416 symbol_type(elf_file_t ef, Elf_Size r_info) 1417 { 1418 const Elf_Sym *ref; 1419 1420 if (ELF_R_SYM(r_info)) { 1421 ref = ef->symtab + ELF_R_SYM(r_info); 1422 return (ELF_ST_TYPE(ref->st_info)); 1423 } 1424 return (STT_NOTYPE); 1425 } 1426 1427 static int 1428 relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, 1429 bool ifuncs) 1430 { 1431 const Elf_Rel *rel; 1432 const Elf_Rela *rela; 1433 const char *symname; 1434 1435 TSENTER(); 1436 #define APPLY_RELOCS(iter, tbl, tblsize, type) do { \ 1437 for ((iter) = (tbl); (iter) != NULL && \ 1438 (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \ 1439 if ((symbol_type(ef, (iter)->r_info) == \ 1440 STT_GNU_IFUNC || \ 1441 elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \ 1442 continue; \ 1443 if (reloc(&ef->lf, (Elf_Addr)ef->address, \ 1444 (iter), (type), lookup)) { \ 1445 symname = symbol_name(ef, (iter)->r_info); \ 1446 printf("link_elf: symbol %s undefined\n", \ 1447 symname); \ 1448 return (ENOENT); \ 1449 } \ 1450 } \ 1451 } while (0) 1452 1453 APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL); 1454 TSENTER2("ef->rela"); 1455 APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA); 1456 TSEXIT2("ef->rela"); 1457 APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL); 1458 APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA); 1459 1460 #undef APPLY_RELOCS 1461 1462 TSEXIT(); 1463 return (0); 1464 } 1465 1466 static int 1467 relocate_file(elf_file_t ef) 1468 { 1469 int error; 1470 1471 error = relocate_file1(ef, elf_lookup, elf_reloc, false); 1472 if (error == 0) 1473 error = relocate_file1(ef, elf_lookup, elf_reloc, true); 1474 return (error); 1475 } 1476 1477 /* 1478 * SysV hash function for symbol table lookup. It is specified by the 1479 * System V ABI. 1480 */ 1481 static Elf32_Word 1482 elf_hash(const char *name) 1483 { 1484 const unsigned char *p = (const unsigned char *)name; 1485 Elf32_Word h = 0; 1486 1487 while (*p != '\0') { 1488 h = (h << 4) + *p++; 1489 h ^= (h >> 24) & 0xf0; 1490 } 1491 return (h & 0x0fffffff); 1492 } 1493 1494 static int 1495 link_elf_lookup_symbol1(linker_file_t lf, const char *name, c_linker_sym_t *sym, 1496 bool see_local) 1497 { 1498 elf_file_t ef = (elf_file_t) lf; 1499 unsigned long symnum; 1500 const Elf_Sym* symp; 1501 const char *strp; 1502 Elf32_Word hash; 1503 1504 /* If we don't have a hash, bail. */ 1505 if (ef->buckets == NULL || ef->nbuckets == 0) { 1506 printf("link_elf_lookup_symbol: missing symbol hash table\n"); 1507 return (ENOENT); 1508 } 1509 1510 /* First, search hashed global symbols */ 1511 hash = elf_hash(name); 1512 symnum = ef->buckets[hash % ef->nbuckets]; 1513 1514 while (symnum != STN_UNDEF) { 1515 if (symnum >= ef->nchains) { 1516 printf("%s: corrupt symbol table\n", __func__); 1517 return (ENOENT); 1518 } 1519 1520 symp = ef->symtab + symnum; 1521 if (symp->st_name == 0) { 1522 printf("%s: corrupt symbol table\n", __func__); 1523 return (ENOENT); 1524 } 1525 1526 strp = ef->strtab + symp->st_name; 1527 1528 if (strcmp(name, strp) == 0) { 1529 if (symp->st_shndx != SHN_UNDEF || 1530 (symp->st_value != 0 && 1531 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1532 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1533 if (see_local || 1534 ELF_ST_BIND(symp->st_info) != STB_LOCAL) { 1535 *sym = (c_linker_sym_t) symp; 1536 return (0); 1537 } 1538 } 1539 return (ENOENT); 1540 } 1541 1542 symnum = ef->chains[symnum]; 1543 } 1544 1545 return (ENOENT); 1546 } 1547 1548 static int 1549 link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) 1550 { 1551 if (link_elf_leak_locals) 1552 return (link_elf_lookup_debug_symbol(lf, name, sym)); 1553 return (link_elf_lookup_symbol1(lf, name, sym, false)); 1554 } 1555 1556 static int 1557 link_elf_lookup_debug_symbol(linker_file_t lf, const char *name, 1558 c_linker_sym_t *sym) 1559 { 1560 elf_file_t ef = (elf_file_t)lf; 1561 const Elf_Sym* symp; 1562 const char *strp; 1563 int i; 1564 1565 if (link_elf_lookup_symbol1(lf, name, sym, true) == 0) 1566 return (0); 1567 1568 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1569 strp = ef->ddbstrtab + symp->st_name; 1570 if (strcmp(name, strp) == 0) { 1571 if (symp->st_shndx != SHN_UNDEF || 1572 (symp->st_value != 0 && 1573 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1574 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1575 *sym = (c_linker_sym_t) symp; 1576 return (0); 1577 } 1578 return (ENOENT); 1579 } 1580 } 1581 1582 return (ENOENT); 1583 } 1584 1585 static int 1586 link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, 1587 linker_symval_t *symval, bool see_local) 1588 { 1589 elf_file_t ef; 1590 const Elf_Sym *es; 1591 caddr_t val; 1592 1593 ef = (elf_file_t)lf; 1594 es = (const Elf_Sym *)sym; 1595 if (es >= ef->symtab && es < ef->symtab + ef->nchains) { 1596 if (!see_local && ELF_ST_BIND(es->st_info) == STB_LOCAL) 1597 return (ENOENT); 1598 symval->name = ef->strtab + es->st_name; 1599 val = (caddr_t)ef->address + es->st_value; 1600 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1601 val = ((caddr_t (*)(void))val)(); 1602 symval->value = val; 1603 symval->size = es->st_size; 1604 return (0); 1605 } 1606 return (ENOENT); 1607 } 1608 1609 static int 1610 link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1611 linker_symval_t *symval) 1612 { 1613 if (link_elf_leak_locals) 1614 return (link_elf_debug_symbol_values(lf, sym, symval)); 1615 return (link_elf_symbol_values1(lf, sym, symval, false)); 1616 } 1617 1618 static int 1619 link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1620 linker_symval_t *symval) 1621 { 1622 elf_file_t ef = (elf_file_t)lf; 1623 const Elf_Sym *es = (const Elf_Sym *)sym; 1624 caddr_t val; 1625 1626 if (link_elf_symbol_values1(lf, sym, symval, true) == 0) 1627 return (0); 1628 if (ef->symtab == ef->ddbsymtab) 1629 return (ENOENT); 1630 1631 if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { 1632 symval->name = ef->ddbstrtab + es->st_name; 1633 val = (caddr_t)ef->address + es->st_value; 1634 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1635 val = ((caddr_t (*)(void))val)(); 1636 symval->value = val; 1637 symval->size = es->st_size; 1638 return (0); 1639 } 1640 return (ENOENT); 1641 } 1642 1643 static int 1644 link_elf_search_symbol(linker_file_t lf, caddr_t value, 1645 c_linker_sym_t *sym, long *diffp) 1646 { 1647 elf_file_t ef = (elf_file_t)lf; 1648 u_long off = (uintptr_t)(void *)value; 1649 u_long diff = off; 1650 u_long st_value; 1651 const Elf_Sym *es; 1652 const Elf_Sym *best = NULL; 1653 int i; 1654 1655 for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) { 1656 if (es->st_name == 0) 1657 continue; 1658 st_value = es->st_value + (uintptr_t) (void *) ef->address; 1659 if (off >= st_value) { 1660 if (off - st_value < diff) { 1661 diff = off - st_value; 1662 best = es; 1663 if (diff == 0) 1664 break; 1665 } else if (off - st_value == diff) { 1666 best = es; 1667 } 1668 } 1669 } 1670 if (best == NULL) 1671 *diffp = off; 1672 else 1673 *diffp = diff; 1674 *sym = (c_linker_sym_t) best; 1675 1676 return (0); 1677 } 1678 1679 /* 1680 * Look up a linker set on an ELF system. 1681 */ 1682 static int 1683 link_elf_lookup_set(linker_file_t lf, const char *name, 1684 void ***startp, void ***stopp, int *countp) 1685 { 1686 c_linker_sym_t sym; 1687 linker_symval_t symval; 1688 char *setsym; 1689 void **start, **stop; 1690 int len, error = 0, count; 1691 1692 len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */ 1693 setsym = malloc(len, M_LINKER, M_WAITOK); 1694 1695 /* get address of first entry */ 1696 snprintf(setsym, len, "%s%s", "__start_set_", name); 1697 error = link_elf_lookup_symbol(lf, setsym, &sym); 1698 if (error != 0) 1699 goto out; 1700 link_elf_symbol_values(lf, sym, &symval); 1701 if (symval.value == 0) { 1702 error = ESRCH; 1703 goto out; 1704 } 1705 start = (void **)symval.value; 1706 1707 /* get address of last entry */ 1708 snprintf(setsym, len, "%s%s", "__stop_set_", name); 1709 error = link_elf_lookup_symbol(lf, setsym, &sym); 1710 if (error != 0) 1711 goto out; 1712 link_elf_symbol_values(lf, sym, &symval); 1713 if (symval.value == 0) { 1714 error = ESRCH; 1715 goto out; 1716 } 1717 stop = (void **)symval.value; 1718 1719 /* and the number of entries */ 1720 count = stop - start; 1721 1722 /* and copy out */ 1723 if (startp != NULL) 1724 *startp = start; 1725 if (stopp != NULL) 1726 *stopp = stop; 1727 if (countp != NULL) 1728 *countp = count; 1729 1730 out: 1731 free(setsym, M_LINKER); 1732 return (error); 1733 } 1734 1735 static int 1736 link_elf_each_function_name(linker_file_t file, 1737 int (*callback)(const char *, void *), void *opaque) 1738 { 1739 elf_file_t ef = (elf_file_t)file; 1740 const Elf_Sym *symp; 1741 int i, error; 1742 1743 /* Exhaustive search */ 1744 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1745 if (symp->st_value != 0 && 1746 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1747 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1748 error = callback(ef->ddbstrtab + symp->st_name, opaque); 1749 if (error != 0) 1750 return (error); 1751 } 1752 } 1753 return (0); 1754 } 1755 1756 static int 1757 link_elf_each_function_nameval(linker_file_t file, 1758 linker_function_nameval_callback_t callback, void *opaque) 1759 { 1760 linker_symval_t symval; 1761 elf_file_t ef = (elf_file_t)file; 1762 const Elf_Sym *symp; 1763 int i, error; 1764 1765 /* Exhaustive search */ 1766 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1767 if (symp->st_value != 0 && 1768 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1769 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1770 error = link_elf_debug_symbol_values(file, 1771 (c_linker_sym_t) symp, &symval); 1772 if (error == 0) 1773 error = callback(file, i, &symval, opaque); 1774 if (error != 0) 1775 return (error); 1776 } 1777 } 1778 return (0); 1779 } 1780 1781 const Elf_Sym * 1782 elf_get_sym(linker_file_t lf, Elf_Size symidx) 1783 { 1784 elf_file_t ef = (elf_file_t)lf; 1785 1786 if (symidx >= ef->nchains) 1787 return (NULL); 1788 return (ef->symtab + symidx); 1789 } 1790 1791 const char * 1792 elf_get_symname(linker_file_t lf, Elf_Size symidx) 1793 { 1794 elf_file_t ef = (elf_file_t)lf; 1795 const Elf_Sym *sym; 1796 1797 if (symidx >= ef->nchains) 1798 return (NULL); 1799 sym = ef->symtab + symidx; 1800 return (ef->strtab + sym->st_name); 1801 } 1802 1803 /* 1804 * Symbol lookup function that can be used when the symbol index is known (ie 1805 * in relocations). It uses the symbol index instead of doing a fully fledged 1806 * hash table based lookup when such is valid. For example for local symbols. 1807 * This is not only more efficient, it's also more correct. It's not always 1808 * the case that the symbol can be found through the hash table. 1809 */ 1810 static int 1811 elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) 1812 { 1813 elf_file_t ef = (elf_file_t)lf; 1814 const Elf_Sym *sym; 1815 const char *symbol; 1816 Elf_Addr addr, start, base; 1817 1818 /* Don't even try to lookup the symbol if the index is bogus. */ 1819 if (symidx >= ef->nchains) { 1820 *res = 0; 1821 return (EINVAL); 1822 } 1823 1824 sym = ef->symtab + symidx; 1825 1826 /* 1827 * Don't do a full lookup when the symbol is local. It may even 1828 * fail because it may not be found through the hash table. 1829 */ 1830 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) { 1831 /* Force lookup failure when we have an insanity. */ 1832 if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) { 1833 *res = 0; 1834 return (EINVAL); 1835 } 1836 *res = ((Elf_Addr)ef->address + sym->st_value); 1837 return (0); 1838 } 1839 1840 /* 1841 * XXX we can avoid doing a hash table based lookup for global 1842 * symbols as well. This however is not always valid, so we'll 1843 * just do it the hard way for now. Performance tweaks can 1844 * always be added. 1845 */ 1846 1847 symbol = ef->strtab + sym->st_name; 1848 1849 /* Force a lookup failure if the symbol name is bogus. */ 1850 if (*symbol == 0) { 1851 *res = 0; 1852 return (EINVAL); 1853 } 1854 1855 addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps)); 1856 if (addr == 0 && ELF_ST_BIND(sym->st_info) != STB_WEAK) { 1857 *res = 0; 1858 return (EINVAL); 1859 } 1860 1861 if (elf_set_find(&set_pcpu_list, addr, &start, &base)) 1862 addr = addr - start + base; 1863 #ifdef VIMAGE 1864 else if (elf_set_find(&set_vnet_list, addr, &start, &base)) 1865 addr = addr - start + base; 1866 #endif 1867 *res = addr; 1868 return (0); 1869 } 1870 1871 static void 1872 link_elf_reloc_local(linker_file_t lf) 1873 { 1874 const Elf_Rel *rellim; 1875 const Elf_Rel *rel; 1876 const Elf_Rela *relalim; 1877 const Elf_Rela *rela; 1878 elf_file_t ef = (elf_file_t)lf; 1879 1880 /* Perform relocations without addend if there are any: */ 1881 if ((rel = ef->rel) != NULL) { 1882 rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize); 1883 while (rel < rellim) { 1884 elf_reloc_local(lf, (Elf_Addr)ef->address, rel, 1885 ELF_RELOC_REL, elf_lookup); 1886 rel++; 1887 } 1888 } 1889 1890 /* Perform relocations with addend if there are any: */ 1891 if ((rela = ef->rela) != NULL) { 1892 relalim = (const Elf_Rela *) 1893 ((const char *)ef->rela + ef->relasize); 1894 while (rela < relalim) { 1895 elf_reloc_local(lf, (Elf_Addr)ef->address, rela, 1896 ELF_RELOC_RELA, elf_lookup); 1897 rela++; 1898 } 1899 } 1900 } 1901 1902 static long 1903 link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab) 1904 { 1905 elf_file_t ef = (elf_file_t)lf; 1906 1907 *symtab = ef->ddbsymtab; 1908 1909 if (*symtab == NULL) 1910 return (0); 1911 1912 return (ef->ddbsymcnt); 1913 } 1914 1915 static long 1916 link_elf_strtab_get(linker_file_t lf, caddr_t *strtab) 1917 { 1918 elf_file_t ef = (elf_file_t)lf; 1919 1920 *strtab = ef->ddbstrtab; 1921 1922 if (*strtab == NULL) 1923 return (0); 1924 1925 return (ef->ddbstrcnt); 1926 } 1927 1928 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) || defined(__powerpc__) 1929 /* 1930 * Use this lookup routine when performing relocations early during boot. 1931 * The generic lookup routine depends on kobj, which is not initialized 1932 * at that point. 1933 */ 1934 static int 1935 elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, 1936 Elf_Addr *res) 1937 { 1938 elf_file_t ef; 1939 const Elf_Sym *symp; 1940 caddr_t val; 1941 1942 ef = (elf_file_t)lf; 1943 symp = ef->symtab + symidx; 1944 if (ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC) { 1945 val = (caddr_t)ef->address + symp->st_value; 1946 *res = ((Elf_Addr (*)(void))val)(); 1947 return (0); 1948 } 1949 return (ENOENT); 1950 } 1951 1952 void 1953 link_elf_ireloc(caddr_t kmdp) 1954 { 1955 struct elf_file eff; 1956 elf_file_t ef; 1957 1958 TSENTER(); 1959 ef = &eff; 1960 1961 bzero_early(ef, sizeof(*ef)); 1962 1963 ef->modptr = kmdp; 1964 ef->dynamic = (Elf_Dyn *)&_DYNAMIC; 1965 1966 #ifdef RELOCATABLE_KERNEL 1967 ef->address = (caddr_t) (__startkernel - KERNBASE); 1968 #else 1969 ef->address = 0; 1970 #endif 1971 parse_dynamic(ef); 1972 1973 link_elf_preload_parse_symbols(ef); 1974 relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); 1975 TSEXIT(); 1976 } 1977 1978 #if defined(__aarch64__) || defined(__amd64__) 1979 void 1980 link_elf_late_ireloc(void) 1981 { 1982 elf_file_t ef; 1983 1984 KASSERT(linker_kernel_file != NULL, 1985 ("link_elf_late_ireloc: No kernel linker file found")); 1986 ef = (elf_file_t)linker_kernel_file; 1987 1988 relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true); 1989 } 1990 #endif 1991 #endif 1992