1a4f67738SDoug Rabson /*- 28a36da99SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 38a36da99SPedro F. Giffuni * 4326e27d8SDoug Rabson * Copyright (c) 1998-2000 Doug Rabson 5a4f67738SDoug Rabson * All rights reserved. 6a4f67738SDoug Rabson * 7a4f67738SDoug Rabson * Redistribution and use in source and binary forms, with or without 8a4f67738SDoug Rabson * modification, are permitted provided that the following conditions 9a4f67738SDoug Rabson * are met: 10a4f67738SDoug Rabson * 1. Redistributions of source code must retain the above copyright 11a4f67738SDoug Rabson * notice, this list of conditions and the following disclaimer. 12a4f67738SDoug Rabson * 2. Redistributions in binary form must reproduce the above copyright 13a4f67738SDoug Rabson * notice, this list of conditions and the following disclaimer in the 14a4f67738SDoug Rabson * documentation and/or other materials provided with the distribution. 15a4f67738SDoug Rabson * 16a4f67738SDoug Rabson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17a4f67738SDoug Rabson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18a4f67738SDoug Rabson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19a4f67738SDoug Rabson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20a4f67738SDoug Rabson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21a4f67738SDoug Rabson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22a4f67738SDoug Rabson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23a4f67738SDoug Rabson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24a4f67738SDoug Rabson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25a4f67738SDoug Rabson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26a4f67738SDoug Rabson * SUCH DAMAGE. 27a4f67738SDoug Rabson */ 28a4f67738SDoug Rabson 29677b542eSDavid E. O'Brien #include <sys/cdefs.h> 30677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 31677b542eSDavid E. O'Brien 32a2024a3eSJohn Birrell #include "opt_ddb.h" 33ecb01c64SMarcel Moolenaar #include "opt_gdb.h" 34326e27d8SDoug Rabson 35a4f67738SDoug Rabson #include <sys/param.h> 36a4f67738SDoug Rabson #include <sys/systm.h> 375ad6c3b1SBruce Evans #ifdef GPROF 385ad6c3b1SBruce Evans #include <sys/gmon.h> 395ad6c3b1SBruce Evans #endif 40fb919e4dSMark Murray #include <sys/kernel.h> 41fb919e4dSMark Murray #include <sys/lock.h> 42a4f67738SDoug Rabson #include <sys/malloc.h> 431aa696baSMark Johnston #ifdef SPARSE_MAPPING 441aa696baSMark Johnston #include <sys/mman.h> 451aa696baSMark Johnston #endif 46c0b824f9SDima Dorfman #include <sys/mutex.h> 4762d615d5SJohn Baldwin #include <sys/mount.h> 4850c202c5SJeff Roberson #include <sys/pcpu.h> 49a4f67738SDoug Rabson #include <sys/proc.h> 50a4f67738SDoug Rabson #include <sys/namei.h> 51a4f67738SDoug Rabson #include <sys/fcntl.h> 52a4f67738SDoug Rabson #include <sys/vnode.h> 53a4f67738SDoug Rabson #include <sys/linker.h> 54eb5a41cfSLeandro Lupori #include <sys/sysctl.h> 55fb919e4dSMark Murray 56a4f67738SDoug Rabson #include <machine/elf.h> 57a4f67738SDoug Rabson 58eddfbb76SRobert Watson #include <net/vnet.h> 59eddfbb76SRobert Watson 60aed55708SRobert Watson #include <security/mac/mac_framework.h> 61aed55708SRobert Watson 62fe3db7c7SDoug Rabson #include <vm/vm.h> 63fe3db7c7SDoug Rabson #include <vm/vm_param.h> 64d1f088daSPeter Wemm #ifdef SPARSE_MAPPING 65fe3db7c7SDoug Rabson #include <vm/vm_object.h> 66fe3db7c7SDoug Rabson #include <vm/vm_kern.h> 67fe3db7c7SDoug Rabson #include <vm/vm_extern.h> 68d1f088daSPeter Wemm #endif 69fe3db7c7SDoug Rabson #include <vm/pmap.h> 70fe3db7c7SDoug Rabson #include <vm/vm_map.h> 71fb919e4dSMark Murray 72f9980387SPeter Wemm #include <sys/link_elf.h> 73fe3db7c7SDoug Rabson 74326e27d8SDoug Rabson #include "linker_if.h" 75a4f67738SDoug Rabson 761a330eb0SPeter Wemm #define MAXSEGS 4 771a330eb0SPeter Wemm 784f2945f8SDavid E. O'Brien typedef struct elf_file { 794f2945f8SDavid E. O'Brien struct linker_file lf; /* Common fields */ 804f2945f8SDavid E. O'Brien int preloaded; /* Was file pre-loaded */ 814f2945f8SDavid E. O'Brien caddr_t address; /* Relocation address */ 824f2945f8SDavid E. O'Brien #ifdef SPARSE_MAPPING 834f2945f8SDavid E. O'Brien vm_object_t object; /* VM object to hold file pages */ 844f2945f8SDavid E. O'Brien #endif 854f2945f8SDavid E. O'Brien Elf_Dyn *dynamic; /* Symbol table etc. */ 864f2945f8SDavid E. O'Brien Elf_Hashelt nbuckets; /* DT_HASH info */ 874f2945f8SDavid E. O'Brien Elf_Hashelt nchains; 884f2945f8SDavid E. O'Brien const Elf_Hashelt *buckets; 894f2945f8SDavid E. O'Brien const Elf_Hashelt *chains; 904f2945f8SDavid E. O'Brien caddr_t hash; 914f2945f8SDavid E. O'Brien caddr_t strtab; /* DT_STRTAB */ 924f2945f8SDavid E. O'Brien int strsz; /* DT_STRSZ */ 934f2945f8SDavid E. O'Brien const Elf_Sym *symtab; /* DT_SYMTAB */ 944f2945f8SDavid E. O'Brien Elf_Addr *got; /* DT_PLTGOT */ 954f2945f8SDavid E. O'Brien const Elf_Rel *pltrel; /* DT_JMPREL */ 964f2945f8SDavid E. O'Brien int pltrelsize; /* DT_PLTRELSZ */ 974f2945f8SDavid E. O'Brien const Elf_Rela *pltrela; /* DT_JMPREL */ 984f2945f8SDavid E. O'Brien int pltrelasize; /* DT_PLTRELSZ */ 994f2945f8SDavid E. O'Brien const Elf_Rel *rel; /* DT_REL */ 1004f2945f8SDavid E. O'Brien int relsize; /* DT_RELSZ */ 1014f2945f8SDavid E. O'Brien const Elf_Rela *rela; /* DT_RELA */ 1024f2945f8SDavid E. O'Brien int relasize; /* DT_RELASZ */ 1034f2945f8SDavid E. O'Brien caddr_t modptr; 1044f2945f8SDavid E. O'Brien const Elf_Sym *ddbsymtab; /* The symbol table we are using */ 1054f2945f8SDavid E. O'Brien long ddbsymcnt; /* Number of symbols */ 1064f2945f8SDavid E. O'Brien caddr_t ddbstrtab; /* String table */ 1074f2945f8SDavid E. O'Brien long ddbstrcnt; /* number of bytes in string table */ 1084f2945f8SDavid E. O'Brien caddr_t symbase; /* malloc'ed symbold base */ 1094f2945f8SDavid E. O'Brien caddr_t strbase; /* malloc'ed string base */ 1104f2945f8SDavid E. O'Brien caddr_t ctftab; /* CTF table */ 1114f2945f8SDavid E. O'Brien long ctfcnt; /* number of bytes in CTF table */ 1124f2945f8SDavid E. O'Brien caddr_t ctfoff; /* CTF offset table */ 1134f2945f8SDavid E. O'Brien caddr_t typoff; /* Type offset table */ 1144f2945f8SDavid E. O'Brien long typlen; /* Number of type entries. */ 11550c202c5SJeff Roberson Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */ 11650c202c5SJeff Roberson Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */ 11750c202c5SJeff Roberson Elf_Addr pcpu_base; /* Relocated pcpu set address. */ 118eddfbb76SRobert Watson #ifdef VIMAGE 119eddfbb76SRobert Watson Elf_Addr vnet_start; /* Pre-relocation vnet set start. */ 120eddfbb76SRobert Watson Elf_Addr vnet_stop; /* Pre-relocation vnet set stop. */ 121eddfbb76SRobert Watson Elf_Addr vnet_base; /* Relocated vnet set address. */ 122eddfbb76SRobert Watson #endif 1234f2945f8SDavid E. O'Brien #ifdef GDB 1244f2945f8SDavid E. O'Brien struct link_map gdb; /* hooks for gdb */ 1254f2945f8SDavid E. O'Brien #endif 1264f2945f8SDavid E. O'Brien } *elf_file_t; 1274f2945f8SDavid E. O'Brien 12847813f5dSMikolaj Golub struct elf_set { 12947813f5dSMikolaj Golub Elf_Addr es_start; 13047813f5dSMikolaj Golub Elf_Addr es_stop; 13147813f5dSMikolaj Golub Elf_Addr es_base; 13247813f5dSMikolaj Golub TAILQ_ENTRY(elf_set) es_link; 13347813f5dSMikolaj Golub }; 13447813f5dSMikolaj Golub 13547813f5dSMikolaj Golub TAILQ_HEAD(elf_set_head, elf_set); 13647813f5dSMikolaj Golub 137a2024a3eSJohn Birrell #include <kern/kern_ctf.c> 138a2024a3eSJohn Birrell 139c143d6c2SMarcel Moolenaar static int link_elf_link_common_finish(linker_file_t); 14054823af2SPeter Wemm static int link_elf_link_preload(linker_class_t cls, 141326e27d8SDoug Rabson const char *, linker_file_t *); 14254823af2SPeter Wemm static int link_elf_link_preload_finish(linker_file_t); 1432f22b3ffSMatthew D Fleming static int link_elf_load_file(linker_class_t, const char *, 1442f22b3ffSMatthew D Fleming linker_file_t *); 145326e27d8SDoug Rabson static int link_elf_lookup_symbol(linker_file_t, const char *, 146326e27d8SDoug Rabson c_linker_sym_t *); 14795c20fafSKonstantin Belousov static int link_elf_lookup_debug_symbol(linker_file_t, const char *, 14895c20fafSKonstantin Belousov c_linker_sym_t *); 1492f22b3ffSMatthew D Fleming static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, 1502f22b3ffSMatthew D Fleming linker_symval_t *); 15195c20fafSKonstantin Belousov static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t, 15295c20fafSKonstantin Belousov linker_symval_t*); 1532f22b3ffSMatthew D Fleming static int link_elf_search_symbol(linker_file_t, caddr_t, 1542f22b3ffSMatthew D Fleming c_linker_sym_t *, long *); 155326e27d8SDoug Rabson 156326e27d8SDoug Rabson static void link_elf_unload_file(linker_file_t); 15754823af2SPeter Wemm static void link_elf_unload_preload(linker_file_t); 158f41325dbSPeter Wemm static int link_elf_lookup_set(linker_file_t, const char *, 159f41325dbSPeter Wemm void ***, void ***, int *); 160bb9fe9ddSBrian Feldman static int link_elf_each_function_name(linker_file_t, 1612f22b3ffSMatthew D Fleming int (*)(const char *, void *), void *); 162a2024a3eSJohn Birrell static int link_elf_each_function_nameval(linker_file_t, 1632f22b3ffSMatthew D Fleming linker_function_nameval_callback_t, void *); 1647251b4bfSJake Burkholder static void link_elf_reloc_local(linker_file_t); 16500a5db46SStacey Son static long link_elf_symtab_get(linker_file_t, const Elf_Sym **); 16600a5db46SStacey Son static long link_elf_strtab_get(linker_file_t, caddr_t *); 167cff8c6f2SKonstantin Belousov static int elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *); 168326e27d8SDoug Rabson 169326e27d8SDoug Rabson static kobj_method_t link_elf_methods[] = { 170326e27d8SDoug Rabson KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol), 17195c20fafSKonstantin Belousov KOBJMETHOD(linker_lookup_debug_symbol, link_elf_lookup_debug_symbol), 172326e27d8SDoug Rabson KOBJMETHOD(linker_symbol_values, link_elf_symbol_values), 17395c20fafSKonstantin Belousov KOBJMETHOD(linker_debug_symbol_values, link_elf_debug_symbol_values), 174326e27d8SDoug Rabson KOBJMETHOD(linker_search_symbol, link_elf_search_symbol), 175326e27d8SDoug Rabson KOBJMETHOD(linker_unload, link_elf_unload_file), 17654823af2SPeter Wemm KOBJMETHOD(linker_load_file, link_elf_load_file), 17754823af2SPeter Wemm KOBJMETHOD(linker_link_preload, link_elf_link_preload), 17854823af2SPeter Wemm KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish), 179f41325dbSPeter Wemm KOBJMETHOD(linker_lookup_set, link_elf_lookup_set), 180bb9fe9ddSBrian Feldman KOBJMETHOD(linker_each_function_name, link_elf_each_function_name), 181a2024a3eSJohn Birrell KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval), 182a2024a3eSJohn Birrell KOBJMETHOD(linker_ctf_get, link_elf_ctf_get), 18300a5db46SStacey Son KOBJMETHOD(linker_symtab_get, link_elf_symtab_get), 18400a5db46SStacey Son KOBJMETHOD(linker_strtab_get, link_elf_strtab_get), 1856d775f0bSMark Johnston KOBJMETHOD_END 186326e27d8SDoug Rabson }; 187326e27d8SDoug Rabson 188326e27d8SDoug Rabson static struct linker_class link_elf_class = { 189326e27d8SDoug Rabson #if ELF_TARG_CLASS == ELFCLASS32 190326e27d8SDoug Rabson "elf32", 191326e27d8SDoug Rabson #else 192326e27d8SDoug Rabson "elf64", 193326e27d8SDoug Rabson #endif 194326e27d8SDoug Rabson link_elf_methods, sizeof(struct elf_file) 195326e27d8SDoug Rabson }; 196326e27d8SDoug Rabson 197*ecd8245eSKonstantin Belousov static bool link_elf_leak_locals = true; 198*ecd8245eSKonstantin Belousov SYSCTL_BOOL(_debug, OID_AUTO, link_elf_leak_locals, 199*ecd8245eSKonstantin Belousov CTLFLAG_RWTUN, &link_elf_leak_locals, 0, 200*ecd8245eSKonstantin Belousov "Allow local symbols to participate in global module symbol resolution"); 201*ecd8245eSKonstantin Belousov 20236716fe2SMark Johnston typedef int (*elf_reloc_fn)(linker_file_t lf, Elf_Addr relocbase, 20336716fe2SMark Johnston const void *data, int type, elf_lookup_fn lookup); 20436716fe2SMark Johnston 2052f22b3ffSMatthew D Fleming static int parse_dynamic(elf_file_t); 2062f22b3ffSMatthew D Fleming static int relocate_file(elf_file_t); 20736716fe2SMark Johnston static int relocate_file1(elf_file_t ef, elf_lookup_fn lookup, 20836716fe2SMark Johnston elf_reloc_fn reloc, bool ifuncs); 2092f22b3ffSMatthew D Fleming static int link_elf_preload_parse_symbols(elf_file_t); 210326e27d8SDoug Rabson 21147813f5dSMikolaj Golub static struct elf_set_head set_pcpu_list; 21247813f5dSMikolaj Golub #ifdef VIMAGE 21347813f5dSMikolaj Golub static struct elf_set_head set_vnet_list; 21447813f5dSMikolaj Golub #endif 21547813f5dSMikolaj Golub 21647813f5dSMikolaj Golub static void 21747813f5dSMikolaj Golub elf_set_add(struct elf_set_head *list, Elf_Addr start, Elf_Addr stop, Elf_Addr base) 21847813f5dSMikolaj Golub { 21947813f5dSMikolaj Golub struct elf_set *set, *iter; 22047813f5dSMikolaj Golub 22147813f5dSMikolaj Golub set = malloc(sizeof(*set), M_LINKER, M_WAITOK); 22247813f5dSMikolaj Golub set->es_start = start; 22347813f5dSMikolaj Golub set->es_stop = stop; 22447813f5dSMikolaj Golub set->es_base = base; 22547813f5dSMikolaj Golub 22647813f5dSMikolaj Golub TAILQ_FOREACH(iter, list, es_link) { 22747813f5dSMikolaj Golub KASSERT((set->es_start < iter->es_start && set->es_stop < iter->es_stop) || 22847813f5dSMikolaj Golub (set->es_start > iter->es_start && set->es_stop > iter->es_stop), 22947813f5dSMikolaj Golub ("linker sets intersection: to insert: 0x%jx-0x%jx; inserted: 0x%jx-0x%jx", 23047813f5dSMikolaj Golub (uintmax_t)set->es_start, (uintmax_t)set->es_stop, 23147813f5dSMikolaj Golub (uintmax_t)iter->es_start, (uintmax_t)iter->es_stop)); 23247813f5dSMikolaj Golub 23347813f5dSMikolaj Golub if (iter->es_start > set->es_start) { 23447813f5dSMikolaj Golub TAILQ_INSERT_BEFORE(iter, set, es_link); 23547813f5dSMikolaj Golub break; 23647813f5dSMikolaj Golub } 23747813f5dSMikolaj Golub } 23847813f5dSMikolaj Golub 23947813f5dSMikolaj Golub if (iter == NULL) 24047813f5dSMikolaj Golub TAILQ_INSERT_TAIL(list, set, es_link); 24147813f5dSMikolaj Golub } 24247813f5dSMikolaj Golub 24347813f5dSMikolaj Golub static int 24447813f5dSMikolaj Golub elf_set_find(struct elf_set_head *list, Elf_Addr addr, Elf_Addr *start, Elf_Addr *base) 24547813f5dSMikolaj Golub { 24647813f5dSMikolaj Golub struct elf_set *set; 24747813f5dSMikolaj Golub 24847813f5dSMikolaj Golub TAILQ_FOREACH(set, list, es_link) { 24947813f5dSMikolaj Golub if (addr < set->es_start) 25047813f5dSMikolaj Golub return (0); 25147813f5dSMikolaj Golub if (addr < set->es_stop) { 25247813f5dSMikolaj Golub *start = set->es_start; 25347813f5dSMikolaj Golub *base = set->es_base; 25447813f5dSMikolaj Golub return (1); 25547813f5dSMikolaj Golub } 25647813f5dSMikolaj Golub } 25747813f5dSMikolaj Golub 25847813f5dSMikolaj Golub return (0); 25947813f5dSMikolaj Golub } 26047813f5dSMikolaj Golub 26147813f5dSMikolaj Golub static void 26247813f5dSMikolaj Golub elf_set_delete(struct elf_set_head *list, Elf_Addr start) 26347813f5dSMikolaj Golub { 26447813f5dSMikolaj Golub struct elf_set *set; 26547813f5dSMikolaj Golub 26647813f5dSMikolaj Golub TAILQ_FOREACH(set, list, es_link) { 26747813f5dSMikolaj Golub if (start < set->es_start) 26847813f5dSMikolaj Golub break; 26947813f5dSMikolaj Golub if (start == set->es_start) { 27047813f5dSMikolaj Golub TAILQ_REMOVE(list, set, es_link); 27147813f5dSMikolaj Golub free(set, M_LINKER); 27247813f5dSMikolaj Golub return; 27347813f5dSMikolaj Golub } 27447813f5dSMikolaj Golub } 27547813f5dSMikolaj Golub KASSERT(0, ("deleting unknown linker set (start = 0x%jx)", 27647813f5dSMikolaj Golub (uintmax_t)start)); 27747813f5dSMikolaj Golub } 27847813f5dSMikolaj Golub 279ecb01c64SMarcel Moolenaar #ifdef GDB 2802f22b3ffSMatthew D Fleming static void r_debug_state(struct r_debug *, struct link_map *); 281326e27d8SDoug Rabson 282326e27d8SDoug Rabson /* 283326e27d8SDoug Rabson * A list of loaded modules for GDB to use for loading symbols. 284326e27d8SDoug Rabson */ 285326e27d8SDoug Rabson struct r_debug r_debug; 286326e27d8SDoug Rabson 2872f22b3ffSMatthew D Fleming #define GDB_STATE(s) do { \ 2882f22b3ffSMatthew D Fleming r_debug.r_state = s; r_debug_state(NULL, NULL); \ 2892f22b3ffSMatthew D Fleming } while (0) 290326e27d8SDoug Rabson 291326e27d8SDoug Rabson /* 292326e27d8SDoug Rabson * Function for the debugger to set a breakpoint on to gain control. 293326e27d8SDoug Rabson */ 29437c84183SPoul-Henning Kamp static void 295ca29467eSJohn Baldwin r_debug_state(struct r_debug *dummy_one __unused, 296ca29467eSJohn Baldwin struct link_map *dummy_two __unused) 297326e27d8SDoug Rabson { 298326e27d8SDoug Rabson } 299326e27d8SDoug Rabson 3001720979bSMarcel Moolenaar static void 3011720979bSMarcel Moolenaar link_elf_add_gdb(struct link_map *l) 3021720979bSMarcel Moolenaar { 3031720979bSMarcel Moolenaar struct link_map *prev; 3041720979bSMarcel Moolenaar 305c143d6c2SMarcel Moolenaar l->l_next = NULL; 306c143d6c2SMarcel Moolenaar 307c143d6c2SMarcel Moolenaar if (r_debug.r_map == NULL) { 308c143d6c2SMarcel Moolenaar /* Add first. */ 309c143d6c2SMarcel Moolenaar l->l_prev = NULL; 310c143d6c2SMarcel Moolenaar r_debug.r_map = l; 311c143d6c2SMarcel Moolenaar } else { 312c143d6c2SMarcel Moolenaar /* Append to list. */ 3132f22b3ffSMatthew D Fleming for (prev = r_debug.r_map; 3142f22b3ffSMatthew D Fleming prev->l_next != NULL; 3152f22b3ffSMatthew D Fleming prev = prev->l_next) 3161720979bSMarcel Moolenaar ; 3171720979bSMarcel Moolenaar l->l_prev = prev; 3181720979bSMarcel Moolenaar prev->l_next = l; 3191720979bSMarcel Moolenaar } 320c143d6c2SMarcel Moolenaar } 3211720979bSMarcel Moolenaar 3221720979bSMarcel Moolenaar static void 3231720979bSMarcel Moolenaar link_elf_delete_gdb(struct link_map *l) 3241720979bSMarcel Moolenaar { 3251720979bSMarcel Moolenaar if (l->l_prev == NULL) { 326c143d6c2SMarcel Moolenaar /* Remove first. */ 3271720979bSMarcel Moolenaar if ((r_debug.r_map = l->l_next) != NULL) 3281720979bSMarcel Moolenaar l->l_next->l_prev = NULL; 329c143d6c2SMarcel Moolenaar } else { 330c143d6c2SMarcel Moolenaar /* Remove any but first. */ 3311720979bSMarcel Moolenaar if ((l->l_prev->l_next = l->l_next) != NULL) 3321720979bSMarcel Moolenaar l->l_next->l_prev = l->l_prev; 3331720979bSMarcel Moolenaar } 334c143d6c2SMarcel Moolenaar } 335ecb01c64SMarcel Moolenaar #endif /* GDB */ 336a4f67738SDoug Rabson 3374f2945f8SDavid E. O'Brien /* 3384f2945f8SDavid E. O'Brien * The kernel symbol table starts here. 3394f2945f8SDavid E. O'Brien */ 3404f2945f8SDavid E. O'Brien extern struct _dynamic _DYNAMIC; 3414f2945f8SDavid E. O'Brien 342a4f67738SDoug Rabson static void 343552f9f63SEdwin Groothuis link_elf_error(const char *filename, const char *s) 3441720979bSMarcel Moolenaar { 345552f9f63SEdwin Groothuis if (filename == NULL) 3461720979bSMarcel Moolenaar printf("kldload: %s\n", s); 347552f9f63SEdwin Groothuis else 348552f9f63SEdwin Groothuis printf("kldload: %s: %s\n", filename, s); 3491720979bSMarcel Moolenaar } 3501720979bSMarcel Moolenaar 3510067051fSMarcel Moolenaar static void 3520067051fSMarcel Moolenaar link_elf_invoke_ctors(caddr_t addr, size_t size) 3530067051fSMarcel Moolenaar { 3540067051fSMarcel Moolenaar void (**ctor)(void); 3550067051fSMarcel Moolenaar size_t i, cnt; 3560067051fSMarcel Moolenaar 3570067051fSMarcel Moolenaar if (addr == NULL || size == 0) 3580067051fSMarcel Moolenaar return; 3590067051fSMarcel Moolenaar cnt = size / sizeof(*ctor); 3600067051fSMarcel Moolenaar ctor = (void *)addr; 3610067051fSMarcel Moolenaar for (i = 0; i < cnt; i++) { 3620067051fSMarcel Moolenaar if (ctor[i] != NULL) 3630067051fSMarcel Moolenaar (*ctor[i])(); 3640067051fSMarcel Moolenaar } 3650067051fSMarcel Moolenaar } 3660067051fSMarcel Moolenaar 367c143d6c2SMarcel Moolenaar /* 368c143d6c2SMarcel Moolenaar * Actions performed after linking/loading both the preloaded kernel and any 369c143d6c2SMarcel Moolenaar * modules; whether preloaded or dynamicly loaded. 370c143d6c2SMarcel Moolenaar */ 371c143d6c2SMarcel Moolenaar static int 372c143d6c2SMarcel Moolenaar link_elf_link_common_finish(linker_file_t lf) 373c143d6c2SMarcel Moolenaar { 374ecb01c64SMarcel Moolenaar #ifdef GDB 375c143d6c2SMarcel Moolenaar elf_file_t ef = (elf_file_t)lf; 376c143d6c2SMarcel Moolenaar char *newfilename; 377c143d6c2SMarcel Moolenaar #endif 3781aeb23cdSMarcel Moolenaar int error; 3791aeb23cdSMarcel Moolenaar 3801aeb23cdSMarcel Moolenaar /* Notify MD code that a module is being loaded. */ 3811aeb23cdSMarcel Moolenaar error = elf_cpu_load_file(lf); 3822f22b3ffSMatthew D Fleming if (error != 0) 3831aeb23cdSMarcel Moolenaar return (error); 384c143d6c2SMarcel Moolenaar 385ecb01c64SMarcel Moolenaar #ifdef GDB 386c143d6c2SMarcel Moolenaar GDB_STATE(RT_ADD); 387c143d6c2SMarcel Moolenaar ef->gdb.l_addr = lf->address; 388a163d034SWarner Losh newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK); 389c143d6c2SMarcel Moolenaar strcpy(newfilename, lf->filename); 390c143d6c2SMarcel Moolenaar ef->gdb.l_name = newfilename; 391c143d6c2SMarcel Moolenaar ef->gdb.l_ld = ef->dynamic; 392c143d6c2SMarcel Moolenaar link_elf_add_gdb(&ef->gdb); 393c143d6c2SMarcel Moolenaar GDB_STATE(RT_CONSISTENT); 394c143d6c2SMarcel Moolenaar #endif 395c143d6c2SMarcel Moolenaar 3960067051fSMarcel Moolenaar /* Invoke .ctors */ 3970067051fSMarcel Moolenaar link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size); 398c143d6c2SMarcel Moolenaar return (0); 399c143d6c2SMarcel Moolenaar } 400c143d6c2SMarcel Moolenaar 4019411e24dSBrandon Bergren #ifdef RELOCATABLE_KERNEL 40240b664f6SBrandon Bergren /* 40340b664f6SBrandon Bergren * __startkernel and __endkernel are symbols set up as relocation canaries. 40440b664f6SBrandon Bergren * 40540b664f6SBrandon Bergren * They are defined in locore to reference linker script symbols at the 40640b664f6SBrandon Bergren * beginning and end of the LOAD area. This has the desired side effect of 40740b664f6SBrandon Bergren * giving us variables that have relative relocations pointing at them, so 40840b664f6SBrandon Bergren * relocation of the kernel object will cause the variables to be updated 40940b664f6SBrandon Bergren * automatically by the runtime linker when we initialize. 41040b664f6SBrandon Bergren * 41140b664f6SBrandon Bergren * There are two main reasons to relocate the kernel: 41240b664f6SBrandon Bergren * 1) If the loader needed to load the kernel at an alternate load address. 41340b664f6SBrandon Bergren * 2) If the kernel is switching address spaces on machines like POWER9 41440b664f6SBrandon Bergren * under Radix where the high bits of the effective address are used to 41540b664f6SBrandon Bergren * differentiate between hypervisor, host, guest, and problem state. 41640b664f6SBrandon Bergren */ 417ee900504SNathan Whitehorn extern vm_offset_t __startkernel, __endkernel; 4189411e24dSBrandon Bergren #endif 419c6014c73SNathan Whitehorn 420eb5a41cfSLeandro Lupori static unsigned long kern_relbase = KERNBASE; 421eb5a41cfSLeandro Lupori 422eb5a41cfSLeandro Lupori SYSCTL_ULONG(_kern, OID_AUTO, base_address, CTLFLAG_RD, 423eb5a41cfSLeandro Lupori SYSCTL_NULL_ULONG_PTR, KERNBASE, "Kernel base address"); 424eb5a41cfSLeandro Lupori SYSCTL_ULONG(_kern, OID_AUTO, relbase_address, CTLFLAG_RD, 425eb5a41cfSLeandro Lupori &kern_relbase, 0, "Kernel relocated base address"); 426eb5a41cfSLeandro Lupori 4271720979bSMarcel Moolenaar static void 428a4f67738SDoug Rabson link_elf_init(void* arg) 429a4f67738SDoug Rabson { 430de78ca7eSPeter Wemm Elf_Dyn *dp; 4310067051fSMarcel Moolenaar Elf_Addr *ctors_addrp; 4320067051fSMarcel Moolenaar Elf_Size *ctors_sizep; 433de78ca7eSPeter Wemm caddr_t modptr, baseptr, sizeptr; 434de78ca7eSPeter Wemm elf_file_t ef; 435fe20aaecSRyan Libby const char *modname; 436a4f67738SDoug Rabson 437326e27d8SDoug Rabson linker_add_class(&link_elf_class); 438a4f67738SDoug Rabson 439de78ca7eSPeter Wemm dp = (Elf_Dyn *)&_DYNAMIC; 440de78ca7eSPeter Wemm modname = NULL; 44167db8b23SPeter Wemm modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel"); 44267db8b23SPeter Wemm if (modptr == NULL) 443de78ca7eSPeter Wemm modptr = preload_search_by_type("elf kernel"); 444de78ca7eSPeter Wemm modname = (char *)preload_search_info(modptr, MODINFO_NAME); 445de78ca7eSPeter Wemm if (modname == NULL) 446de78ca7eSPeter Wemm modname = "kernel"; 447326e27d8SDoug Rabson linker_kernel_file = linker_make_file(modname, &link_elf_class); 448a4f67738SDoug Rabson if (linker_kernel_file == NULL) 4492f22b3ffSMatthew D Fleming panic("%s: Can't create linker structures for kernel", 4502f22b3ffSMatthew D Fleming __func__); 451326e27d8SDoug Rabson 452326e27d8SDoug Rabson ef = (elf_file_t) linker_kernel_file; 45354823af2SPeter Wemm ef->preloaded = 1; 4549411e24dSBrandon Bergren #ifdef RELOCATABLE_KERNEL 45540b664f6SBrandon Bergren /* Compute relative displacement */ 456c6014c73SNathan Whitehorn ef->address = (caddr_t) (__startkernel - KERNBASE); 457c6014c73SNathan Whitehorn #else 458326e27d8SDoug Rabson ef->address = 0; 459c6014c73SNathan Whitehorn #endif 460326e27d8SDoug Rabson #ifdef SPARSE_MAPPING 4611aa696baSMark Johnston ef->object = NULL; 462326e27d8SDoug Rabson #endif 463326e27d8SDoug Rabson ef->dynamic = dp; 464326e27d8SDoug Rabson 4652f22b3ffSMatthew D Fleming if (dp != NULL) 466326e27d8SDoug Rabson parse_dynamic(ef); 4679411e24dSBrandon Bergren #ifdef RELOCATABLE_KERNEL 468ee900504SNathan Whitehorn linker_kernel_file->address = (caddr_t)__startkernel; 469ee900504SNathan Whitehorn linker_kernel_file->size = (intptr_t)(__endkernel - __startkernel); 470eb5a41cfSLeandro Lupori kern_relbase = (unsigned long)__startkernel; 471ee900504SNathan Whitehorn #else 472c6014c73SNathan Whitehorn linker_kernel_file->address += KERNBASE; 473d009ccfaSBruce Evans linker_kernel_file->size = -(intptr_t)linker_kernel_file->address; 474ee900504SNathan Whitehorn #endif 475de78ca7eSPeter Wemm 4762f22b3ffSMatthew D Fleming if (modptr != NULL) { 4772d636ab0SPeter Wemm ef->modptr = modptr; 478de78ca7eSPeter Wemm baseptr = preload_search_info(modptr, MODINFO_ADDR); 4792f22b3ffSMatthew D Fleming if (baseptr != NULL) 480de78ca7eSPeter Wemm linker_kernel_file->address = *(caddr_t *)baseptr; 481de78ca7eSPeter Wemm sizeptr = preload_search_info(modptr, MODINFO_SIZE); 4822f22b3ffSMatthew D Fleming if (sizeptr != NULL) 483de78ca7eSPeter Wemm linker_kernel_file->size = *(size_t *)sizeptr; 4840067051fSMarcel Moolenaar ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 4850067051fSMarcel Moolenaar MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 4860067051fSMarcel Moolenaar ctors_sizep = (Elf_Size *)preload_search_info(modptr, 4870067051fSMarcel Moolenaar MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 4880067051fSMarcel Moolenaar if (ctors_addrp != NULL && ctors_sizep != NULL) { 489853a17adSEnji Cooper linker_kernel_file->ctors_addr = ef->address + 4900067051fSMarcel Moolenaar *ctors_addrp; 4910067051fSMarcel Moolenaar linker_kernel_file->ctors_size = *ctors_sizep; 4920067051fSMarcel Moolenaar } 493de78ca7eSPeter Wemm } 49454823af2SPeter Wemm (void)link_elf_preload_parse_symbols(ef); 495326e27d8SDoug Rabson 496ecb01c64SMarcel Moolenaar #ifdef GDB 497c143d6c2SMarcel Moolenaar r_debug.r_map = NULL; 498326e27d8SDoug Rabson r_debug.r_brk = r_debug_state; 499326e27d8SDoug Rabson r_debug.r_state = RT_CONSISTENT; 500326e27d8SDoug Rabson #endif 501c143d6c2SMarcel Moolenaar 502c143d6c2SMarcel Moolenaar (void)link_elf_link_common_finish(linker_kernel_file); 50359800afcSJohn Baldwin linker_kernel_file->flags |= LINKER_FILE_LINKED; 50447813f5dSMikolaj Golub TAILQ_INIT(&set_pcpu_list); 50547813f5dSMikolaj Golub #ifdef VIMAGE 50647813f5dSMikolaj Golub TAILQ_INIT(&set_vnet_list); 50747813f5dSMikolaj Golub #endif 508a4f67738SDoug Rabson } 509a4f67738SDoug Rabson 510891cf3edSEd Maste SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, NULL); 511a4f67738SDoug Rabson 512a4f67738SDoug Rabson static int 51354823af2SPeter Wemm link_elf_preload_parse_symbols(elf_file_t ef) 5142d636ab0SPeter Wemm { 5152d636ab0SPeter Wemm caddr_t pointer; 5162d636ab0SPeter Wemm caddr_t ssym, esym, base; 5172d636ab0SPeter Wemm caddr_t strtab; 5182d636ab0SPeter Wemm int strcnt; 5192d636ab0SPeter Wemm Elf_Sym *symtab; 5202d636ab0SPeter Wemm int symcnt; 5212d636ab0SPeter Wemm 522ca65d5c7SPeter Wemm if (ef->modptr == NULL) 5232f22b3ffSMatthew D Fleming return (0); 5242f22b3ffSMatthew D Fleming pointer = preload_search_info(ef->modptr, 5252f22b3ffSMatthew D Fleming MODINFO_METADATA | MODINFOMD_SSYM); 5262d636ab0SPeter Wemm if (pointer == NULL) 5272f22b3ffSMatthew D Fleming return (0); 5282d636ab0SPeter Wemm ssym = *(caddr_t *)pointer; 5292f22b3ffSMatthew D Fleming pointer = preload_search_info(ef->modptr, 5302f22b3ffSMatthew D Fleming MODINFO_METADATA | MODINFOMD_ESYM); 5312d636ab0SPeter Wemm if (pointer == NULL) 5322f22b3ffSMatthew D Fleming return (0); 5332d636ab0SPeter Wemm esym = *(caddr_t *)pointer; 5342d636ab0SPeter Wemm 5352d636ab0SPeter Wemm base = ssym; 5362d636ab0SPeter Wemm 5372d636ab0SPeter Wemm symcnt = *(long *)base; 5382d636ab0SPeter Wemm base += sizeof(long); 5392d636ab0SPeter Wemm symtab = (Elf_Sym *)base; 5402d636ab0SPeter Wemm base += roundup(symcnt, sizeof(long)); 5412d636ab0SPeter Wemm 5422d636ab0SPeter Wemm if (base > esym || base < ssym) { 5432d636ab0SPeter Wemm printf("Symbols are corrupt!\n"); 5442f22b3ffSMatthew D Fleming return (EINVAL); 5452d636ab0SPeter Wemm } 5462d636ab0SPeter Wemm 5472d636ab0SPeter Wemm strcnt = *(long *)base; 5482d636ab0SPeter Wemm base += sizeof(long); 5492d636ab0SPeter Wemm strtab = base; 5502d636ab0SPeter Wemm base += roundup(strcnt, sizeof(long)); 5512d636ab0SPeter Wemm 5522d636ab0SPeter Wemm if (base > esym || base < ssym) { 5532d636ab0SPeter Wemm printf("Symbols are corrupt!\n"); 5542f22b3ffSMatthew D Fleming return (EINVAL); 5552d636ab0SPeter Wemm } 5562d636ab0SPeter Wemm 5572d636ab0SPeter Wemm ef->ddbsymtab = symtab; 5582d636ab0SPeter Wemm ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 5592d636ab0SPeter Wemm ef->ddbstrtab = strtab; 5602d636ab0SPeter Wemm ef->ddbstrcnt = strcnt; 5612d636ab0SPeter Wemm 5622f22b3ffSMatthew D Fleming return (0); 5632d636ab0SPeter Wemm } 5642d636ab0SPeter Wemm 5652d636ab0SPeter Wemm static int 566326e27d8SDoug Rabson parse_dynamic(elf_file_t ef) 567a4f67738SDoug Rabson { 568326e27d8SDoug Rabson Elf_Dyn *dp; 569fe3db7c7SDoug Rabson int plttype = DT_REL; 570a4f67738SDoug Rabson 571a4f67738SDoug Rabson for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) { 572a4f67738SDoug Rabson switch (dp->d_tag) { 573a4f67738SDoug Rabson case DT_HASH: 574a4f67738SDoug Rabson { 575a4f67738SDoug Rabson /* From src/libexec/rtld-elf/rtld.c */ 5767e510298SJake Burkholder const Elf_Hashelt *hashtab = (const Elf_Hashelt *) 577a4f67738SDoug Rabson (ef->address + dp->d_un.d_ptr); 578a4f67738SDoug Rabson ef->nbuckets = hashtab[0]; 579a4f67738SDoug Rabson ef->nchains = hashtab[1]; 580a4f67738SDoug Rabson ef->buckets = hashtab + 2; 581a4f67738SDoug Rabson ef->chains = ef->buckets + ef->nbuckets; 582a4f67738SDoug Rabson break; 583a4f67738SDoug Rabson } 584a4f67738SDoug Rabson case DT_STRTAB: 585fe3db7c7SDoug Rabson ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr); 586a4f67738SDoug Rabson break; 5872d636ab0SPeter Wemm case DT_STRSZ: 5882d636ab0SPeter Wemm ef->strsz = dp->d_un.d_val; 5892d636ab0SPeter Wemm break; 590a4f67738SDoug Rabson case DT_SYMTAB: 591fe3db7c7SDoug Rabson ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr); 592a4f67738SDoug Rabson break; 593a4f67738SDoug Rabson case DT_SYMENT: 594a4f67738SDoug Rabson if (dp->d_un.d_val != sizeof(Elf_Sym)) 5952f22b3ffSMatthew D Fleming return (ENOEXEC); 596fe3db7c7SDoug Rabson break; 597fe3db7c7SDoug Rabson case DT_PLTGOT: 598fe3db7c7SDoug Rabson ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr); 599fe3db7c7SDoug Rabson break; 600fe3db7c7SDoug Rabson case DT_REL: 601fe3db7c7SDoug Rabson ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 602fe3db7c7SDoug Rabson break; 603fe3db7c7SDoug Rabson case DT_RELSZ: 604fe3db7c7SDoug Rabson ef->relsize = dp->d_un.d_val; 605fe3db7c7SDoug Rabson break; 606fe3db7c7SDoug Rabson case DT_RELENT: 607fe3db7c7SDoug Rabson if (dp->d_un.d_val != sizeof(Elf_Rel)) 6082f22b3ffSMatthew D Fleming return (ENOEXEC); 609fe3db7c7SDoug Rabson break; 610fe3db7c7SDoug Rabson case DT_JMPREL: 611fe3db7c7SDoug Rabson ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr); 612fe3db7c7SDoug Rabson break; 613fe3db7c7SDoug Rabson case DT_PLTRELSZ: 614fe3db7c7SDoug Rabson ef->pltrelsize = dp->d_un.d_val; 615fe3db7c7SDoug Rabson break; 616fe3db7c7SDoug Rabson case DT_RELA: 617fe3db7c7SDoug Rabson ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr); 618fe3db7c7SDoug Rabson break; 619fe3db7c7SDoug Rabson case DT_RELASZ: 620fe3db7c7SDoug Rabson ef->relasize = dp->d_un.d_val; 621fe3db7c7SDoug Rabson break; 622fe3db7c7SDoug Rabson case DT_RELAENT: 623fe3db7c7SDoug Rabson if (dp->d_un.d_val != sizeof(Elf_Rela)) 6242f22b3ffSMatthew D Fleming return (ENOEXEC); 625fe3db7c7SDoug Rabson break; 626fe3db7c7SDoug Rabson case DT_PLTREL: 627fe3db7c7SDoug Rabson plttype = dp->d_un.d_val; 628fe3db7c7SDoug Rabson if (plttype != DT_REL && plttype != DT_RELA) 6292f22b3ffSMatthew D Fleming return (ENOEXEC); 630fe3db7c7SDoug Rabson break; 631ecb01c64SMarcel Moolenaar #ifdef GDB 632326e27d8SDoug Rabson case DT_DEBUG: 633326e27d8SDoug Rabson dp->d_un.d_ptr = (Elf_Addr)&r_debug; 634326e27d8SDoug Rabson break; 635326e27d8SDoug Rabson #endif 636a4f67738SDoug Rabson } 637a4f67738SDoug Rabson } 638fe3db7c7SDoug Rabson 639fe3db7c7SDoug Rabson if (plttype == DT_RELA) { 640fe3db7c7SDoug Rabson ef->pltrela = (const Elf_Rela *)ef->pltrel; 641fe3db7c7SDoug Rabson ef->pltrel = NULL; 642fe3db7c7SDoug Rabson ef->pltrelasize = ef->pltrelsize; 643fe3db7c7SDoug Rabson ef->pltrelsize = 0; 644fe3db7c7SDoug Rabson } 645fe3db7c7SDoug Rabson 6462d636ab0SPeter Wemm ef->ddbsymtab = ef->symtab; 6472d636ab0SPeter Wemm ef->ddbsymcnt = ef->nchains; 6482d636ab0SPeter Wemm ef->ddbstrtab = ef->strtab; 6492d636ab0SPeter Wemm ef->ddbstrcnt = ef->strsz; 6502d636ab0SPeter Wemm 65130693808SBrandon Bergren return elf_cpu_parse_dynamic(ef->address, ef->dynamic); 652a4f67738SDoug Rabson } 653a4f67738SDoug Rabson 6544c62bffeSBjoern A. Zeeb #define LS_PADDING 0x90909090 655a4f67738SDoug Rabson static int 65650c202c5SJeff Roberson parse_dpcpu(elf_file_t ef) 65750c202c5SJeff Roberson { 6589ae7bc39SBjoern A. Zeeb int error, size; 6594c62bffeSBjoern A. Zeeb #if defined(__i386__) 6604c62bffeSBjoern A. Zeeb uint32_t pad; 6614c62bffeSBjoern A. Zeeb #endif 66250c202c5SJeff Roberson 66350c202c5SJeff Roberson ef->pcpu_start = 0; 66450c202c5SJeff Roberson ef->pcpu_stop = 0; 66550c202c5SJeff Roberson error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start, 6669ae7bc39SBjoern A. Zeeb (void ***)&ef->pcpu_stop, NULL); 66750c202c5SJeff Roberson /* Error just means there is no pcpu set to relocate. */ 6682f22b3ffSMatthew D Fleming if (error != 0) 66950c202c5SJeff Roberson return (0); 6709ae7bc39SBjoern A. Zeeb size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start; 6719ae7bc39SBjoern A. Zeeb /* Empty set? */ 6729ae7bc39SBjoern A. Zeeb if (size < 1) 6739ae7bc39SBjoern A. Zeeb return (0); 6744c62bffeSBjoern A. Zeeb #if defined(__i386__) 6754c62bffeSBjoern A. Zeeb /* In case we do find __start/stop_set_ symbols double-check. */ 6764c62bffeSBjoern A. Zeeb if (size < 4) { 6774c62bffeSBjoern A. Zeeb uprintf("Kernel module '%s' must be recompiled with " 6784c62bffeSBjoern A. Zeeb "linker script\n", ef->lf.pathname); 6794c62bffeSBjoern A. Zeeb return (ENOEXEC); 6804c62bffeSBjoern A. Zeeb } 6814c62bffeSBjoern A. Zeeb 6824c62bffeSBjoern A. Zeeb /* Padding from linker-script correct? */ 6834c62bffeSBjoern A. Zeeb pad = *(uint32_t *)((uintptr_t)ef->pcpu_stop - sizeof(pad)); 6844c62bffeSBjoern A. Zeeb if (pad != LS_PADDING) { 6854c62bffeSBjoern A. Zeeb uprintf("Kernel module '%s' must be recompiled with " 6864c62bffeSBjoern A. Zeeb "linker script, invalid padding %#04x (%#04x)\n", 6874c62bffeSBjoern A. Zeeb ef->lf.pathname, pad, LS_PADDING); 6884c62bffeSBjoern A. Zeeb return (ENOEXEC); 6894c62bffeSBjoern A. Zeeb } 6904c62bffeSBjoern A. Zeeb /* If we only have valid padding, nothing to do. */ 6914c62bffeSBjoern A. Zeeb if (size == 4) 6924c62bffeSBjoern A. Zeeb return (0); 6934c62bffeSBjoern A. Zeeb #endif 69450c202c5SJeff Roberson /* 6952f22b3ffSMatthew D Fleming * Allocate space in the primary pcpu area. Copy in our 6962f22b3ffSMatthew D Fleming * initialization from the data section and then initialize 6972f22b3ffSMatthew D Fleming * all per-cpu storage from that. 69850c202c5SJeff Roberson */ 6999ae7bc39SBjoern A. Zeeb ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size); 7000f823b64SBjoern A. Zeeb if (ef->pcpu_base == 0) { 7010f823b64SBjoern A. Zeeb printf("%s: pcpu module space is out of space; " 7020f823b64SBjoern A. Zeeb "cannot allocate %d for %s\n", 7030f823b64SBjoern A. Zeeb __func__, size, ef->lf.pathname); 70450c202c5SJeff Roberson return (ENOSPC); 7050f823b64SBjoern A. Zeeb } 7069ae7bc39SBjoern A. Zeeb memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size); 7079ae7bc39SBjoern A. Zeeb dpcpu_copy((void *)ef->pcpu_base, size); 70847813f5dSMikolaj Golub elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop, 70947813f5dSMikolaj Golub ef->pcpu_base); 71050c202c5SJeff Roberson 71150c202c5SJeff Roberson return (0); 71250c202c5SJeff Roberson } 71350c202c5SJeff Roberson 714eddfbb76SRobert Watson #ifdef VIMAGE 715eddfbb76SRobert Watson static int 716eddfbb76SRobert Watson parse_vnet(elf_file_t ef) 717eddfbb76SRobert Watson { 7189ae7bc39SBjoern A. Zeeb int error, size; 7194c62bffeSBjoern A. Zeeb #if defined(__i386__) 7204c62bffeSBjoern A. Zeeb uint32_t pad; 7214c62bffeSBjoern A. Zeeb #endif 722eddfbb76SRobert Watson 723eddfbb76SRobert Watson ef->vnet_start = 0; 724eddfbb76SRobert Watson ef->vnet_stop = 0; 725eddfbb76SRobert Watson error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start, 7269ae7bc39SBjoern A. Zeeb (void ***)&ef->vnet_stop, NULL); 727eddfbb76SRobert Watson /* Error just means there is no vnet data set to relocate. */ 7282f22b3ffSMatthew D Fleming if (error != 0) 729eddfbb76SRobert Watson return (0); 7309ae7bc39SBjoern A. Zeeb size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start; 7319ae7bc39SBjoern A. Zeeb /* Empty set? */ 7329ae7bc39SBjoern A. Zeeb if (size < 1) 7339ae7bc39SBjoern A. Zeeb return (0); 7344c62bffeSBjoern A. Zeeb #if defined(__i386__) 7354c62bffeSBjoern A. Zeeb /* In case we do find __start/stop_set_ symbols double-check. */ 7364c62bffeSBjoern A. Zeeb if (size < 4) { 7374c62bffeSBjoern A. Zeeb uprintf("Kernel module '%s' must be recompiled with " 7384c62bffeSBjoern A. Zeeb "linker script\n", ef->lf.pathname); 7394c62bffeSBjoern A. Zeeb return (ENOEXEC); 7404c62bffeSBjoern A. Zeeb } 7414c62bffeSBjoern A. Zeeb 7424c62bffeSBjoern A. Zeeb /* Padding from linker-script correct? */ 7434c62bffeSBjoern A. Zeeb pad = *(uint32_t *)((uintptr_t)ef->vnet_stop - sizeof(pad)); 7444c62bffeSBjoern A. Zeeb if (pad != LS_PADDING) { 7454c62bffeSBjoern A. Zeeb uprintf("Kernel module '%s' must be recompiled with " 7464c62bffeSBjoern A. Zeeb "linker script, invalid padding %#04x (%#04x)\n", 7474c62bffeSBjoern A. Zeeb ef->lf.pathname, pad, LS_PADDING); 7484c62bffeSBjoern A. Zeeb return (ENOEXEC); 7494c62bffeSBjoern A. Zeeb } 7504c62bffeSBjoern A. Zeeb /* If we only have valid padding, nothing to do. */ 7514c62bffeSBjoern A. Zeeb if (size == 4) 7524c62bffeSBjoern A. Zeeb return (0); 7534c62bffeSBjoern A. Zeeb #endif 754eddfbb76SRobert Watson /* 7552f22b3ffSMatthew D Fleming * Allocate space in the primary vnet area. Copy in our 7562f22b3ffSMatthew D Fleming * initialization from the data section and then initialize 7572f22b3ffSMatthew D Fleming * all per-vnet storage from that. 758eddfbb76SRobert Watson */ 7599ae7bc39SBjoern A. Zeeb ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size); 7600f823b64SBjoern A. Zeeb if (ef->vnet_base == 0) { 7610f823b64SBjoern A. Zeeb printf("%s: vnet module space is out of space; " 7620f823b64SBjoern A. Zeeb "cannot allocate %d for %s\n", 7630f823b64SBjoern A. Zeeb __func__, size, ef->lf.pathname); 764eddfbb76SRobert Watson return (ENOSPC); 7650f823b64SBjoern A. Zeeb } 7669ae7bc39SBjoern A. Zeeb memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size); 7679ae7bc39SBjoern A. Zeeb vnet_data_copy((void *)ef->vnet_base, size); 76847813f5dSMikolaj Golub elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop, 76947813f5dSMikolaj Golub ef->vnet_base); 770eddfbb76SRobert Watson 771eddfbb76SRobert Watson return (0); 772eddfbb76SRobert Watson } 773eddfbb76SRobert Watson #endif 7744c62bffeSBjoern A. Zeeb #undef LS_PADDING 775eddfbb76SRobert Watson 776f822c9e2SMark Johnston /* 777f822c9e2SMark Johnston * Apply the specified protection to the loadable segments of a preloaded linker 778f822c9e2SMark Johnston * file. 779f822c9e2SMark Johnston */ 78050c202c5SJeff Roberson static int 781f822c9e2SMark Johnston preload_protect(elf_file_t ef, vm_prot_t prot) 782f822c9e2SMark Johnston { 783a85ce4adSAndrew Turner #if defined(__aarch64__) || defined(__amd64__) 784f822c9e2SMark Johnston Elf_Ehdr *hdr; 785f822c9e2SMark Johnston Elf_Phdr *phdr, *phlimit; 786f822c9e2SMark Johnston vm_prot_t nprot; 787f822c9e2SMark Johnston int error; 788f822c9e2SMark Johnston 789f822c9e2SMark Johnston error = 0; 790f822c9e2SMark Johnston hdr = (Elf_Ehdr *)ef->address; 791f822c9e2SMark Johnston phdr = (Elf_Phdr *)(ef->address + hdr->e_phoff); 792f822c9e2SMark Johnston phlimit = phdr + hdr->e_phnum; 793f822c9e2SMark Johnston for (; phdr < phlimit; phdr++) { 794f822c9e2SMark Johnston if (phdr->p_type != PT_LOAD) 795f822c9e2SMark Johnston continue; 796f822c9e2SMark Johnston 797f822c9e2SMark Johnston nprot = prot | VM_PROT_READ; 798f822c9e2SMark Johnston if ((phdr->p_flags & PF_W) != 0) 799f822c9e2SMark Johnston nprot |= VM_PROT_WRITE; 800f822c9e2SMark Johnston if ((phdr->p_flags & PF_X) != 0) 801f822c9e2SMark Johnston nprot |= VM_PROT_EXECUTE; 802f822c9e2SMark Johnston error = pmap_change_prot((vm_offset_t)ef->address + 803f822c9e2SMark Johnston phdr->p_vaddr, round_page(phdr->p_memsz), nprot); 804f822c9e2SMark Johnston if (error != 0) 805f822c9e2SMark Johnston break; 806f822c9e2SMark Johnston } 807f822c9e2SMark Johnston return (error); 808f822c9e2SMark Johnston #else 809f822c9e2SMark Johnston return (0); 810f822c9e2SMark Johnston #endif 811f822c9e2SMark Johnston } 812f822c9e2SMark Johnston 813b19c9deaSIan Lepore #ifdef __arm__ 814b19c9deaSIan Lepore /* 815b19c9deaSIan Lepore * Locate the ARM exception/unwind table info for DDB and stack(9) use by 816b19c9deaSIan Lepore * searching for the section header that describes it. There may be no unwind 817b19c9deaSIan Lepore * info, for example in a module containing only data. 818b19c9deaSIan Lepore */ 819b19c9deaSIan Lepore static void 820b19c9deaSIan Lepore link_elf_locate_exidx(linker_file_t lf, Elf_Shdr *shdr, int nhdr) 821b19c9deaSIan Lepore { 822b19c9deaSIan Lepore int i; 823b19c9deaSIan Lepore 824b19c9deaSIan Lepore for (i = 0; i < nhdr; i++) { 825b19c9deaSIan Lepore if (shdr[i].sh_type == SHT_ARM_EXIDX) { 826b19c9deaSIan Lepore lf->exidx_addr = shdr[i].sh_addr + lf->address; 827b19c9deaSIan Lepore lf->exidx_size = shdr[i].sh_size; 828b19c9deaSIan Lepore break; 829b19c9deaSIan Lepore } 830b19c9deaSIan Lepore } 831b19c9deaSIan Lepore } 832b19c9deaSIan Lepore 833b19c9deaSIan Lepore /* 834b19c9deaSIan Lepore * Locate the section headers metadata in a preloaded module, then use it to 835b19c9deaSIan Lepore * locate the exception/unwind table in the module. The size of the metadata 836b19c9deaSIan Lepore * block is stored in a uint32 word immediately before the data itself, and a 837b19c9deaSIan Lepore * comment in preload_search_info() says it is safe to rely on that. 838b19c9deaSIan Lepore */ 839b19c9deaSIan Lepore static void 840b19c9deaSIan Lepore link_elf_locate_exidx_preload(struct linker_file *lf, caddr_t modptr) 841b19c9deaSIan Lepore { 842b19c9deaSIan Lepore uint32_t *modinfo; 843b19c9deaSIan Lepore Elf_Shdr *shdr; 844b19c9deaSIan Lepore uint32_t nhdr; 845b19c9deaSIan Lepore 846b19c9deaSIan Lepore modinfo = (uint32_t *)preload_search_info(modptr, 847b19c9deaSIan Lepore MODINFO_METADATA | MODINFOMD_SHDR); 848b19c9deaSIan Lepore if (modinfo != NULL) { 849b19c9deaSIan Lepore shdr = (Elf_Shdr *)modinfo; 850b19c9deaSIan Lepore nhdr = modinfo[-1] / sizeof(Elf_Shdr); 851b19c9deaSIan Lepore link_elf_locate_exidx(lf, shdr, nhdr); 852b19c9deaSIan Lepore } 853b19c9deaSIan Lepore } 854b19c9deaSIan Lepore 855b19c9deaSIan Lepore #endif /* __arm__ */ 856b19c9deaSIan Lepore 857f822c9e2SMark Johnston static int 858f822c9e2SMark Johnston link_elf_link_preload(linker_class_t cls, const char *filename, 859f822c9e2SMark Johnston linker_file_t *result) 860de78ca7eSPeter Wemm { 8610067051fSMarcel Moolenaar Elf_Addr *ctors_addrp; 8620067051fSMarcel Moolenaar Elf_Size *ctors_sizep; 863de78ca7eSPeter Wemm caddr_t modptr, baseptr, sizeptr, dynptr; 864de78ca7eSPeter Wemm char *type; 865de78ca7eSPeter Wemm elf_file_t ef; 866de78ca7eSPeter Wemm linker_file_t lf; 867de78ca7eSPeter Wemm int error; 868de78ca7eSPeter Wemm vm_offset_t dp; 869de78ca7eSPeter Wemm 87054823af2SPeter Wemm /* Look to see if we have the file preloaded */ 871de78ca7eSPeter Wemm modptr = preload_search_by_name(filename); 872de78ca7eSPeter Wemm if (modptr == NULL) 8732f22b3ffSMatthew D Fleming return (ENOENT); 874de78ca7eSPeter Wemm 875de78ca7eSPeter Wemm type = (char *)preload_search_info(modptr, MODINFO_TYPE); 876de78ca7eSPeter Wemm baseptr = preload_search_info(modptr, MODINFO_ADDR); 877de78ca7eSPeter Wemm sizeptr = preload_search_info(modptr, MODINFO_SIZE); 8782f22b3ffSMatthew D Fleming dynptr = preload_search_info(modptr, 8792f22b3ffSMatthew D Fleming MODINFO_METADATA | MODINFOMD_DYNAMIC); 88067db8b23SPeter Wemm if (type == NULL || 88167db8b23SPeter Wemm (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 && 88267db8b23SPeter Wemm strcmp(type, "elf module") != 0)) 883de78ca7eSPeter Wemm return (EFTYPE); 884de78ca7eSPeter Wemm if (baseptr == NULL || sizeptr == NULL || dynptr == NULL) 885de78ca7eSPeter Wemm return (EINVAL); 886de78ca7eSPeter Wemm 887326e27d8SDoug Rabson lf = linker_make_file(filename, &link_elf_class); 8882f22b3ffSMatthew D Fleming if (lf == NULL) 8892f22b3ffSMatthew D Fleming return (ENOMEM); 890326e27d8SDoug Rabson 891326e27d8SDoug Rabson ef = (elf_file_t) lf; 892326e27d8SDoug Rabson ef->preloaded = 1; 893ca65d5c7SPeter Wemm ef->modptr = modptr; 894de78ca7eSPeter Wemm ef->address = *(caddr_t *)baseptr; 895de78ca7eSPeter Wemm #ifdef SPARSE_MAPPING 8961aa696baSMark Johnston ef->object = NULL; 897de78ca7eSPeter Wemm #endif 898de78ca7eSPeter Wemm dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr; 899de78ca7eSPeter Wemm ef->dynamic = (Elf_Dyn *)dp; 900de78ca7eSPeter Wemm lf->address = ef->address; 901de78ca7eSPeter Wemm lf->size = *(size_t *)sizeptr; 902de78ca7eSPeter Wemm 9030067051fSMarcel Moolenaar ctors_addrp = (Elf_Addr *)preload_search_info(modptr, 9040067051fSMarcel Moolenaar MODINFO_METADATA | MODINFOMD_CTORS_ADDR); 9050067051fSMarcel Moolenaar ctors_sizep = (Elf_Size *)preload_search_info(modptr, 9060067051fSMarcel Moolenaar MODINFO_METADATA | MODINFOMD_CTORS_SIZE); 9070067051fSMarcel Moolenaar if (ctors_addrp != NULL && ctors_sizep != NULL) { 9080067051fSMarcel Moolenaar lf->ctors_addr = ef->address + *ctors_addrp; 9090067051fSMarcel Moolenaar lf->ctors_size = *ctors_sizep; 9100067051fSMarcel Moolenaar } 9110067051fSMarcel Moolenaar 912b19c9deaSIan Lepore #ifdef __arm__ 913b19c9deaSIan Lepore link_elf_locate_exidx_preload(lf, modptr); 914b19c9deaSIan Lepore #endif 915b19c9deaSIan Lepore 916326e27d8SDoug Rabson error = parse_dynamic(ef); 91750c202c5SJeff Roberson if (error == 0) 91850c202c5SJeff Roberson error = parse_dpcpu(ef); 919eddfbb76SRobert Watson #ifdef VIMAGE 920eddfbb76SRobert Watson if (error == 0) 921eddfbb76SRobert Watson error = parse_vnet(ef); 922eddfbb76SRobert Watson #endif 923f822c9e2SMark Johnston if (error == 0) 924f822c9e2SMark Johnston error = preload_protect(ef, VM_PROT_ALL); 9252f22b3ffSMatthew D Fleming if (error != 0) { 92665a311fcSPoul-Henning Kamp linker_file_unload(lf, LINKER_UNLOAD_FORCE); 9272f22b3ffSMatthew D Fleming return (error); 928de78ca7eSPeter Wemm } 9297251b4bfSJake Burkholder link_elf_reloc_local(lf); 930de78ca7eSPeter Wemm *result = lf; 931de78ca7eSPeter Wemm return (0); 932de78ca7eSPeter Wemm } 933de78ca7eSPeter Wemm 934de78ca7eSPeter Wemm static int 93554823af2SPeter Wemm link_elf_link_preload_finish(linker_file_t lf) 93654823af2SPeter Wemm { 93754823af2SPeter Wemm elf_file_t ef; 93854823af2SPeter Wemm int error; 93954823af2SPeter Wemm 94054823af2SPeter Wemm ef = (elf_file_t) lf; 94154823af2SPeter Wemm error = relocate_file(ef); 942f822c9e2SMark Johnston if (error == 0) 943f822c9e2SMark Johnston error = preload_protect(ef, VM_PROT_NONE); 9442f22b3ffSMatthew D Fleming if (error != 0) 9452f22b3ffSMatthew D Fleming return (error); 94654823af2SPeter Wemm (void)link_elf_preload_parse_symbols(ef); 94754823af2SPeter Wemm 948c143d6c2SMarcel Moolenaar return (link_elf_link_common_finish(lf)); 94954823af2SPeter Wemm } 95054823af2SPeter Wemm 95154823af2SPeter Wemm static int 952c143d6c2SMarcel Moolenaar link_elf_load_file(linker_class_t cls, const char* filename, 953c143d6c2SMarcel Moolenaar linker_file_t* result) 954a4f67738SDoug Rabson { 955a4f67738SDoug Rabson struct nameidata nd; 956b40ce416SJulian Elischer struct thread* td = curthread; /* XXX */ 957caab6e90SPeter Wemm Elf_Ehdr *hdr; 9581aa696baSMark Johnston caddr_t firstpage, segbase; 959fe3db7c7SDoug Rabson int nbytes, i; 960fe3db7c7SDoug Rabson Elf_Phdr *phdr; 961fe3db7c7SDoug Rabson Elf_Phdr *phlimit; 9621a330eb0SPeter Wemm Elf_Phdr *segs[MAXSEGS]; 963fe3db7c7SDoug Rabson int nsegs; 964fe3db7c7SDoug Rabson Elf_Phdr *phdyn; 965fe3db7c7SDoug Rabson caddr_t mapbase; 966fe3db7c7SDoug Rabson size_t mapsize; 967fe3db7c7SDoug Rabson Elf_Addr base_vaddr; 968fe3db7c7SDoug Rabson Elf_Addr base_vlimit; 969a4f67738SDoug Rabson int error = 0; 970526d0bd5SKonstantin Belousov ssize_t resid; 971526d0bd5SKonstantin Belousov int flags; 972a4f67738SDoug Rabson elf_file_t ef; 973a4f67738SDoug Rabson linker_file_t lf; 974ca65d5c7SPeter Wemm Elf_Shdr *shdr; 975ca65d5c7SPeter Wemm int symtabindex; 976ca65d5c7SPeter Wemm int symstrindex; 9770067051fSMarcel Moolenaar int shstrindex; 978ca65d5c7SPeter Wemm int symcnt; 979ca65d5c7SPeter Wemm int strcnt; 9800067051fSMarcel Moolenaar char *shstrs; 981ca65d5c7SPeter Wemm 982ca65d5c7SPeter Wemm shdr = NULL; 983ca65d5c7SPeter Wemm lf = NULL; 9840067051fSMarcel Moolenaar shstrs = NULL; 985a4f67738SDoug Rabson 9867e1d3eefSMateusz Guzik NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename); 987e6796b67SKirk McKusick flags = FREAD; 9889e223287SKonstantin Belousov error = vn_open(&nd, &flags, 0, NULL); 9892f22b3ffSMatthew D Fleming if (error != 0) 9902f22b3ffSMatthew D Fleming return (error); 991762e6b85SEivind Eklund NDFREE(&nd, NDF_ONLY_PNBUF); 9921f49b573SRuslan Ermilov if (nd.ni_vp->v_type != VREG) { 9931f49b573SRuslan Ermilov error = ENOEXEC; 9941f49b573SRuslan Ermilov firstpage = NULL; 9951f49b573SRuslan Ermilov goto out; 9961f49b573SRuslan Ermilov } 997a3df768bSRobert Watson #ifdef MAC 99830d239bcSRobert Watson error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp); 9992f22b3ffSMatthew D Fleming if (error != 0) { 1000a3df768bSRobert Watson firstpage = NULL; 1001a3df768bSRobert Watson goto out; 1002a3df768bSRobert Watson } 1003a3df768bSRobert Watson #endif 1004a4f67738SDoug Rabson 1005a4f67738SDoug Rabson /* 1006fe3db7c7SDoug Rabson * Read the elf header from the file. 1007a4f67738SDoug Rabson */ 1008a163d034SWarner Losh firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK); 1009caab6e90SPeter Wemm hdr = (Elf_Ehdr *)firstpage; 1010caab6e90SPeter Wemm error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0, 10119ca43589SRobert Watson UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 10129ca43589SRobert Watson &resid, td); 1013caab6e90SPeter Wemm nbytes = PAGE_SIZE - resid; 10142f22b3ffSMatthew D Fleming if (error != 0) 1015a4f67738SDoug Rabson goto out; 1016a4f67738SDoug Rabson 1017caab6e90SPeter Wemm if (!IS_ELF(*hdr)) { 1018fe3db7c7SDoug Rabson error = ENOEXEC; 1019a4f67738SDoug Rabson goto out; 1020fe3db7c7SDoug Rabson } 1021fe3db7c7SDoug Rabson 10222f22b3ffSMatthew D Fleming if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || 10232f22b3ffSMatthew D Fleming hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { 1024552f9f63SEdwin Groothuis link_elf_error(filename, "Unsupported file layout"); 1025fe3db7c7SDoug Rabson error = ENOEXEC; 1026fe3db7c7SDoug Rabson goto out; 1027fe3db7c7SDoug Rabson } 10282f22b3ffSMatthew D Fleming if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 10292f22b3ffSMatthew D Fleming hdr->e_version != EV_CURRENT) { 1030552f9f63SEdwin Groothuis link_elf_error(filename, "Unsupported file version"); 1031fe3db7c7SDoug Rabson error = ENOEXEC; 1032fe3db7c7SDoug Rabson goto out; 1033fe3db7c7SDoug Rabson } 1034caab6e90SPeter Wemm if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) { 1035a1d7ce03SAttilio Rao error = ENOSYS; 1036fe3db7c7SDoug Rabson goto out; 1037fe3db7c7SDoug Rabson } 1038caab6e90SPeter Wemm if (hdr->e_machine != ELF_TARG_MACH) { 1039552f9f63SEdwin Groothuis link_elf_error(filename, "Unsupported machine"); 1040fe3db7c7SDoug Rabson error = ENOEXEC; 1041fe3db7c7SDoug Rabson goto out; 1042fe3db7c7SDoug Rabson } 1043a4f67738SDoug Rabson 1044a4f67738SDoug Rabson /* 10452f22b3ffSMatthew D Fleming * We rely on the program header being in the first page. 10462f22b3ffSMatthew D Fleming * This is not strictly required by the ABI specification, but 10472f22b3ffSMatthew D Fleming * it seems to always true in practice. And, it simplifies 10482f22b3ffSMatthew D Fleming * things considerably. 1049a4f67738SDoug Rabson */ 1050caab6e90SPeter Wemm if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) && 1051caab6e90SPeter Wemm (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) && 1052caab6e90SPeter Wemm (hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes))) 1053552f9f63SEdwin Groothuis link_elf_error(filename, "Unreadable program headers"); 1054fe3db7c7SDoug Rabson 1055fe3db7c7SDoug Rabson /* 1056fe3db7c7SDoug Rabson * Scan the program header entries, and save key information. 1057fe3db7c7SDoug Rabson * 1058fe3db7c7SDoug Rabson * We rely on there being exactly two load segments, text and data, 1059fe3db7c7SDoug Rabson * in that order. 1060fe3db7c7SDoug Rabson */ 1061caab6e90SPeter Wemm phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff); 1062caab6e90SPeter Wemm phlimit = phdr + hdr->e_phnum; 1063fe3db7c7SDoug Rabson nsegs = 0; 1064fe3db7c7SDoug Rabson phdyn = NULL; 1065fe3db7c7SDoug Rabson while (phdr < phlimit) { 1066fe3db7c7SDoug Rabson switch (phdr->p_type) { 1067fe3db7c7SDoug Rabson case PT_LOAD: 10681a330eb0SPeter Wemm if (nsegs == MAXSEGS) { 1069552f9f63SEdwin Groothuis link_elf_error(filename, "Too many sections"); 1070fe3db7c7SDoug Rabson error = ENOEXEC; 1071fe3db7c7SDoug Rabson goto out; 1072fe3db7c7SDoug Rabson } 107387b1831fSPoul-Henning Kamp /* 107487b1831fSPoul-Henning Kamp * XXX: We just trust they come in right order ?? 107587b1831fSPoul-Henning Kamp */ 1076fe3db7c7SDoug Rabson segs[nsegs] = phdr; 1077fe3db7c7SDoug Rabson ++nsegs; 1078fe3db7c7SDoug Rabson break; 1079fe3db7c7SDoug Rabson 1080fe3db7c7SDoug Rabson case PT_DYNAMIC: 1081fe3db7c7SDoug Rabson phdyn = phdr; 1082fe3db7c7SDoug Rabson break; 10839548091bSBoris Popov 10849548091bSBoris Popov case PT_INTERP: 1085a1d7ce03SAttilio Rao error = ENOSYS; 10869548091bSBoris Popov goto out; 1087fe3db7c7SDoug Rabson } 1088fe3db7c7SDoug Rabson 1089fe3db7c7SDoug Rabson ++phdr; 1090fe3db7c7SDoug Rabson } 1091fe3db7c7SDoug Rabson if (phdyn == NULL) { 1092552f9f63SEdwin Groothuis link_elf_error(filename, "Object is not dynamically-linked"); 1093fe3db7c7SDoug Rabson error = ENOEXEC; 1094fe3db7c7SDoug Rabson goto out; 1095fe3db7c7SDoug Rabson } 10961a330eb0SPeter Wemm if (nsegs == 0) { 1097552f9f63SEdwin Groothuis link_elf_error(filename, "No sections"); 109887b1831fSPoul-Henning Kamp error = ENOEXEC; 109987b1831fSPoul-Henning Kamp goto out; 110087b1831fSPoul-Henning Kamp } 1101fe3db7c7SDoug Rabson 1102fe3db7c7SDoug Rabson /* 11032f22b3ffSMatthew D Fleming * Allocate the entire address space of the object, to stake 11042f22b3ffSMatthew D Fleming * out our contiguous region, and to establish the base 11052f22b3ffSMatthew D Fleming * address for relocation. 1106fe3db7c7SDoug Rabson */ 1107fe3db7c7SDoug Rabson base_vaddr = trunc_page(segs[0]->p_vaddr); 11081a330eb0SPeter Wemm base_vlimit = round_page(segs[nsegs - 1]->p_vaddr + 11091a330eb0SPeter Wemm segs[nsegs - 1]->p_memsz); 1110fe3db7c7SDoug Rabson mapsize = base_vlimit - base_vaddr; 1111fe3db7c7SDoug Rabson 1112326e27d8SDoug Rabson lf = linker_make_file(filename, &link_elf_class); 11132f22b3ffSMatthew D Fleming if (lf == NULL) { 1114326e27d8SDoug Rabson error = ENOMEM; 1115326e27d8SDoug Rabson goto out; 1116326e27d8SDoug Rabson } 1117326e27d8SDoug Rabson 1118326e27d8SDoug Rabson ef = (elf_file_t) lf; 1119fe3db7c7SDoug Rabson #ifdef SPARSE_MAPPING 1120fbf2a778SKonstantin Belousov ef->object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL, 1121fbf2a778SKonstantin Belousov 0, thread0.td_ucred); 1122fe3db7c7SDoug Rabson if (ef->object == NULL) { 1123fe3db7c7SDoug Rabson error = ENOMEM; 1124fe3db7c7SDoug Rabson goto out; 1125fe3db7c7SDoug Rabson } 11261aa696baSMark Johnston #ifdef __amd64__ 11271aa696baSMark Johnston mapbase = (caddr_t)KERNBASE; 11281aa696baSMark Johnston #else 11291aa696baSMark Johnston mapbase = (caddr_t)vm_map_min(kernel_map); 11301aa696baSMark Johnston #endif 11311aa696baSMark Johnston /* 11321aa696baSMark Johnston * Mapping protections are downgraded after relocation processing. 11331aa696baSMark Johnston */ 1134fe3db7c7SDoug Rabson error = vm_map_find(kernel_map, ef->object, 0, 11351aa696baSMark Johnston (vm_offset_t *)&mapbase, mapsize, 0, VMFS_OPTIMAL_SPACE, 1136fe3db7c7SDoug Rabson VM_PROT_ALL, VM_PROT_ALL, 0); 11372f22b3ffSMatthew D Fleming if (error != 0) { 1138fe3db7c7SDoug Rabson vm_object_deallocate(ef->object); 11391aa696baSMark Johnston ef->object = NULL; 1140fe3db7c7SDoug Rabson goto out; 1141fe3db7c7SDoug Rabson } 1142fe3db7c7SDoug Rabson #else 114382c174a3SMateusz Guzik mapbase = malloc_exec(mapsize, M_LINKER, M_WAITOK); 1144fe3db7c7SDoug Rabson #endif 11451aa696baSMark Johnston ef->address = mapbase; 1146a4f67738SDoug Rabson 1147a4f67738SDoug Rabson /* 1148a4f67738SDoug Rabson * Read the text and data sections and zero the bss. 1149a4f67738SDoug Rabson */ 11501a330eb0SPeter Wemm for (i = 0; i < nsegs; i++) { 11511aa696baSMark Johnston segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 11521aa696baSMark Johnston 11531aa696baSMark Johnston #ifdef SPARSE_MAPPING 11541aa696baSMark Johnston /* 11551aa696baSMark Johnston * Consecutive segments may have different mapping permissions, 11561aa696baSMark Johnston * so be strict and verify that their mappings do not overlap. 11571aa696baSMark Johnston */ 11581aa696baSMark Johnston if (((vm_offset_t)segbase & PAGE_MASK) != 0) { 11591aa696baSMark Johnston error = EINVAL; 11601aa696baSMark Johnston goto out; 11611aa696baSMark Johnston } 11621aa696baSMark Johnston 11631aa696baSMark Johnston error = vm_map_wire(kernel_map, 11641aa696baSMark Johnston (vm_offset_t)segbase, 11651aa696baSMark Johnston (vm_offset_t)segbase + round_page(segs[i]->p_memsz), 11661aa696baSMark Johnston VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 11671aa696baSMark Johnston if (error != KERN_SUCCESS) { 11681aa696baSMark Johnston error = ENOMEM; 11691aa696baSMark Johnston goto out; 11701aa696baSMark Johnston } 11711aa696baSMark Johnston #endif 11721aa696baSMark Johnston 1173fe3db7c7SDoug Rabson error = vn_rdwr(UIO_READ, nd.ni_vp, 1174fe3db7c7SDoug Rabson segbase, segs[i]->p_filesz, segs[i]->p_offset, 11759ca43589SRobert Watson UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 11769ca43589SRobert Watson &resid, td); 11772f22b3ffSMatthew D Fleming if (error != 0) 1178a4f67738SDoug Rabson goto out; 1179fe3db7c7SDoug Rabson bzero(segbase + segs[i]->p_filesz, 1180fe3db7c7SDoug Rabson segs[i]->p_memsz - segs[i]->p_filesz); 1181fe3db7c7SDoug Rabson } 1182fe3db7c7SDoug Rabson 1183f99502a4SBrian Feldman #ifdef GPROF 1184f99502a4SBrian Feldman /* Update profiling information with the new text segment. */ 118570f37788SJohn Baldwin mtx_lock(&Giant); 1186853a17adSEnji Cooper kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr + 1187f99502a4SBrian Feldman segs[0]->p_memsz)); 118870f37788SJohn Baldwin mtx_unlock(&Giant); 1189f99502a4SBrian Feldman #endif 1190f99502a4SBrian Feldman 1191326e27d8SDoug Rabson ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr); 1192a4f67738SDoug Rabson 1193a4f67738SDoug Rabson lf->address = ef->address; 1194fe3db7c7SDoug Rabson lf->size = mapsize; 1195a4f67738SDoug Rabson 1196326e27d8SDoug Rabson error = parse_dynamic(ef); 11972f22b3ffSMatthew D Fleming if (error != 0) 1198de78ca7eSPeter Wemm goto out; 119950c202c5SJeff Roberson error = parse_dpcpu(ef); 12002f22b3ffSMatthew D Fleming if (error != 0) 120150c202c5SJeff Roberson goto out; 1202eddfbb76SRobert Watson #ifdef VIMAGE 1203eddfbb76SRobert Watson error = parse_vnet(ef); 12042f22b3ffSMatthew D Fleming if (error != 0) 1205eddfbb76SRobert Watson goto out; 1206eddfbb76SRobert Watson #endif 12077251b4bfSJake Burkholder link_elf_reloc_local(lf); 12087251b4bfSJake Burkholder 1209b249ce48SMateusz Guzik VOP_UNLOCK(nd.ni_vp); 12107b9716baSIan Dowse error = linker_load_dependencies(lf); 12114f7afc20SKonstantin Belousov vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY); 12122f22b3ffSMatthew D Fleming if (error != 0) 1213de78ca7eSPeter Wemm goto out; 1214326e27d8SDoug Rabson error = relocate_file(ef); 12152f22b3ffSMatthew D Fleming if (error != 0) 1216ca65d5c7SPeter Wemm goto out; 1217ca65d5c7SPeter Wemm 12181aa696baSMark Johnston #ifdef SPARSE_MAPPING 12191aa696baSMark Johnston /* 12201aa696baSMark Johnston * Downgrade permissions on text segment mappings now that relocation 12211aa696baSMark Johnston * processing is complete. Restrict permissions on read-only segments. 12221aa696baSMark Johnston */ 12231aa696baSMark Johnston for (i = 0; i < nsegs; i++) { 12241aa696baSMark Johnston vm_prot_t prot; 12251aa696baSMark Johnston 12261aa696baSMark Johnston if (segs[i]->p_type != PT_LOAD) 12271aa696baSMark Johnston continue; 12281aa696baSMark Johnston 12291aa696baSMark Johnston prot = VM_PROT_READ; 12301aa696baSMark Johnston if ((segs[i]->p_flags & PF_W) != 0) 12311aa696baSMark Johnston prot |= VM_PROT_WRITE; 12321aa696baSMark Johnston if ((segs[i]->p_flags & PF_X) != 0) 12331aa696baSMark Johnston prot |= VM_PROT_EXECUTE; 12341aa696baSMark Johnston segbase = mapbase + segs[i]->p_vaddr - base_vaddr; 12351aa696baSMark Johnston error = vm_map_protect(kernel_map, 12361aa696baSMark Johnston (vm_offset_t)segbase, 12371aa696baSMark Johnston (vm_offset_t)segbase + round_page(segs[i]->p_memsz), 12380659df6fSKonstantin Belousov prot, 0, VM_MAP_PROTECT_SET_PROT); 12391aa696baSMark Johnston if (error != KERN_SUCCESS) { 12401aa696baSMark Johnston error = ENOMEM; 12411aa696baSMark Johnston goto out; 12421aa696baSMark Johnston } 12431aa696baSMark Johnston } 12441aa696baSMark Johnston #endif 12451aa696baSMark Johnston 12462f22b3ffSMatthew D Fleming /* 12472f22b3ffSMatthew D Fleming * Try and load the symbol table if it's present. (you can 12482f22b3ffSMatthew D Fleming * strip it!) 12492f22b3ffSMatthew D Fleming */ 1250caab6e90SPeter Wemm nbytes = hdr->e_shnum * hdr->e_shentsize; 1251caab6e90SPeter Wemm if (nbytes == 0 || hdr->e_shoff == 0) 1252ca65d5c7SPeter Wemm goto nosyms; 1253a163d034SWarner Losh shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 1254ca65d5c7SPeter Wemm error = vn_rdwr(UIO_READ, nd.ni_vp, 1255caab6e90SPeter Wemm (caddr_t)shdr, nbytes, hdr->e_shoff, 12569ca43589SRobert Watson UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 12579ca43589SRobert Watson &resid, td); 12582f22b3ffSMatthew D Fleming if (error != 0) 1259ca65d5c7SPeter Wemm goto out; 12600067051fSMarcel Moolenaar 12610067051fSMarcel Moolenaar /* Read section string table */ 12620067051fSMarcel Moolenaar shstrindex = hdr->e_shstrndx; 12630067051fSMarcel Moolenaar if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB && 12640067051fSMarcel Moolenaar shdr[shstrindex].sh_size != 0) { 12650067051fSMarcel Moolenaar nbytes = shdr[shstrindex].sh_size; 12660067051fSMarcel Moolenaar shstrs = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO); 12670067051fSMarcel Moolenaar error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs, nbytes, 12680067051fSMarcel Moolenaar shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED, 12690067051fSMarcel Moolenaar td->td_ucred, NOCRED, &resid, td); 12700067051fSMarcel Moolenaar if (error) 12710067051fSMarcel Moolenaar goto out; 12720067051fSMarcel Moolenaar } 12730067051fSMarcel Moolenaar 1274ca65d5c7SPeter Wemm symtabindex = -1; 1275ca65d5c7SPeter Wemm symstrindex = -1; 1276caab6e90SPeter Wemm for (i = 0; i < hdr->e_shnum; i++) { 1277ca65d5c7SPeter Wemm if (shdr[i].sh_type == SHT_SYMTAB) { 1278ca65d5c7SPeter Wemm symtabindex = i; 1279ca65d5c7SPeter Wemm symstrindex = shdr[i].sh_link; 12800067051fSMarcel Moolenaar } else if (shstrs != NULL && shdr[i].sh_name != 0 && 12810067051fSMarcel Moolenaar strcmp(shstrs + shdr[i].sh_name, ".ctors") == 0) { 12820067051fSMarcel Moolenaar /* Record relocated address and size of .ctors. */ 12830067051fSMarcel Moolenaar lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr; 12840067051fSMarcel Moolenaar lf->ctors_size = shdr[i].sh_size; 1285ca65d5c7SPeter Wemm } 1286ca65d5c7SPeter Wemm } 1287ca65d5c7SPeter Wemm if (symtabindex < 0 || symstrindex < 0) 1288ca65d5c7SPeter Wemm goto nosyms; 1289ca65d5c7SPeter Wemm 1290ca65d5c7SPeter Wemm symcnt = shdr[symtabindex].sh_size; 1291a163d034SWarner Losh ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK); 1292ca65d5c7SPeter Wemm strcnt = shdr[symstrindex].sh_size; 1293a163d034SWarner Losh ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK); 1294ca65d5c7SPeter Wemm 1295ca65d5c7SPeter Wemm error = vn_rdwr(UIO_READ, nd.ni_vp, 1296ca65d5c7SPeter Wemm ef->symbase, symcnt, shdr[symtabindex].sh_offset, 12979ca43589SRobert Watson UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 12989ca43589SRobert Watson &resid, td); 12992f22b3ffSMatthew D Fleming if (error != 0) 1300ca65d5c7SPeter Wemm goto out; 1301ca65d5c7SPeter Wemm error = vn_rdwr(UIO_READ, nd.ni_vp, 1302ca65d5c7SPeter Wemm ef->strbase, strcnt, shdr[symstrindex].sh_offset, 13039ca43589SRobert Watson UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 13049ca43589SRobert Watson &resid, td); 13052f22b3ffSMatthew D Fleming if (error != 0) 1306ca65d5c7SPeter Wemm goto out; 1307ca65d5c7SPeter Wemm 1308ca65d5c7SPeter Wemm ef->ddbsymcnt = symcnt / sizeof(Elf_Sym); 1309ca65d5c7SPeter Wemm ef->ddbsymtab = (const Elf_Sym *)ef->symbase; 1310ca65d5c7SPeter Wemm ef->ddbstrcnt = strcnt; 1311ca65d5c7SPeter Wemm ef->ddbstrtab = ef->strbase; 1312ca65d5c7SPeter Wemm 13138a710317SMarcel Moolenaar nosyms: 1314b19c9deaSIan Lepore 1315b19c9deaSIan Lepore #ifdef __arm__ 1316b19c9deaSIan Lepore link_elf_locate_exidx(lf, shdr, hdr->e_shnum); 1317b19c9deaSIan Lepore #endif 1318b19c9deaSIan Lepore 1319c143d6c2SMarcel Moolenaar error = link_elf_link_common_finish(lf); 13202f22b3ffSMatthew D Fleming if (error != 0) 1321c143d6c2SMarcel Moolenaar goto out; 1322326e27d8SDoug Rabson 1323a4f67738SDoug Rabson *result = lf; 1324a4f67738SDoug Rabson 1325a4f67738SDoug Rabson out: 1326b249ce48SMateusz Guzik VOP_UNLOCK(nd.ni_vp); 1327f1d2d305SKonstantin Belousov vn_close(nd.ni_vp, FREAD, td->td_ucred, td); 13282f22b3ffSMatthew D Fleming if (error != 0 && lf != NULL) 132965a311fcSPoul-Henning Kamp linker_file_unload(lf, LINKER_UNLOAD_FORCE); 1330ca65d5c7SPeter Wemm free(shdr, M_LINKER); 1331caab6e90SPeter Wemm free(firstpage, M_LINKER); 13320067051fSMarcel Moolenaar free(shstrs, M_LINKER); 1333a4f67738SDoug Rabson 13342f22b3ffSMatthew D Fleming return (error); 1335a4f67738SDoug Rabson } 1336a4f67738SDoug Rabson 133750c202c5SJeff Roberson Elf_Addr 133850c202c5SJeff Roberson elf_relocaddr(linker_file_t lf, Elf_Addr x) 133950c202c5SJeff Roberson { 134050c202c5SJeff Roberson elf_file_t ef; 134150c202c5SJeff Roberson 134238dae42cSMark Johnston KASSERT(lf->ops->cls == (kobj_class_t)&link_elf_class, 134338dae42cSMark Johnston ("elf_relocaddr: unexpected linker file %p", lf)); 134438dae42cSMark Johnston 134550c202c5SJeff Roberson ef = (elf_file_t)lf; 134650c202c5SJeff Roberson if (x >= ef->pcpu_start && x < ef->pcpu_stop) 134750c202c5SJeff Roberson return ((x - ef->pcpu_start) + ef->pcpu_base); 1348eddfbb76SRobert Watson #ifdef VIMAGE 1349eddfbb76SRobert Watson if (x >= ef->vnet_start && x < ef->vnet_stop) 1350eddfbb76SRobert Watson return ((x - ef->vnet_start) + ef->vnet_base); 1351eddfbb76SRobert Watson #endif 135250c202c5SJeff Roberson return (x); 135350c202c5SJeff Roberson } 135450c202c5SJeff Roberson 1355a4f67738SDoug Rabson static void 1356de78ca7eSPeter Wemm link_elf_unload_file(linker_file_t file) 1357a4f67738SDoug Rabson { 1358326e27d8SDoug Rabson elf_file_t ef = (elf_file_t) file; 1359a4f67738SDoug Rabson 13602f22b3ffSMatthew D Fleming if (ef->pcpu_base != 0) { 13612f22b3ffSMatthew D Fleming dpcpu_free((void *)ef->pcpu_base, 13622f22b3ffSMatthew D Fleming ef->pcpu_stop - ef->pcpu_start); 136347813f5dSMikolaj Golub elf_set_delete(&set_pcpu_list, ef->pcpu_start); 136450c202c5SJeff Roberson } 1365eddfbb76SRobert Watson #ifdef VIMAGE 13662f22b3ffSMatthew D Fleming if (ef->vnet_base != 0) { 13672f22b3ffSMatthew D Fleming vnet_data_free((void *)ef->vnet_base, 13682f22b3ffSMatthew D Fleming ef->vnet_stop - ef->vnet_start); 136947813f5dSMikolaj Golub elf_set_delete(&set_vnet_list, ef->vnet_start); 1370eddfbb76SRobert Watson } 1371eddfbb76SRobert Watson #endif 1372ecb01c64SMarcel Moolenaar #ifdef GDB 13732f22b3ffSMatthew D Fleming if (ef->gdb.l_ld != NULL) { 1374326e27d8SDoug Rabson GDB_STATE(RT_DELETE); 1375ed6c3888SPeter Wemm free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER); 1376326e27d8SDoug Rabson link_elf_delete_gdb(&ef->gdb); 1377326e27d8SDoug Rabson GDB_STATE(RT_CONSISTENT); 1378326e27d8SDoug Rabson } 1379326e27d8SDoug Rabson #endif 1380326e27d8SDoug Rabson 13811aeb23cdSMarcel Moolenaar /* Notify MD code that a module is being unloaded. */ 13821aeb23cdSMarcel Moolenaar elf_cpu_unload_file(file); 13831aeb23cdSMarcel Moolenaar 1384326e27d8SDoug Rabson if (ef->preloaded) { 138554823af2SPeter Wemm link_elf_unload_preload(file); 1386326e27d8SDoug Rabson return; 1387326e27d8SDoug Rabson } 1388c143d6c2SMarcel Moolenaar 1389fe3db7c7SDoug Rabson #ifdef SPARSE_MAPPING 13902f22b3ffSMatthew D Fleming if (ef->object != NULL) { 1391fe3db7c7SDoug Rabson vm_map_remove(kernel_map, (vm_offset_t) ef->address, 1392fe3db7c7SDoug Rabson (vm_offset_t) ef->address 1393fe3db7c7SDoug Rabson + (ef->object->size << PAGE_SHIFT)); 1394fe3db7c7SDoug Rabson } 1395fe3db7c7SDoug Rabson #else 1396a4f67738SDoug Rabson free(ef->address, M_LINKER); 1397fe3db7c7SDoug Rabson #endif 1398ca65d5c7SPeter Wemm free(ef->symbase, M_LINKER); 1399ca65d5c7SPeter Wemm free(ef->strbase, M_LINKER); 1400a2024a3eSJohn Birrell free(ef->ctftab, M_LINKER); 1401a2024a3eSJohn Birrell free(ef->ctfoff, M_LINKER); 1402a2024a3eSJohn Birrell free(ef->typoff, M_LINKER); 1403a4f67738SDoug Rabson } 1404a4f67738SDoug Rabson 1405de78ca7eSPeter Wemm static void 140654823af2SPeter Wemm link_elf_unload_preload(linker_file_t file) 1407de78ca7eSPeter Wemm { 1408f822c9e2SMark Johnston 140973624a80SMark Johnston if (file->pathname != NULL) 141073624a80SMark Johnston preload_delete_name(file->pathname); 1411de78ca7eSPeter Wemm } 1412de78ca7eSPeter Wemm 1413fe3db7c7SDoug Rabson static const char * 1414757686b1SMarcel Moolenaar symbol_name(elf_file_t ef, Elf_Size r_info) 1415a4f67738SDoug Rabson { 1416fe3db7c7SDoug Rabson const Elf_Sym *ref; 1417a4f67738SDoug Rabson 1418aa855a59SPeter Wemm if (ELF_R_SYM(r_info)) { 1419aa855a59SPeter Wemm ref = ef->symtab + ELF_R_SYM(r_info); 14202f22b3ffSMatthew D Fleming return (ef->strtab + ref->st_name); 14212f22b3ffSMatthew D Fleming } 14222f22b3ffSMatthew D Fleming return (NULL); 1423a4f67738SDoug Rabson } 1424a4f67738SDoug Rabson 1425a4f67738SDoug Rabson static int 142636716fe2SMark Johnston symbol_type(elf_file_t ef, Elf_Size r_info) 1427a4f67738SDoug Rabson { 142836716fe2SMark Johnston const Elf_Sym *ref; 142936716fe2SMark Johnston 143036716fe2SMark Johnston if (ELF_R_SYM(r_info)) { 143136716fe2SMark Johnston ref = ef->symtab + ELF_R_SYM(r_info); 143236716fe2SMark Johnston return (ELF_ST_TYPE(ref->st_info)); 143336716fe2SMark Johnston } 143436716fe2SMark Johnston return (STT_NOTYPE); 143536716fe2SMark Johnston } 143636716fe2SMark Johnston 143736716fe2SMark Johnston static int 143836716fe2SMark Johnston relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc, 143936716fe2SMark Johnston bool ifuncs) 144036716fe2SMark Johnston { 1441fe3db7c7SDoug Rabson const Elf_Rel *rel; 1442fe3db7c7SDoug Rabson const Elf_Rela *rela; 1443aa855a59SPeter Wemm const char *symname; 1444a4f67738SDoug Rabson 144536716fe2SMark Johnston #define APPLY_RELOCS(iter, tbl, tblsize, type) do { \ 144636716fe2SMark Johnston for ((iter) = (tbl); (iter) != NULL && \ 144736716fe2SMark Johnston (iter) < (tbl) + (tblsize) / sizeof(*(iter)); (iter)++) { \ 144836716fe2SMark Johnston if ((symbol_type(ef, (iter)->r_info) == \ 144936716fe2SMark Johnston STT_GNU_IFUNC || \ 145036716fe2SMark Johnston elf_is_ifunc_reloc((iter)->r_info)) != ifuncs) \ 145136716fe2SMark Johnston continue; \ 145236716fe2SMark Johnston if (reloc(&ef->lf, (Elf_Addr)ef->address, \ 145336716fe2SMark Johnston (iter), (type), lookup)) { \ 145436716fe2SMark Johnston symname = symbol_name(ef, (iter)->r_info); \ 145536716fe2SMark Johnston printf("link_elf: symbol %s undefined\n", \ 145636716fe2SMark Johnston symname); \ 145736716fe2SMark Johnston return (ENOENT); \ 145836716fe2SMark Johnston } \ 145936716fe2SMark Johnston } \ 146036716fe2SMark Johnston } while (0) 1461a4f67738SDoug Rabson 146236716fe2SMark Johnston APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL); 146336716fe2SMark Johnston APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA); 146436716fe2SMark Johnston APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL); 146536716fe2SMark Johnston APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA); 1466a4f67738SDoug Rabson 146736716fe2SMark Johnston #undef APPLY_RELOCS 1468a4f67738SDoug Rabson 14692f22b3ffSMatthew D Fleming return (0); 1470a4f67738SDoug Rabson } 1471a4f67738SDoug Rabson 14727035cf14SKonstantin Belousov static int 14737035cf14SKonstantin Belousov relocate_file(elf_file_t ef) 14747035cf14SKonstantin Belousov { 147536716fe2SMark Johnston int error; 14767035cf14SKonstantin Belousov 147736716fe2SMark Johnston error = relocate_file1(ef, elf_lookup, elf_reloc, false); 147836716fe2SMark Johnston if (error == 0) 147936716fe2SMark Johnston error = relocate_file1(ef, elf_lookup, elf_reloc, true); 148036716fe2SMark Johnston return (error); 14817035cf14SKonstantin Belousov } 14827035cf14SKonstantin Belousov 1483fe3db7c7SDoug Rabson /* 1484fe3db7c7SDoug Rabson * Hash function for symbol table lookup. Don't even think about changing 1485fe3db7c7SDoug Rabson * this. It is specified by the System V ABI. 1486fe3db7c7SDoug Rabson */ 1487fe3db7c7SDoug Rabson static unsigned long 1488fe3db7c7SDoug Rabson elf_hash(const char *name) 1489a4f67738SDoug Rabson { 1490fe3db7c7SDoug Rabson const unsigned char *p = (const unsigned char *) name; 1491fe3db7c7SDoug Rabson unsigned long h = 0; 1492fe3db7c7SDoug Rabson unsigned long g; 1493a4f67738SDoug Rabson 1494fe3db7c7SDoug Rabson while (*p != '\0') { 1495fe3db7c7SDoug Rabson h = (h << 4) + *p++; 1496fe3db7c7SDoug Rabson if ((g = h & 0xf0000000) != 0) 1497fe3db7c7SDoug Rabson h ^= g >> 24; 1498fe3db7c7SDoug Rabson h &= ~g; 1499a4f67738SDoug Rabson } 15002f22b3ffSMatthew D Fleming return (h); 1501fe3db7c7SDoug Rabson } 1502a4f67738SDoug Rabson 150337c84183SPoul-Henning Kamp static int 150495c20fafSKonstantin Belousov link_elf_lookup_symbol1(linker_file_t lf, const char *name, c_linker_sym_t *sym, 150595c20fafSKonstantin Belousov bool see_local) 1506a4f67738SDoug Rabson { 1507326e27d8SDoug Rabson elf_file_t ef = (elf_file_t) lf; 1508fe3db7c7SDoug Rabson unsigned long symnum; 15092d636ab0SPeter Wemm const Elf_Sym* symp; 15102d636ab0SPeter Wemm const char *strp; 1511fe3db7c7SDoug Rabson unsigned long hash; 1512a4f67738SDoug Rabson 1513f6494f2eSJohn Baldwin /* If we don't have a hash, bail. */ 1514f6494f2eSJohn Baldwin if (ef->buckets == NULL || ef->nbuckets == 0) { 1515f6494f2eSJohn Baldwin printf("link_elf_lookup_symbol: missing symbol hash table\n"); 15162f22b3ffSMatthew D Fleming return (ENOENT); 1517f6494f2eSJohn Baldwin } 1518f6494f2eSJohn Baldwin 15192d636ab0SPeter Wemm /* First, search hashed global symbols */ 1520fe3db7c7SDoug Rabson hash = elf_hash(name); 1521fe3db7c7SDoug Rabson symnum = ef->buckets[hash % ef->nbuckets]; 1522fe3db7c7SDoug Rabson 1523fe3db7c7SDoug Rabson while (symnum != STN_UNDEF) { 1524fe3db7c7SDoug Rabson if (symnum >= ef->nchains) { 15252f22b3ffSMatthew D Fleming printf("%s: corrupt symbol table\n", __func__); 15262f22b3ffSMatthew D Fleming return (ENOENT); 1527a4f67738SDoug Rabson } 1528fe3db7c7SDoug Rabson 1529fe3db7c7SDoug Rabson symp = ef->symtab + symnum; 1530fe3db7c7SDoug Rabson if (symp->st_name == 0) { 15312f22b3ffSMatthew D Fleming printf("%s: corrupt symbol table\n", __func__); 15322f22b3ffSMatthew D Fleming return (ENOENT); 1533fe3db7c7SDoug Rabson } 1534fe3db7c7SDoug Rabson 1535fe3db7c7SDoug Rabson strp = ef->strtab + symp->st_name; 1536fe3db7c7SDoug Rabson 1537fe3db7c7SDoug Rabson if (strcmp(name, strp) == 0) { 1538fe3db7c7SDoug Rabson if (symp->st_shndx != SHN_UNDEF || 1539fe3db7c7SDoug Rabson (symp->st_value != 0 && 15407035cf14SKonstantin Belousov (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 15417035cf14SKonstantin Belousov ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 154295c20fafSKonstantin Belousov if (see_local || 154395c20fafSKonstantin Belousov ELF_ST_BIND(symp->st_info) != STB_LOCAL) { 1544d254af07SMatthew Dillon *sym = (c_linker_sym_t) symp; 15452f22b3ffSMatthew D Fleming return (0); 15462f22b3ffSMatthew D Fleming } 154795c20fafSKonstantin Belousov } 15482f22b3ffSMatthew D Fleming return (ENOENT); 1549fe3db7c7SDoug Rabson } 1550fe3db7c7SDoug Rabson 1551fe3db7c7SDoug Rabson symnum = ef->chains[symnum]; 1552a4f67738SDoug Rabson } 1553a4f67738SDoug Rabson 15542f22b3ffSMatthew D Fleming return (ENOENT); 155595c20fafSKonstantin Belousov } 15562d636ab0SPeter Wemm 155795c20fafSKonstantin Belousov static int 155895c20fafSKonstantin Belousov link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) 155995c20fafSKonstantin Belousov { 1560*ecd8245eSKonstantin Belousov if (link_elf_leak_locals) 1561*ecd8245eSKonstantin Belousov return (link_elf_lookup_debug_symbol(lf, name, sym)); 156295c20fafSKonstantin Belousov return (link_elf_lookup_symbol1(lf, name, sym, false)); 156395c20fafSKonstantin Belousov } 156495c20fafSKonstantin Belousov 156595c20fafSKonstantin Belousov static int 156695c20fafSKonstantin Belousov link_elf_lookup_debug_symbol(linker_file_t lf, const char *name, 156795c20fafSKonstantin Belousov c_linker_sym_t *sym) 156895c20fafSKonstantin Belousov { 156995c20fafSKonstantin Belousov elf_file_t ef = (elf_file_t)lf; 157095c20fafSKonstantin Belousov const Elf_Sym* symp; 157195c20fafSKonstantin Belousov const char *strp; 157295c20fafSKonstantin Belousov int i; 157395c20fafSKonstantin Belousov 157495c20fafSKonstantin Belousov if (link_elf_lookup_symbol1(lf, name, sym, true) == 0) 157595c20fafSKonstantin Belousov return (0); 157695c20fafSKonstantin Belousov 15772d636ab0SPeter Wemm for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 15782d636ab0SPeter Wemm strp = ef->ddbstrtab + symp->st_name; 15792d636ab0SPeter Wemm if (strcmp(name, strp) == 0) { 15802d636ab0SPeter Wemm if (symp->st_shndx != SHN_UNDEF || 15812d636ab0SPeter Wemm (symp->st_value != 0 && 15827035cf14SKonstantin Belousov (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 15837035cf14SKonstantin Belousov ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) { 1584d254af07SMatthew Dillon *sym = (c_linker_sym_t) symp; 15852f22b3ffSMatthew D Fleming return (0); 15862f22b3ffSMatthew D Fleming } 15872f22b3ffSMatthew D Fleming return (ENOENT); 15882d636ab0SPeter Wemm } 15892d636ab0SPeter Wemm } 15902d636ab0SPeter Wemm 15912f22b3ffSMatthew D Fleming return (ENOENT); 1592a4f67738SDoug Rabson } 1593a4f67738SDoug Rabson 1594de78ca7eSPeter Wemm static int 159595c20fafSKonstantin Belousov link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, 159695c20fafSKonstantin Belousov linker_symval_t *symval, bool see_local) 1597a4f67738SDoug Rabson { 15987035cf14SKonstantin Belousov elf_file_t ef; 15997035cf14SKonstantin Belousov const Elf_Sym *es; 16007035cf14SKonstantin Belousov caddr_t val; 1601a4f67738SDoug Rabson 16027035cf14SKonstantin Belousov ef = (elf_file_t)lf; 16037035cf14SKonstantin Belousov es = (const Elf_Sym *)sym; 160495c20fafSKonstantin Belousov if (es >= ef->symtab && es < ef->symtab + ef->nchains) { 160595c20fafSKonstantin Belousov if (!see_local && ELF_ST_BIND(es->st_info) == STB_LOCAL) 160695c20fafSKonstantin Belousov return (ENOENT); 1607a4f67738SDoug Rabson symval->name = ef->strtab + es->st_name; 16087035cf14SKonstantin Belousov val = (caddr_t)ef->address + es->st_value; 16097035cf14SKonstantin Belousov if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 16107035cf14SKonstantin Belousov val = ((caddr_t (*)(void))val)(); 16117035cf14SKonstantin Belousov symval->value = val; 1612a4f67738SDoug Rabson symval->size = es->st_size; 16132f22b3ffSMatthew D Fleming return (0); 1614a4f67738SDoug Rabson } 161595c20fafSKonstantin Belousov return (ENOENT); 161695c20fafSKonstantin Belousov } 161795c20fafSKonstantin Belousov 161895c20fafSKonstantin Belousov static int 161995c20fafSKonstantin Belousov link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, 162095c20fafSKonstantin Belousov linker_symval_t *symval) 162195c20fafSKonstantin Belousov { 1622*ecd8245eSKonstantin Belousov if (link_elf_leak_locals) 1623*ecd8245eSKonstantin Belousov return (link_elf_debug_symbol_values(lf, sym, symval)); 162495c20fafSKonstantin Belousov return (link_elf_symbol_values1(lf, sym, symval, false)); 162595c20fafSKonstantin Belousov } 162695c20fafSKonstantin Belousov 162795c20fafSKonstantin Belousov static int 162895c20fafSKonstantin Belousov link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym, 162995c20fafSKonstantin Belousov linker_symval_t *symval) 163095c20fafSKonstantin Belousov { 163195c20fafSKonstantin Belousov elf_file_t ef = (elf_file_t)lf; 163295c20fafSKonstantin Belousov const Elf_Sym *es = (const Elf_Sym *)sym; 163395c20fafSKonstantin Belousov caddr_t val; 163495c20fafSKonstantin Belousov 163595c20fafSKonstantin Belousov if (link_elf_symbol_values1(lf, sym, symval, true) == 0) 163695c20fafSKonstantin Belousov return (0); 16372d636ab0SPeter Wemm if (ef->symtab == ef->ddbsymtab) 16382f22b3ffSMatthew D Fleming return (ENOENT); 163995c20fafSKonstantin Belousov 16405cf87418SMarcel Moolenaar if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { 16412d636ab0SPeter Wemm symval->name = ef->ddbstrtab + es->st_name; 16427035cf14SKonstantin Belousov val = (caddr_t)ef->address + es->st_value; 16437035cf14SKonstantin Belousov if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 16447035cf14SKonstantin Belousov val = ((caddr_t (*)(void))val)(); 16457035cf14SKonstantin Belousov symval->value = val; 16462d636ab0SPeter Wemm symval->size = es->st_size; 16472f22b3ffSMatthew D Fleming return (0); 16482d636ab0SPeter Wemm } 16492f22b3ffSMatthew D Fleming return (ENOENT); 16502d636ab0SPeter Wemm } 1651a4f67738SDoug Rabson 1652a4f67738SDoug Rabson static int 1653a4f67738SDoug Rabson link_elf_search_symbol(linker_file_t lf, caddr_t value, 1654d254af07SMatthew Dillon c_linker_sym_t *sym, long *diffp) 1655a4f67738SDoug Rabson { 1656326e27d8SDoug Rabson elf_file_t ef = (elf_file_t)lf; 1657586453feSBruce Evans u_long off = (uintptr_t)(void *)value; 1658a4f67738SDoug Rabson u_long diff = off; 1659586453feSBruce Evans u_long st_value; 1660fe3db7c7SDoug Rabson const Elf_Sym *es; 1661b85f65afSPedro F. Giffuni const Elf_Sym *best = NULL; 1662a4f67738SDoug Rabson int i; 1663a4f67738SDoug Rabson 16642d636ab0SPeter Wemm for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) { 1665a4f67738SDoug Rabson if (es->st_name == 0) 1666a4f67738SDoug Rabson continue; 1667586453feSBruce Evans st_value = es->st_value + (uintptr_t) (void *) ef->address; 1668b5abfb70SPeter Wemm if (off >= st_value) { 1669b5abfb70SPeter Wemm if (off - st_value < diff) { 1670b5abfb70SPeter Wemm diff = off - st_value; 1671a4f67738SDoug Rabson best = es; 1672a4f67738SDoug Rabson if (diff == 0) 1673a4f67738SDoug Rabson break; 1674b5abfb70SPeter Wemm } else if (off - st_value == diff) { 1675a4f67738SDoug Rabson best = es; 1676a4f67738SDoug Rabson } 1677a4f67738SDoug Rabson } 1678a4f67738SDoug Rabson } 1679b85f65afSPedro F. Giffuni if (best == NULL) 1680a4f67738SDoug Rabson *diffp = off; 1681a4f67738SDoug Rabson else 1682a4f67738SDoug Rabson *diffp = diff; 1683d254af07SMatthew Dillon *sym = (c_linker_sym_t) best; 1684a4f67738SDoug Rabson 16852f22b3ffSMatthew D Fleming return (0); 1686a4f67738SDoug Rabson } 1687f41325dbSPeter Wemm 1688f41325dbSPeter Wemm /* 1689f41325dbSPeter Wemm * Look up a linker set on an ELF system. 1690f41325dbSPeter Wemm */ 1691f41325dbSPeter Wemm static int 1692f41325dbSPeter Wemm link_elf_lookup_set(linker_file_t lf, const char *name, 1693f41325dbSPeter Wemm void ***startp, void ***stopp, int *countp) 1694f41325dbSPeter Wemm { 1695f41325dbSPeter Wemm c_linker_sym_t sym; 1696f41325dbSPeter Wemm linker_symval_t symval; 1697f41325dbSPeter Wemm char *setsym; 1698f41325dbSPeter Wemm void **start, **stop; 1699f41325dbSPeter Wemm int len, error = 0, count; 1700f41325dbSPeter Wemm 1701f41325dbSPeter Wemm len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */ 1702a163d034SWarner Losh setsym = malloc(len, M_LINKER, M_WAITOK); 1703f41325dbSPeter Wemm 1704f41325dbSPeter Wemm /* get address of first entry */ 1705f41325dbSPeter Wemm snprintf(setsym, len, "%s%s", "__start_set_", name); 1706f41325dbSPeter Wemm error = link_elf_lookup_symbol(lf, setsym, &sym); 17072f22b3ffSMatthew D Fleming if (error != 0) 1708f41325dbSPeter Wemm goto out; 1709f41325dbSPeter Wemm link_elf_symbol_values(lf, sym, &symval); 1710f41325dbSPeter Wemm if (symval.value == 0) { 1711f41325dbSPeter Wemm error = ESRCH; 1712f41325dbSPeter Wemm goto out; 1713f41325dbSPeter Wemm } 1714f41325dbSPeter Wemm start = (void **)symval.value; 1715f41325dbSPeter Wemm 1716f41325dbSPeter Wemm /* get address of last entry */ 1717f41325dbSPeter Wemm snprintf(setsym, len, "%s%s", "__stop_set_", name); 1718f41325dbSPeter Wemm error = link_elf_lookup_symbol(lf, setsym, &sym); 17192f22b3ffSMatthew D Fleming if (error != 0) 1720f41325dbSPeter Wemm goto out; 1721f41325dbSPeter Wemm link_elf_symbol_values(lf, sym, &symval); 1722f41325dbSPeter Wemm if (symval.value == 0) { 1723f41325dbSPeter Wemm error = ESRCH; 1724f41325dbSPeter Wemm goto out; 1725f41325dbSPeter Wemm } 1726f41325dbSPeter Wemm stop = (void **)symval.value; 1727f41325dbSPeter Wemm 1728f41325dbSPeter Wemm /* and the number of entries */ 1729f41325dbSPeter Wemm count = stop - start; 1730f41325dbSPeter Wemm 1731f41325dbSPeter Wemm /* and copy out */ 17322f22b3ffSMatthew D Fleming if (startp != NULL) 1733f41325dbSPeter Wemm *startp = start; 17342f22b3ffSMatthew D Fleming if (stopp != NULL) 1735f41325dbSPeter Wemm *stopp = stop; 17362f22b3ffSMatthew D Fleming if (countp != NULL) 1737f41325dbSPeter Wemm *countp = count; 1738f41325dbSPeter Wemm 1739f41325dbSPeter Wemm out: 1740f41325dbSPeter Wemm free(setsym, M_LINKER); 17412f22b3ffSMatthew D Fleming return (error); 1742f41325dbSPeter Wemm } 1743bb9fe9ddSBrian Feldman 1744bb9fe9ddSBrian Feldman static int 1745bb9fe9ddSBrian Feldman link_elf_each_function_name(linker_file_t file, 17462f22b3ffSMatthew D Fleming int (*callback)(const char *, void *), void *opaque) 17472f22b3ffSMatthew D Fleming { 1748bb9fe9ddSBrian Feldman elf_file_t ef = (elf_file_t)file; 1749bb9fe9ddSBrian Feldman const Elf_Sym *symp; 1750bb9fe9ddSBrian Feldman int i, error; 1751bb9fe9ddSBrian Feldman 1752bb9fe9ddSBrian Feldman /* Exhaustive search */ 1753bb9fe9ddSBrian Feldman for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1754bb9fe9ddSBrian Feldman if (symp->st_value != 0 && 17557035cf14SKonstantin Belousov (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 17567035cf14SKonstantin Belousov ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1757bb9fe9ddSBrian Feldman error = callback(ef->ddbstrtab + symp->st_name, opaque); 17582f22b3ffSMatthew D Fleming if (error != 0) 1759bb9fe9ddSBrian Feldman return (error); 1760bb9fe9ddSBrian Feldman } 1761bb9fe9ddSBrian Feldman } 1762bb9fe9ddSBrian Feldman return (0); 1763bb9fe9ddSBrian Feldman } 176484201059SMarcel Moolenaar 1765a2024a3eSJohn Birrell static int 1766a2024a3eSJohn Birrell link_elf_each_function_nameval(linker_file_t file, 1767a2024a3eSJohn Birrell linker_function_nameval_callback_t callback, void *opaque) 1768a2024a3eSJohn Birrell { 1769a2024a3eSJohn Birrell linker_symval_t symval; 1770a2024a3eSJohn Birrell elf_file_t ef = (elf_file_t)file; 1771a2024a3eSJohn Birrell const Elf_Sym *symp; 1772a2024a3eSJohn Birrell int i, error; 1773a2024a3eSJohn Birrell 1774a2024a3eSJohn Birrell /* Exhaustive search */ 1775a2024a3eSJohn Birrell for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1776a2024a3eSJohn Birrell if (symp->st_value != 0 && 17777035cf14SKonstantin Belousov (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 17787035cf14SKonstantin Belousov ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 177995c20fafSKonstantin Belousov error = link_elf_debug_symbol_values(file, 17802f22b3ffSMatthew D Fleming (c_linker_sym_t) symp, &symval); 178195c20fafSKonstantin Belousov if (error == 0) 1782a2024a3eSJohn Birrell error = callback(file, i, &symval, opaque); 17832f22b3ffSMatthew D Fleming if (error != 0) 1784a2024a3eSJohn Birrell return (error); 1785a2024a3eSJohn Birrell } 1786a2024a3eSJohn Birrell } 1787a2024a3eSJohn Birrell return (0); 1788a2024a3eSJohn Birrell } 1789a2024a3eSJohn Birrell 1790169d513cSJake Burkholder const Elf_Sym * 1791757686b1SMarcel Moolenaar elf_get_sym(linker_file_t lf, Elf_Size symidx) 1792169d513cSJake Burkholder { 1793169d513cSJake Burkholder elf_file_t ef = (elf_file_t)lf; 1794169d513cSJake Burkholder 1795169d513cSJake Burkholder if (symidx >= ef->nchains) 1796169d513cSJake Burkholder return (NULL); 1797169d513cSJake Burkholder return (ef->symtab + symidx); 1798169d513cSJake Burkholder } 1799169d513cSJake Burkholder 180047f75012SMarcel Moolenaar const char * 1801757686b1SMarcel Moolenaar elf_get_symname(linker_file_t lf, Elf_Size symidx) 180247f75012SMarcel Moolenaar { 180347f75012SMarcel Moolenaar elf_file_t ef = (elf_file_t)lf; 180447f75012SMarcel Moolenaar const Elf_Sym *sym; 180547f75012SMarcel Moolenaar 180647f75012SMarcel Moolenaar if (symidx >= ef->nchains) 180747f75012SMarcel Moolenaar return (NULL); 180847f75012SMarcel Moolenaar sym = ef->symtab + symidx; 180947f75012SMarcel Moolenaar return (ef->strtab + sym->st_name); 181047f75012SMarcel Moolenaar } 181147f75012SMarcel Moolenaar 1812d297ad16SMarcel Moolenaar /* 1813d297ad16SMarcel Moolenaar * Symbol lookup function that can be used when the symbol index is known (ie 1814d297ad16SMarcel Moolenaar * in relocations). It uses the symbol index instead of doing a fully fledged 1815d297ad16SMarcel Moolenaar * hash table based lookup when such is valid. For example for local symbols. 1816d297ad16SMarcel Moolenaar * This is not only more efficient, it's also more correct. It's not always 1817d297ad16SMarcel Moolenaar * the case that the symbol can be found through the hash table. 1818d297ad16SMarcel Moolenaar */ 1819cff8c6f2SKonstantin Belousov static int 1820cff8c6f2SKonstantin Belousov elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) 1821d297ad16SMarcel Moolenaar { 1822d297ad16SMarcel Moolenaar elf_file_t ef = (elf_file_t)lf; 1823d297ad16SMarcel Moolenaar const Elf_Sym *sym; 1824d297ad16SMarcel Moolenaar const char *symbol; 182547813f5dSMikolaj Golub Elf_Addr addr, start, base; 1826d297ad16SMarcel Moolenaar 1827d297ad16SMarcel Moolenaar /* Don't even try to lookup the symbol if the index is bogus. */ 1828cff8c6f2SKonstantin Belousov if (symidx >= ef->nchains) { 1829cff8c6f2SKonstantin Belousov *res = 0; 1830cff8c6f2SKonstantin Belousov return (EINVAL); 1831cff8c6f2SKonstantin Belousov } 1832d297ad16SMarcel Moolenaar 1833d297ad16SMarcel Moolenaar sym = ef->symtab + symidx; 1834d297ad16SMarcel Moolenaar 1835d297ad16SMarcel Moolenaar /* 1836d297ad16SMarcel Moolenaar * Don't do a full lookup when the symbol is local. It may even 1837d297ad16SMarcel Moolenaar * fail because it may not be found through the hash table. 1838d297ad16SMarcel Moolenaar */ 1839d297ad16SMarcel Moolenaar if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) { 1840d297ad16SMarcel Moolenaar /* Force lookup failure when we have an insanity. */ 1841cff8c6f2SKonstantin Belousov if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) { 1842cff8c6f2SKonstantin Belousov *res = 0; 1843cff8c6f2SKonstantin Belousov return (EINVAL); 1844cff8c6f2SKonstantin Belousov } 1845cff8c6f2SKonstantin Belousov *res = ((Elf_Addr)ef->address + sym->st_value); 1846d297ad16SMarcel Moolenaar return (0); 1847d297ad16SMarcel Moolenaar } 1848d297ad16SMarcel Moolenaar 1849d297ad16SMarcel Moolenaar /* 1850d297ad16SMarcel Moolenaar * XXX we can avoid doing a hash table based lookup for global 1851d297ad16SMarcel Moolenaar * symbols as well. This however is not always valid, so we'll 1852d297ad16SMarcel Moolenaar * just do it the hard way for now. Performance tweaks can 1853d297ad16SMarcel Moolenaar * always be added. 1854d297ad16SMarcel Moolenaar */ 1855d297ad16SMarcel Moolenaar 1856d297ad16SMarcel Moolenaar symbol = ef->strtab + sym->st_name; 1857d297ad16SMarcel Moolenaar 1858d297ad16SMarcel Moolenaar /* Force a lookup failure if the symbol name is bogus. */ 1859cff8c6f2SKonstantin Belousov if (*symbol == 0) { 1860cff8c6f2SKonstantin Belousov *res = 0; 1861cff8c6f2SKonstantin Belousov return (EINVAL); 1862cff8c6f2SKonstantin Belousov } 1863d297ad16SMarcel Moolenaar 186447813f5dSMikolaj Golub addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps)); 1865e8e0fac5SMark Johnston if (addr == 0 && ELF_ST_BIND(sym->st_info) != STB_WEAK) { 1866e8e0fac5SMark Johnston *res = 0; 1867e8e0fac5SMark Johnston return (EINVAL); 1868e8e0fac5SMark Johnston } 186947813f5dSMikolaj Golub 187047813f5dSMikolaj Golub if (elf_set_find(&set_pcpu_list, addr, &start, &base)) 187147813f5dSMikolaj Golub addr = addr - start + base; 187247813f5dSMikolaj Golub #ifdef VIMAGE 187347813f5dSMikolaj Golub else if (elf_set_find(&set_vnet_list, addr, &start, &base)) 187447813f5dSMikolaj Golub addr = addr - start + base; 187547813f5dSMikolaj Golub #endif 1876cff8c6f2SKonstantin Belousov *res = addr; 1877cff8c6f2SKonstantin Belousov return (0); 1878d297ad16SMarcel Moolenaar } 18797251b4bfSJake Burkholder 18807251b4bfSJake Burkholder static void 18817251b4bfSJake Burkholder link_elf_reloc_local(linker_file_t lf) 18827251b4bfSJake Burkholder { 18837251b4bfSJake Burkholder const Elf_Rel *rellim; 18847251b4bfSJake Burkholder const Elf_Rel *rel; 18857251b4bfSJake Burkholder const Elf_Rela *relalim; 18867251b4bfSJake Burkholder const Elf_Rela *rela; 18877251b4bfSJake Burkholder elf_file_t ef = (elf_file_t)lf; 18887251b4bfSJake Burkholder 18897251b4bfSJake Burkholder /* Perform relocations without addend if there are any: */ 18907251b4bfSJake Burkholder if ((rel = ef->rel) != NULL) { 18917251b4bfSJake Burkholder rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize); 18927251b4bfSJake Burkholder while (rel < rellim) { 18932f22b3ffSMatthew D Fleming elf_reloc_local(lf, (Elf_Addr)ef->address, rel, 18942f22b3ffSMatthew D Fleming ELF_RELOC_REL, elf_lookup); 18957251b4bfSJake Burkholder rel++; 18967251b4bfSJake Burkholder } 18977251b4bfSJake Burkholder } 18987251b4bfSJake Burkholder 18997251b4bfSJake Burkholder /* Perform relocations with addend if there are any: */ 19007251b4bfSJake Burkholder if ((rela = ef->rela) != NULL) { 19012f22b3ffSMatthew D Fleming relalim = (const Elf_Rela *) 19022f22b3ffSMatthew D Fleming ((const char *)ef->rela + ef->relasize); 19037251b4bfSJake Burkholder while (rela < relalim) { 19042f22b3ffSMatthew D Fleming elf_reloc_local(lf, (Elf_Addr)ef->address, rela, 19052f22b3ffSMatthew D Fleming ELF_RELOC_RELA, elf_lookup); 19067251b4bfSJake Burkholder rela++; 19077251b4bfSJake Burkholder } 19087251b4bfSJake Burkholder } 19097251b4bfSJake Burkholder } 191000a5db46SStacey Son 191100a5db46SStacey Son static long 191200a5db46SStacey Son link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab) 191300a5db46SStacey Son { 191400a5db46SStacey Son elf_file_t ef = (elf_file_t)lf; 191500a5db46SStacey Son 191600a5db46SStacey Son *symtab = ef->ddbsymtab; 191700a5db46SStacey Son 191800a5db46SStacey Son if (*symtab == NULL) 191900a5db46SStacey Son return (0); 192000a5db46SStacey Son 192100a5db46SStacey Son return (ef->ddbsymcnt); 192200a5db46SStacey Son } 192300a5db46SStacey Son 192400a5db46SStacey Son static long 192500a5db46SStacey Son link_elf_strtab_get(linker_file_t lf, caddr_t *strtab) 192600a5db46SStacey Son { 192700a5db46SStacey Son elf_file_t ef = (elf_file_t)lf; 192800a5db46SStacey Son 192900a5db46SStacey Son *strtab = ef->ddbstrtab; 193000a5db46SStacey Son 193100a5db46SStacey Son if (*strtab == NULL) 193200a5db46SStacey Son return (0); 193300a5db46SStacey Son 193400a5db46SStacey Son return (ef->ddbstrcnt); 193500a5db46SStacey Son } 19367035cf14SKonstantin Belousov 19379411e24dSBrandon Bergren #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) || defined(__powerpc__) 193836716fe2SMark Johnston /* 193936716fe2SMark Johnston * Use this lookup routine when performing relocations early during boot. 194036716fe2SMark Johnston * The generic lookup routine depends on kobj, which is not initialized 194136716fe2SMark Johnston * at that point. 194236716fe2SMark Johnston */ 194336716fe2SMark Johnston static int 194436716fe2SMark Johnston elf_lookup_ifunc(linker_file_t lf, Elf_Size symidx, int deps __unused, 194536716fe2SMark Johnston Elf_Addr *res) 194636716fe2SMark Johnston { 194736716fe2SMark Johnston elf_file_t ef; 194836716fe2SMark Johnston const Elf_Sym *symp; 194936716fe2SMark Johnston caddr_t val; 195036716fe2SMark Johnston 195136716fe2SMark Johnston ef = (elf_file_t)lf; 195236716fe2SMark Johnston symp = ef->symtab + symidx; 195336716fe2SMark Johnston if (ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC) { 195436716fe2SMark Johnston val = (caddr_t)ef->address + symp->st_value; 195536716fe2SMark Johnston *res = ((Elf_Addr (*)(void))val)(); 195636716fe2SMark Johnston return (0); 195736716fe2SMark Johnston } 195836716fe2SMark Johnston return (ENOENT); 195936716fe2SMark Johnston } 196036716fe2SMark Johnston 19617035cf14SKonstantin Belousov void 19627035cf14SKonstantin Belousov link_elf_ireloc(caddr_t kmdp) 19637035cf14SKonstantin Belousov { 19647035cf14SKonstantin Belousov struct elf_file eff; 19657035cf14SKonstantin Belousov elf_file_t ef; 19667035cf14SKonstantin Belousov 19677035cf14SKonstantin Belousov ef = &eff; 196855c9d75eSKonstantin Belousov 19693f102f58SMateusz Guzik bzero_early(ef, sizeof(*ef)); 197055c9d75eSKonstantin Belousov 19717035cf14SKonstantin Belousov ef->modptr = kmdp; 19727035cf14SKonstantin Belousov ef->dynamic = (Elf_Dyn *)&_DYNAMIC; 19739411e24dSBrandon Bergren 19749411e24dSBrandon Bergren #ifdef RELOCATABLE_KERNEL 19759411e24dSBrandon Bergren ef->address = (caddr_t) (__startkernel - KERNBASE); 19769411e24dSBrandon Bergren #else 19777035cf14SKonstantin Belousov ef->address = 0; 19789411e24dSBrandon Bergren #endif 19799411e24dSBrandon Bergren parse_dynamic(ef); 19809411e24dSBrandon Bergren 19817035cf14SKonstantin Belousov link_elf_preload_parse_symbols(ef); 198236716fe2SMark Johnston relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true); 19837035cf14SKonstantin Belousov } 1984fcf7a481SAndrew Turner 1985fcf7a481SAndrew Turner #if defined(__aarch64__) || defined(__amd64__) 1986fcf7a481SAndrew Turner void 1987fcf7a481SAndrew Turner link_elf_late_ireloc(void) 1988fcf7a481SAndrew Turner { 1989fcf7a481SAndrew Turner elf_file_t ef; 1990fcf7a481SAndrew Turner 1991fcf7a481SAndrew Turner KASSERT(linker_kernel_file != NULL, 1992fcf7a481SAndrew Turner ("link_elf_late_ireloc: No kernel linker file found")); 1993fcf7a481SAndrew Turner ef = (elf_file_t)linker_kernel_file; 1994fcf7a481SAndrew Turner 1995fcf7a481SAndrew Turner relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true); 1996fcf7a481SAndrew Turner } 1997fcf7a481SAndrew Turner #endif 19987035cf14SKonstantin Belousov #endif 1999