1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1998-2000 Doug Rabson 5 * Copyright (c) 2004 Peter Wemm 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_ddb.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/fcntl.h> 36 #include <sys/kernel.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/linker.h> 40 #include <sys/mutex.h> 41 #include <sys/mount.h> 42 #include <sys/namei.h> 43 #include <sys/proc.h> 44 #include <sys/rwlock.h> 45 #include <sys/sysctl.h> 46 #include <sys/vnode.h> 47 48 #include <machine/elf.h> 49 50 #include <net/vnet.h> 51 52 #include <security/mac/mac_framework.h> 53 54 #include <vm/vm.h> 55 #include <vm/vm_param.h> 56 #include <vm/pmap.h> 57 #include <vm/vm_extern.h> 58 #include <vm/vm_kern.h> 59 #include <vm/vm_map.h> 60 #include <vm/vm_object.h> 61 #include <vm/vm_page.h> 62 #include <vm/vm_pager.h> 63 64 #include <sys/link_elf.h> 65 66 #ifdef DDB_CTF 67 #include <contrib/zlib/zlib.h> 68 #endif 69 70 #include "linker_if.h" 71 72 typedef struct { 73 void *addr; 74 Elf_Off size; 75 int flags; /* Section flags. */ 76 int sec; /* Original section number. */ 77 char *name; 78 } Elf_progent; 79 80 typedef struct { 81 Elf_Rel *rel; 82 int nrel; 83 int sec; 84 } Elf_relent; 85 86 typedef struct { 87 Elf_Rela *rela; 88 int nrela; 89 int sec; 90 } Elf_relaent; 91 92 typedef struct elf_file { 93 struct linker_file lf; /* Common fields */ 94 95 int preloaded; 96 caddr_t address; /* Relocation address */ 97 vm_object_t object; /* VM object to hold file pages */ 98 Elf_Shdr *e_shdr; 99 100 Elf_progent *progtab; 101 u_int nprogtab; 102 103 Elf_relaent *relatab; 104 u_int nrelatab; 105 106 Elf_relent *reltab; 107 int nreltab; 108 109 Elf_Sym *ddbsymtab; /* The symbol table we are using */ 110 long ddbsymcnt; /* Number of symbols */ 111 caddr_t ddbstrtab; /* String table */ 112 long ddbstrcnt; /* number of bytes in string table */ 113 114 caddr_t shstrtab; /* Section name string table */ 115 long shstrcnt; /* number of bytes in string table */ 116 117 caddr_t ctftab; /* CTF table */ 118 long ctfcnt; /* number of bytes in CTF table */ 119 caddr_t ctfoff; /* CTF offset table */ 120 caddr_t typoff; /* Type offset table */ 121 long typlen; /* Number of type entries. */ 122 123 } *elf_file_t; 124 125 #include <kern/kern_ctf.c> 126 127 static int link_elf_link_preload(linker_class_t cls, 128 const char *, linker_file_t *); 129 static int link_elf_link_preload_finish(linker_file_t); 130 static int link_elf_load_file(linker_class_t, const char *, linker_file_t *); 131 static int link_elf_lookup_symbol(linker_file_t, const char *, 132 c_linker_sym_t *); 133 static int link_elf_lookup_debug_symbol(linker_file_t, const char *, 134 c_linker_sym_t *); 135 static int link_elf_symbol_values(linker_file_t, c_linker_sym_t, 136 linker_symval_t *); 137 static int link_elf_debug_symbol_values(linker_file_t, c_linker_sym_t, 138 linker_symval_t *); 139 static int link_elf_search_symbol(linker_file_t, caddr_t value, 140 c_linker_sym_t *sym, long *diffp); 141 142 static void link_elf_unload_file(linker_file_t); 143 static int link_elf_lookup_set(linker_file_t, const char *, 144 void ***, void ***, int *); 145 static int link_elf_each_function_name(linker_file_t, 146 int (*)(const char *, void *), void *); 147 static int link_elf_each_function_nameval(linker_file_t, 148 linker_function_nameval_callback_t, 149 void *); 150 static int link_elf_reloc_local(linker_file_t, bool); 151 static long link_elf_symtab_get(linker_file_t, const Elf_Sym **); 152 static long link_elf_strtab_get(linker_file_t, caddr_t *); 153 #ifdef VIMAGE 154 static void link_elf_propagate_vnets(linker_file_t); 155 #endif 156 157 static int elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, 158 Elf_Addr *); 159 160 static kobj_method_t link_elf_methods[] = { 161 KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol), 162 KOBJMETHOD(linker_lookup_debug_symbol, link_elf_lookup_debug_symbol), 163 KOBJMETHOD(linker_symbol_values, link_elf_symbol_values), 164 KOBJMETHOD(linker_debug_symbol_values, link_elf_debug_symbol_values), 165 KOBJMETHOD(linker_search_symbol, link_elf_search_symbol), 166 KOBJMETHOD(linker_unload, link_elf_unload_file), 167 KOBJMETHOD(linker_load_file, link_elf_load_file), 168 KOBJMETHOD(linker_link_preload, link_elf_link_preload), 169 KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish), 170 KOBJMETHOD(linker_lookup_set, link_elf_lookup_set), 171 KOBJMETHOD(linker_each_function_name, link_elf_each_function_name), 172 KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval), 173 KOBJMETHOD(linker_ctf_get, link_elf_ctf_get), 174 KOBJMETHOD(linker_symtab_get, link_elf_symtab_get), 175 KOBJMETHOD(linker_strtab_get, link_elf_strtab_get), 176 #ifdef VIMAGE 177 KOBJMETHOD(linker_propagate_vnets, link_elf_propagate_vnets), 178 #endif 179 KOBJMETHOD_END 180 }; 181 182 static struct linker_class link_elf_class = { 183 #if ELF_TARG_CLASS == ELFCLASS32 184 "elf32_obj", 185 #else 186 "elf64_obj", 187 #endif 188 link_elf_methods, sizeof(struct elf_file) 189 }; 190 191 static bool link_elf_obj_leak_locals = true; 192 SYSCTL_BOOL(_debug, OID_AUTO, link_elf_obj_leak_locals, 193 CTLFLAG_RWTUN, &link_elf_obj_leak_locals, 0, 194 "Allow local symbols to participate in global module symbol resolution"); 195 196 static int relocate_file(elf_file_t ef); 197 static void elf_obj_cleanup_globals_cache(elf_file_t); 198 199 static void 200 link_elf_error(const char *filename, const char *s) 201 { 202 if (filename == NULL) 203 printf("kldload: %s\n", s); 204 else 205 printf("kldload: %s: %s\n", filename, s); 206 } 207 208 static void 209 link_elf_init(void *arg) 210 { 211 212 linker_add_class(&link_elf_class); 213 } 214 SYSINIT(link_elf_obj, SI_SUB_KLD, SI_ORDER_SECOND, link_elf_init, NULL); 215 216 static void 217 link_elf_protect_range(elf_file_t ef, vm_offset_t start, vm_offset_t end, 218 vm_prot_t prot) 219 { 220 int error __unused; 221 222 KASSERT(start <= end && start >= (vm_offset_t)ef->address && 223 end <= round_page((vm_offset_t)ef->address + ef->lf.size), 224 ("link_elf_protect_range: invalid range %#jx-%#jx", 225 (uintmax_t)start, (uintmax_t)end)); 226 227 if (start == end) 228 return; 229 if (ef->preloaded) { 230 #ifdef __amd64__ 231 error = pmap_change_prot(start, end - start, prot); 232 KASSERT(error == 0, 233 ("link_elf_protect_range: pmap_change_prot() returned %d", 234 error)); 235 #endif 236 return; 237 } 238 error = vm_map_protect(kernel_map, start, end, prot, 0, 239 VM_MAP_PROTECT_SET_PROT); 240 KASSERT(error == KERN_SUCCESS, 241 ("link_elf_protect_range: vm_map_protect() returned %d", error)); 242 } 243 244 /* 245 * Restrict permissions on linker file memory based on section flags. 246 * Sections need not be page-aligned, so overlap within a page is possible. 247 */ 248 static void 249 link_elf_protect(elf_file_t ef) 250 { 251 vm_offset_t end, segend, segstart, start; 252 vm_prot_t gapprot, prot, segprot; 253 int i; 254 255 /* 256 * If the file was preloaded, the last page may contain other preloaded 257 * data which may need to be writeable. ELF files are always 258 * page-aligned, but other preloaded data, such as entropy or CPU 259 * microcode may be loaded with a smaller alignment. 260 */ 261 gapprot = ef->preloaded ? VM_PROT_RW : VM_PROT_READ; 262 263 start = end = (vm_offset_t)ef->address; 264 prot = VM_PROT_READ; 265 for (i = 0; i < ef->nprogtab; i++) { 266 /* 267 * VNET and DPCPU sections have their memory allocated by their 268 * respective subsystems. 269 */ 270 if (ef->progtab[i].name != NULL && ( 271 #ifdef VIMAGE 272 strcmp(ef->progtab[i].name, VNET_SETNAME) == 0 || 273 #endif 274 strcmp(ef->progtab[i].name, DPCPU_SETNAME) == 0)) 275 continue; 276 277 segstart = trunc_page((vm_offset_t)ef->progtab[i].addr); 278 segend = round_page((vm_offset_t)ef->progtab[i].addr + 279 ef->progtab[i].size); 280 segprot = VM_PROT_READ; 281 if ((ef->progtab[i].flags & SHF_WRITE) != 0) 282 segprot |= VM_PROT_WRITE; 283 if ((ef->progtab[i].flags & SHF_EXECINSTR) != 0) 284 segprot |= VM_PROT_EXECUTE; 285 286 if (end <= segstart) { 287 /* 288 * Case 1: there is no overlap between the previous 289 * segment and this one. Apply protections to the 290 * previous segment, and protect the gap between the 291 * previous and current segments, if any. 292 */ 293 link_elf_protect_range(ef, start, end, prot); 294 link_elf_protect_range(ef, end, segstart, gapprot); 295 296 start = segstart; 297 end = segend; 298 prot = segprot; 299 } else if (start < segstart && end == segend) { 300 /* 301 * Case 2: the current segment is a subrange of the 302 * previous segment. Apply protections to the 303 * non-overlapping portion of the previous segment. 304 */ 305 link_elf_protect_range(ef, start, segstart, prot); 306 307 start = segstart; 308 prot |= segprot; 309 } else if (end < segend) { 310 /* 311 * Case 3: there is partial overlap between the previous 312 * and current segments. Apply protections to the 313 * non-overlapping portion of the previous segment, and 314 * then the overlap, which must use the union of the two 315 * segments' protections. 316 */ 317 link_elf_protect_range(ef, start, segstart, prot); 318 link_elf_protect_range(ef, segstart, end, 319 prot | segprot); 320 start = end; 321 end = segend; 322 prot = segprot; 323 } else { 324 /* 325 * Case 4: the two segments reside in the same page. 326 */ 327 prot |= segprot; 328 } 329 } 330 331 /* 332 * Fix up the last unprotected segment and trailing data. 333 */ 334 link_elf_protect_range(ef, start, end, prot); 335 link_elf_protect_range(ef, end, 336 round_page((vm_offset_t)ef->address + ef->lf.size), gapprot); 337 } 338 339 static int 340 link_elf_link_preload(linker_class_t cls, const char *filename, 341 linker_file_t *result) 342 { 343 Elf_Ehdr *hdr; 344 Elf_Shdr *shdr; 345 Elf_Sym *es; 346 void *modptr, *baseptr, *sizeptr; 347 char *type; 348 elf_file_t ef; 349 linker_file_t lf; 350 Elf_Addr off; 351 int error, i, j, pb, ra, rl, shstrindex, symstrindex, symtabindex; 352 353 /* Look to see if we have the file preloaded */ 354 modptr = preload_search_by_name(filename); 355 if (modptr == NULL) 356 return ENOENT; 357 358 type = (char *)preload_search_info(modptr, MODINFO_TYPE); 359 baseptr = preload_search_info(modptr, MODINFO_ADDR); 360 sizeptr = preload_search_info(modptr, MODINFO_SIZE); 361 hdr = (Elf_Ehdr *)preload_search_info(modptr, MODINFO_METADATA | 362 MODINFOMD_ELFHDR); 363 shdr = (Elf_Shdr *)preload_search_info(modptr, MODINFO_METADATA | 364 MODINFOMD_SHDR); 365 if (type == NULL || (strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) 366 " obj module") != 0 && 367 strcmp(type, "elf obj module") != 0)) { 368 return (EFTYPE); 369 } 370 if (baseptr == NULL || sizeptr == NULL || hdr == NULL || 371 shdr == NULL) 372 return (EINVAL); 373 374 lf = linker_make_file(filename, &link_elf_class); 375 if (lf == NULL) 376 return (ENOMEM); 377 378 ef = (elf_file_t)lf; 379 ef->preloaded = 1; 380 ef->address = *(caddr_t *)baseptr; 381 lf->address = *(caddr_t *)baseptr; 382 lf->size = *(size_t *)sizeptr; 383 384 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS || 385 hdr->e_ident[EI_DATA] != ELF_TARG_DATA || 386 hdr->e_ident[EI_VERSION] != EV_CURRENT || 387 hdr->e_version != EV_CURRENT || 388 hdr->e_type != ET_REL || 389 hdr->e_machine != ELF_TARG_MACH) { 390 error = EFTYPE; 391 goto out; 392 } 393 ef->e_shdr = shdr; 394 395 /* Scan the section header for information and table sizing. */ 396 symtabindex = -1; 397 symstrindex = -1; 398 for (i = 0; i < hdr->e_shnum; i++) { 399 switch (shdr[i].sh_type) { 400 case SHT_PROGBITS: 401 case SHT_NOBITS: 402 #ifdef __amd64__ 403 case SHT_X86_64_UNWIND: 404 #endif 405 case SHT_INIT_ARRAY: 406 case SHT_FINI_ARRAY: 407 /* Ignore sections not loaded by the loader. */ 408 if (shdr[i].sh_addr == 0) 409 break; 410 ef->nprogtab++; 411 break; 412 case SHT_SYMTAB: 413 symtabindex = i; 414 symstrindex = shdr[i].sh_link; 415 break; 416 case SHT_REL: 417 /* 418 * Ignore relocation tables for sections not 419 * loaded by the loader. 420 */ 421 if (shdr[shdr[i].sh_info].sh_addr == 0) 422 break; 423 ef->nreltab++; 424 break; 425 case SHT_RELA: 426 if (shdr[shdr[i].sh_info].sh_addr == 0) 427 break; 428 ef->nrelatab++; 429 break; 430 } 431 } 432 433 shstrindex = hdr->e_shstrndx; 434 if (ef->nprogtab == 0 || symstrindex < 0 || 435 symstrindex >= hdr->e_shnum || 436 shdr[symstrindex].sh_type != SHT_STRTAB || shstrindex == 0 || 437 shstrindex >= hdr->e_shnum || 438 shdr[shstrindex].sh_type != SHT_STRTAB) { 439 printf("%s: bad/missing section headers\n", filename); 440 error = ENOEXEC; 441 goto out; 442 } 443 444 /* Allocate space for tracking the load chunks */ 445 if (ef->nprogtab != 0) 446 ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab), 447 M_LINKER, M_WAITOK | M_ZERO); 448 if (ef->nreltab != 0) 449 ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab), 450 M_LINKER, M_WAITOK | M_ZERO); 451 if (ef->nrelatab != 0) 452 ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab), 453 M_LINKER, M_WAITOK | M_ZERO); 454 if ((ef->nprogtab != 0 && ef->progtab == NULL) || 455 (ef->nreltab != 0 && ef->reltab == NULL) || 456 (ef->nrelatab != 0 && ef->relatab == NULL)) { 457 error = ENOMEM; 458 goto out; 459 } 460 461 /* XXX, relocate the sh_addr fields saved by the loader. */ 462 off = 0; 463 for (i = 0; i < hdr->e_shnum; i++) { 464 if (shdr[i].sh_addr != 0 && (off == 0 || shdr[i].sh_addr < off)) 465 off = shdr[i].sh_addr; 466 } 467 for (i = 0; i < hdr->e_shnum; i++) { 468 if (shdr[i].sh_addr != 0) 469 shdr[i].sh_addr = shdr[i].sh_addr - off + 470 (Elf_Addr)ef->address; 471 } 472 473 ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); 474 ef->ddbsymtab = (Elf_Sym *)shdr[symtabindex].sh_addr; 475 ef->ddbstrcnt = shdr[symstrindex].sh_size; 476 ef->ddbstrtab = (char *)shdr[symstrindex].sh_addr; 477 ef->shstrcnt = shdr[shstrindex].sh_size; 478 ef->shstrtab = (char *)shdr[shstrindex].sh_addr; 479 480 /* Now fill out progtab and the relocation tables. */ 481 pb = 0; 482 rl = 0; 483 ra = 0; 484 for (i = 0; i < hdr->e_shnum; i++) { 485 switch (shdr[i].sh_type) { 486 case SHT_PROGBITS: 487 case SHT_NOBITS: 488 #ifdef __amd64__ 489 case SHT_X86_64_UNWIND: 490 #endif 491 case SHT_INIT_ARRAY: 492 case SHT_FINI_ARRAY: 493 if (shdr[i].sh_addr == 0) 494 break; 495 ef->progtab[pb].addr = (void *)shdr[i].sh_addr; 496 if (shdr[i].sh_type == SHT_PROGBITS) 497 ef->progtab[pb].name = "<<PROGBITS>>"; 498 #ifdef __amd64__ 499 else if (shdr[i].sh_type == SHT_X86_64_UNWIND) 500 ef->progtab[pb].name = "<<UNWIND>>"; 501 #endif 502 else if (shdr[i].sh_type == SHT_INIT_ARRAY) 503 ef->progtab[pb].name = "<<INIT_ARRAY>>"; 504 else if (shdr[i].sh_type == SHT_FINI_ARRAY) 505 ef->progtab[pb].name = "<<FINI_ARRAY>>"; 506 else 507 ef->progtab[pb].name = "<<NOBITS>>"; 508 ef->progtab[pb].size = shdr[i].sh_size; 509 ef->progtab[pb].flags = shdr[i].sh_flags; 510 ef->progtab[pb].sec = i; 511 if (ef->shstrtab && shdr[i].sh_name != 0) 512 ef->progtab[pb].name = 513 ef->shstrtab + shdr[i].sh_name; 514 if (ef->progtab[pb].name != NULL && 515 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) { 516 void *dpcpu; 517 518 dpcpu = dpcpu_alloc(shdr[i].sh_size); 519 if (dpcpu == NULL) { 520 printf("%s: pcpu module space is out " 521 "of space; cannot allocate %#jx " 522 "for %s\n", __func__, 523 (uintmax_t)shdr[i].sh_size, 524 filename); 525 error = ENOSPC; 526 goto out; 527 } 528 memcpy(dpcpu, ef->progtab[pb].addr, 529 ef->progtab[pb].size); 530 dpcpu_copy(dpcpu, shdr[i].sh_size); 531 ef->progtab[pb].addr = dpcpu; 532 #ifdef VIMAGE 533 } else if (ef->progtab[pb].name != NULL && 534 !strcmp(ef->progtab[pb].name, VNET_SETNAME)) { 535 void *vnet_data; 536 537 vnet_data = vnet_data_alloc(shdr[i].sh_size); 538 if (vnet_data == NULL) { 539 printf("%s: vnet module space is out " 540 "of space; cannot allocate %#jx " 541 "for %s\n", __func__, 542 (uintmax_t)shdr[i].sh_size, 543 filename); 544 error = ENOSPC; 545 goto out; 546 } 547 memcpy(vnet_data, ef->progtab[pb].addr, 548 ef->progtab[pb].size); 549 ef->progtab[pb].addr = vnet_data; 550 vnet_save_init(ef->progtab[pb].addr, 551 ef->progtab[pb].size); 552 #endif 553 } else if ((ef->progtab[pb].name != NULL && 554 strcmp(ef->progtab[pb].name, ".ctors") == 0) || 555 shdr[i].sh_type == SHT_INIT_ARRAY) { 556 if (lf->ctors_addr != 0) { 557 printf( 558 "%s: multiple ctor sections in %s\n", 559 __func__, filename); 560 } else { 561 lf->ctors_addr = ef->progtab[pb].addr; 562 lf->ctors_size = shdr[i].sh_size; 563 } 564 } else if ((ef->progtab[pb].name != NULL && 565 strcmp(ef->progtab[pb].name, ".dtors") == 0) || 566 shdr[i].sh_type == SHT_FINI_ARRAY) { 567 if (lf->dtors_addr != 0) { 568 printf( 569 "%s: multiple dtor sections in %s\n", 570 __func__, filename); 571 } else { 572 lf->dtors_addr = ef->progtab[pb].addr; 573 lf->dtors_size = shdr[i].sh_size; 574 } 575 } 576 577 /* Update all symbol values with the offset. */ 578 for (j = 0; j < ef->ddbsymcnt; j++) { 579 es = &ef->ddbsymtab[j]; 580 if (es->st_shndx != i) 581 continue; 582 es->st_value += (Elf_Addr)ef->progtab[pb].addr; 583 } 584 pb++; 585 break; 586 case SHT_REL: 587 if (shdr[shdr[i].sh_info].sh_addr == 0) 588 break; 589 ef->reltab[rl].rel = (Elf_Rel *)shdr[i].sh_addr; 590 ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel); 591 ef->reltab[rl].sec = shdr[i].sh_info; 592 rl++; 593 break; 594 case SHT_RELA: 595 if (shdr[shdr[i].sh_info].sh_addr == 0) 596 break; 597 ef->relatab[ra].rela = (Elf_Rela *)shdr[i].sh_addr; 598 ef->relatab[ra].nrela = 599 shdr[i].sh_size / sizeof(Elf_Rela); 600 ef->relatab[ra].sec = shdr[i].sh_info; 601 ra++; 602 break; 603 } 604 } 605 if (pb != ef->nprogtab) { 606 printf("%s: lost progbits\n", filename); 607 error = ENOEXEC; 608 goto out; 609 } 610 if (rl != ef->nreltab) { 611 printf("%s: lost reltab\n", filename); 612 error = ENOEXEC; 613 goto out; 614 } 615 if (ra != ef->nrelatab) { 616 printf("%s: lost relatab\n", filename); 617 error = ENOEXEC; 618 goto out; 619 } 620 621 /* 622 * The file needs to be writeable and executable while applying 623 * relocations. Mapping protections are applied once relocation 624 * processing is complete. 625 */ 626 link_elf_protect_range(ef, (vm_offset_t)ef->address, 627 round_page((vm_offset_t)ef->address + ef->lf.size), VM_PROT_ALL); 628 629 /* Local intra-module relocations */ 630 error = link_elf_reloc_local(lf, false); 631 if (error != 0) 632 goto out; 633 *result = lf; 634 return (0); 635 636 out: 637 /* preload not done this way */ 638 linker_file_unload(lf, LINKER_UNLOAD_FORCE); 639 return (error); 640 } 641 642 static void 643 link_elf_invoke_cbs(caddr_t addr, size_t size) 644 { 645 void (**ctor)(void); 646 size_t i, cnt; 647 648 if (addr == NULL || size == 0) 649 return; 650 cnt = size / sizeof(*ctor); 651 ctor = (void *)addr; 652 for (i = 0; i < cnt; i++) { 653 if (ctor[i] != NULL) 654 (*ctor[i])(); 655 } 656 } 657 658 static int 659 link_elf_link_preload_finish(linker_file_t lf) 660 { 661 elf_file_t ef; 662 int error; 663 664 ef = (elf_file_t)lf; 665 error = relocate_file(ef); 666 if (error) 667 return (error); 668 669 /* Notify MD code that a module is being loaded. */ 670 error = elf_cpu_load_file(lf); 671 if (error) 672 return (error); 673 674 #if defined(__i386__) || defined(__amd64__) 675 /* Now ifuncs. */ 676 error = link_elf_reloc_local(lf, true); 677 if (error != 0) 678 return (error); 679 #endif 680 681 /* Apply protections now that relocation processing is complete. */ 682 link_elf_protect(ef); 683 684 link_elf_invoke_cbs(lf->ctors_addr, lf->ctors_size); 685 return (0); 686 } 687 688 static int 689 link_elf_load_file(linker_class_t cls, const char *filename, 690 linker_file_t *result) 691 { 692 struct nameidata *nd; 693 struct thread *td = curthread; /* XXX */ 694 Elf_Ehdr *hdr; 695 Elf_Shdr *shdr; 696 Elf_Sym *es; 697 int nbytes, i, j; 698 vm_offset_t mapbase; 699 size_t mapsize; 700 int error = 0; 701 ssize_t resid; 702 int flags; 703 elf_file_t ef; 704 linker_file_t lf; 705 int symtabindex; 706 int symstrindex; 707 int shstrindex; 708 int nsym; 709 int pb, rl, ra; 710 int alignmask; 711 712 shdr = NULL; 713 lf = NULL; 714 mapsize = 0; 715 hdr = NULL; 716 717 nd = malloc(sizeof(struct nameidata), M_TEMP, M_WAITOK); 718 NDINIT(nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename); 719 flags = FREAD; 720 error = vn_open(nd, &flags, 0, NULL); 721 if (error) { 722 free(nd, M_TEMP); 723 return error; 724 } 725 NDFREE_PNBUF(nd); 726 if (nd->ni_vp->v_type != VREG) { 727 error = ENOEXEC; 728 goto out; 729 } 730 #ifdef MAC 731 error = mac_kld_check_load(td->td_ucred, nd->ni_vp); 732 if (error) { 733 goto out; 734 } 735 #endif 736 737 /* Read the elf header from the file. */ 738 hdr = malloc(sizeof(*hdr), M_LINKER, M_WAITOK); 739 error = vn_rdwr(UIO_READ, nd->ni_vp, (void *)hdr, sizeof(*hdr), 0, 740 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 741 &resid, td); 742 if (error) 743 goto out; 744 if (resid != 0){ 745 error = ENOEXEC; 746 goto out; 747 } 748 749 if (!IS_ELF(*hdr)) { 750 error = ENOEXEC; 751 goto out; 752 } 753 754 if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS 755 || hdr->e_ident[EI_DATA] != ELF_TARG_DATA) { 756 link_elf_error(filename, "Unsupported file layout"); 757 error = ENOEXEC; 758 goto out; 759 } 760 if (hdr->e_ident[EI_VERSION] != EV_CURRENT 761 || hdr->e_version != EV_CURRENT) { 762 link_elf_error(filename, "Unsupported file version"); 763 error = ENOEXEC; 764 goto out; 765 } 766 if (hdr->e_type != ET_REL) { 767 error = ENOSYS; 768 goto out; 769 } 770 if (hdr->e_machine != ELF_TARG_MACH) { 771 link_elf_error(filename, "Unsupported machine"); 772 error = ENOEXEC; 773 goto out; 774 } 775 776 lf = linker_make_file(filename, &link_elf_class); 777 if (!lf) { 778 error = ENOMEM; 779 goto out; 780 } 781 ef = (elf_file_t) lf; 782 ef->nprogtab = 0; 783 ef->e_shdr = 0; 784 ef->nreltab = 0; 785 ef->nrelatab = 0; 786 787 /* Allocate and read in the section header */ 788 nbytes = hdr->e_shnum * hdr->e_shentsize; 789 if (nbytes == 0 || hdr->e_shoff == 0 || 790 hdr->e_shentsize != sizeof(Elf_Shdr)) { 791 error = ENOEXEC; 792 goto out; 793 } 794 shdr = malloc(nbytes, M_LINKER, M_WAITOK); 795 ef->e_shdr = shdr; 796 error = vn_rdwr(UIO_READ, nd->ni_vp, (caddr_t)shdr, nbytes, 797 hdr->e_shoff, UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, 798 NOCRED, &resid, td); 799 if (error) 800 goto out; 801 if (resid) { 802 error = ENOEXEC; 803 goto out; 804 } 805 806 /* Scan the section header for information and table sizing. */ 807 nsym = 0; 808 symtabindex = -1; 809 symstrindex = -1; 810 for (i = 0; i < hdr->e_shnum; i++) { 811 if (shdr[i].sh_size == 0) 812 continue; 813 switch (shdr[i].sh_type) { 814 case SHT_PROGBITS: 815 case SHT_NOBITS: 816 #ifdef __amd64__ 817 case SHT_X86_64_UNWIND: 818 #endif 819 case SHT_INIT_ARRAY: 820 case SHT_FINI_ARRAY: 821 if ((shdr[i].sh_flags & SHF_ALLOC) == 0) 822 break; 823 ef->nprogtab++; 824 break; 825 case SHT_SYMTAB: 826 nsym++; 827 symtabindex = i; 828 symstrindex = shdr[i].sh_link; 829 break; 830 case SHT_REL: 831 /* 832 * Ignore relocation tables for unallocated 833 * sections. 834 */ 835 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0) 836 break; 837 ef->nreltab++; 838 break; 839 case SHT_RELA: 840 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0) 841 break; 842 ef->nrelatab++; 843 break; 844 case SHT_STRTAB: 845 break; 846 } 847 } 848 if (ef->nprogtab == 0) { 849 link_elf_error(filename, "file has no contents"); 850 error = ENOEXEC; 851 goto out; 852 } 853 if (nsym != 1) { 854 /* Only allow one symbol table for now */ 855 link_elf_error(filename, 856 "file must have exactly one symbol table"); 857 error = ENOEXEC; 858 goto out; 859 } 860 if (symstrindex < 0 || symstrindex > hdr->e_shnum || 861 shdr[symstrindex].sh_type != SHT_STRTAB) { 862 link_elf_error(filename, "file has invalid symbol strings"); 863 error = ENOEXEC; 864 goto out; 865 } 866 867 /* Allocate space for tracking the load chunks */ 868 if (ef->nprogtab != 0) 869 ef->progtab = malloc(ef->nprogtab * sizeof(*ef->progtab), 870 M_LINKER, M_WAITOK | M_ZERO); 871 if (ef->nreltab != 0) 872 ef->reltab = malloc(ef->nreltab * sizeof(*ef->reltab), 873 M_LINKER, M_WAITOK | M_ZERO); 874 if (ef->nrelatab != 0) 875 ef->relatab = malloc(ef->nrelatab * sizeof(*ef->relatab), 876 M_LINKER, M_WAITOK | M_ZERO); 877 878 if (symtabindex == -1) { 879 link_elf_error(filename, "lost symbol table index"); 880 error = ENOEXEC; 881 goto out; 882 } 883 /* Allocate space for and load the symbol table */ 884 ef->ddbsymcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); 885 ef->ddbsymtab = malloc(shdr[symtabindex].sh_size, M_LINKER, M_WAITOK); 886 error = vn_rdwr(UIO_READ, nd->ni_vp, (void *)ef->ddbsymtab, 887 shdr[symtabindex].sh_size, shdr[symtabindex].sh_offset, 888 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 889 &resid, td); 890 if (error) 891 goto out; 892 if (resid != 0){ 893 error = EINVAL; 894 goto out; 895 } 896 897 /* Allocate space for and load the symbol strings */ 898 ef->ddbstrcnt = shdr[symstrindex].sh_size; 899 ef->ddbstrtab = malloc(shdr[symstrindex].sh_size, M_LINKER, M_WAITOK); 900 error = vn_rdwr(UIO_READ, nd->ni_vp, ef->ddbstrtab, 901 shdr[symstrindex].sh_size, shdr[symstrindex].sh_offset, 902 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 903 &resid, td); 904 if (error) 905 goto out; 906 if (resid != 0){ 907 error = EINVAL; 908 goto out; 909 } 910 911 /* Do we have a string table for the section names? */ 912 shstrindex = -1; 913 if (hdr->e_shstrndx != 0 && 914 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { 915 shstrindex = hdr->e_shstrndx; 916 ef->shstrcnt = shdr[shstrindex].sh_size; 917 ef->shstrtab = malloc(shdr[shstrindex].sh_size, M_LINKER, 918 M_WAITOK); 919 error = vn_rdwr(UIO_READ, nd->ni_vp, ef->shstrtab, 920 shdr[shstrindex].sh_size, shdr[shstrindex].sh_offset, 921 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 922 &resid, td); 923 if (error) 924 goto out; 925 if (resid != 0){ 926 error = EINVAL; 927 goto out; 928 } 929 } 930 931 /* Size up code/data(progbits) and bss(nobits). */ 932 alignmask = 0; 933 for (i = 0; i < hdr->e_shnum; i++) { 934 if (shdr[i].sh_size == 0) 935 continue; 936 switch (shdr[i].sh_type) { 937 case SHT_PROGBITS: 938 case SHT_NOBITS: 939 #ifdef __amd64__ 940 case SHT_X86_64_UNWIND: 941 #endif 942 case SHT_INIT_ARRAY: 943 case SHT_FINI_ARRAY: 944 if ((shdr[i].sh_flags & SHF_ALLOC) == 0) 945 break; 946 alignmask = shdr[i].sh_addralign - 1; 947 mapsize += alignmask; 948 mapsize &= ~alignmask; 949 mapsize += shdr[i].sh_size; 950 break; 951 } 952 } 953 954 /* 955 * We know how much space we need for the text/data/bss/etc. 956 * This stuff needs to be in a single chunk so that profiling etc 957 * can get the bounds and gdb can associate offsets with modules 958 */ 959 ef->object = vm_pager_allocate(OBJT_PHYS, NULL, round_page(mapsize), 960 VM_PROT_ALL, 0, thread0.td_ucred); 961 if (ef->object == NULL) { 962 error = ENOMEM; 963 goto out; 964 } 965 #if VM_NRESERVLEVEL > 0 966 vm_object_color(ef->object, 0); 967 #endif 968 969 /* 970 * In order to satisfy amd64's architectural requirements on the 971 * location of code and data in the kernel's address space, request a 972 * mapping that is above the kernel. 973 * 974 * Protections will be restricted once relocations are applied. 975 */ 976 #ifdef __amd64__ 977 mapbase = KERNBASE; 978 #else 979 mapbase = VM_MIN_KERNEL_ADDRESS; 980 #endif 981 error = vm_map_find(kernel_map, ef->object, 0, &mapbase, 982 round_page(mapsize), 0, VMFS_OPTIMAL_SPACE, VM_PROT_ALL, 983 VM_PROT_ALL, 0); 984 if (error != KERN_SUCCESS) { 985 vm_object_deallocate(ef->object); 986 ef->object = NULL; 987 error = ENOMEM; 988 goto out; 989 } 990 991 /* Wire the pages */ 992 error = vm_map_wire(kernel_map, mapbase, 993 mapbase + round_page(mapsize), 994 VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES); 995 if (error != KERN_SUCCESS) { 996 error = ENOMEM; 997 goto out; 998 } 999 1000 /* Inform the kld system about the situation */ 1001 lf->address = ef->address = (caddr_t)mapbase; 1002 lf->size = mapsize; 1003 1004 /* 1005 * Now load code/data(progbits), zero bss(nobits), allocate space for 1006 * and load relocs 1007 */ 1008 pb = 0; 1009 rl = 0; 1010 ra = 0; 1011 alignmask = 0; 1012 for (i = 0; i < hdr->e_shnum; i++) { 1013 if (shdr[i].sh_size == 0) 1014 continue; 1015 switch (shdr[i].sh_type) { 1016 case SHT_PROGBITS: 1017 case SHT_NOBITS: 1018 #ifdef __amd64__ 1019 case SHT_X86_64_UNWIND: 1020 #endif 1021 case SHT_INIT_ARRAY: 1022 case SHT_FINI_ARRAY: 1023 if ((shdr[i].sh_flags & SHF_ALLOC) == 0) 1024 break; 1025 alignmask = shdr[i].sh_addralign - 1; 1026 mapbase += alignmask; 1027 mapbase &= ~alignmask; 1028 if (ef->shstrtab != NULL && shdr[i].sh_name != 0) { 1029 ef->progtab[pb].name = 1030 ef->shstrtab + shdr[i].sh_name; 1031 if (!strcmp(ef->progtab[pb].name, ".ctors") || 1032 shdr[i].sh_type == SHT_INIT_ARRAY) { 1033 if (lf->ctors_addr != 0) { 1034 printf( 1035 "%s: multiple ctor sections in %s\n", 1036 __func__, filename); 1037 } else { 1038 lf->ctors_addr = 1039 (caddr_t)mapbase; 1040 lf->ctors_size = 1041 shdr[i].sh_size; 1042 } 1043 } else if (!strcmp(ef->progtab[pb].name, 1044 ".dtors") || 1045 shdr[i].sh_type == SHT_FINI_ARRAY) { 1046 if (lf->dtors_addr != 0) { 1047 printf( 1048 "%s: multiple dtor sections in %s\n", 1049 __func__, filename); 1050 } else { 1051 lf->dtors_addr = 1052 (caddr_t)mapbase; 1053 lf->dtors_size = 1054 shdr[i].sh_size; 1055 } 1056 } 1057 } else if (shdr[i].sh_type == SHT_PROGBITS) 1058 ef->progtab[pb].name = "<<PROGBITS>>"; 1059 #ifdef __amd64__ 1060 else if (shdr[i].sh_type == SHT_X86_64_UNWIND) 1061 ef->progtab[pb].name = "<<UNWIND>>"; 1062 #endif 1063 else 1064 ef->progtab[pb].name = "<<NOBITS>>"; 1065 if (ef->progtab[pb].name != NULL && 1066 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) { 1067 ef->progtab[pb].addr = 1068 dpcpu_alloc(shdr[i].sh_size); 1069 if (ef->progtab[pb].addr == NULL) { 1070 printf("%s: pcpu module space is out " 1071 "of space; cannot allocate %#jx " 1072 "for %s\n", __func__, 1073 (uintmax_t)shdr[i].sh_size, 1074 filename); 1075 } 1076 } 1077 #ifdef VIMAGE 1078 else if (ef->progtab[pb].name != NULL && 1079 !strcmp(ef->progtab[pb].name, VNET_SETNAME)) { 1080 ef->progtab[pb].addr = 1081 vnet_data_alloc(shdr[i].sh_size); 1082 if (ef->progtab[pb].addr == NULL) { 1083 printf("%s: vnet module space is out " 1084 "of space; cannot allocate %#jx " 1085 "for %s\n", __func__, 1086 (uintmax_t)shdr[i].sh_size, 1087 filename); 1088 } 1089 } 1090 #endif 1091 else 1092 ef->progtab[pb].addr = 1093 (void *)(uintptr_t)mapbase; 1094 if (ef->progtab[pb].addr == NULL) { 1095 error = ENOSPC; 1096 goto out; 1097 } 1098 ef->progtab[pb].size = shdr[i].sh_size; 1099 ef->progtab[pb].flags = shdr[i].sh_flags; 1100 ef->progtab[pb].sec = i; 1101 if (shdr[i].sh_type == SHT_PROGBITS 1102 #ifdef __amd64__ 1103 || shdr[i].sh_type == SHT_X86_64_UNWIND 1104 #endif 1105 ) { 1106 error = vn_rdwr(UIO_READ, nd->ni_vp, 1107 ef->progtab[pb].addr, 1108 shdr[i].sh_size, shdr[i].sh_offset, 1109 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, 1110 NOCRED, &resid, td); 1111 if (error) 1112 goto out; 1113 if (resid != 0){ 1114 error = EINVAL; 1115 goto out; 1116 } 1117 /* Initialize the per-cpu area. */ 1118 if (ef->progtab[pb].addr != (void *)mapbase && 1119 !strcmp(ef->progtab[pb].name, DPCPU_SETNAME)) 1120 dpcpu_copy(ef->progtab[pb].addr, 1121 shdr[i].sh_size); 1122 } else 1123 bzero(ef->progtab[pb].addr, shdr[i].sh_size); 1124 1125 #ifdef VIMAGE 1126 if (ef->progtab[pb].addr != (void *)mapbase && 1127 strcmp(ef->progtab[pb].name, VNET_SETNAME) == 0) 1128 vnet_save_init(ef->progtab[pb].addr, 1129 ef->progtab[pb].size); 1130 #endif 1131 /* Update all symbol values with the offset. */ 1132 for (j = 0; j < ef->ddbsymcnt; j++) { 1133 es = &ef->ddbsymtab[j]; 1134 if (es->st_shndx != i) 1135 continue; 1136 es->st_value += (Elf_Addr)ef->progtab[pb].addr; 1137 } 1138 mapbase += shdr[i].sh_size; 1139 pb++; 1140 break; 1141 case SHT_REL: 1142 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0) 1143 break; 1144 ef->reltab[rl].rel = malloc(shdr[i].sh_size, M_LINKER, 1145 M_WAITOK); 1146 ef->reltab[rl].nrel = shdr[i].sh_size / sizeof(Elf_Rel); 1147 ef->reltab[rl].sec = shdr[i].sh_info; 1148 error = vn_rdwr(UIO_READ, nd->ni_vp, 1149 (void *)ef->reltab[rl].rel, 1150 shdr[i].sh_size, shdr[i].sh_offset, 1151 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1152 &resid, td); 1153 if (error) 1154 goto out; 1155 if (resid != 0){ 1156 error = EINVAL; 1157 goto out; 1158 } 1159 rl++; 1160 break; 1161 case SHT_RELA: 1162 if ((shdr[shdr[i].sh_info].sh_flags & SHF_ALLOC) == 0) 1163 break; 1164 ef->relatab[ra].rela = malloc(shdr[i].sh_size, M_LINKER, 1165 M_WAITOK); 1166 ef->relatab[ra].nrela = 1167 shdr[i].sh_size / sizeof(Elf_Rela); 1168 ef->relatab[ra].sec = shdr[i].sh_info; 1169 error = vn_rdwr(UIO_READ, nd->ni_vp, 1170 (void *)ef->relatab[ra].rela, 1171 shdr[i].sh_size, shdr[i].sh_offset, 1172 UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, 1173 &resid, td); 1174 if (error) 1175 goto out; 1176 if (resid != 0){ 1177 error = EINVAL; 1178 goto out; 1179 } 1180 ra++; 1181 break; 1182 } 1183 } 1184 if (pb != ef->nprogtab) { 1185 link_elf_error(filename, "lost progbits"); 1186 error = ENOEXEC; 1187 goto out; 1188 } 1189 if (rl != ef->nreltab) { 1190 link_elf_error(filename, "lost reltab"); 1191 error = ENOEXEC; 1192 goto out; 1193 } 1194 if (ra != ef->nrelatab) { 1195 link_elf_error(filename, "lost relatab"); 1196 error = ENOEXEC; 1197 goto out; 1198 } 1199 if (mapbase != (vm_offset_t)ef->address + mapsize) { 1200 printf( 1201 "%s: mapbase 0x%lx != address %p + mapsize 0x%lx (0x%lx)\n", 1202 filename != NULL ? filename : "<none>", 1203 (u_long)mapbase, ef->address, (u_long)mapsize, 1204 (u_long)(vm_offset_t)ef->address + mapsize); 1205 error = ENOMEM; 1206 goto out; 1207 } 1208 1209 /* Local intra-module relocations */ 1210 error = link_elf_reloc_local(lf, false); 1211 if (error != 0) 1212 goto out; 1213 1214 /* Pull in dependencies */ 1215 VOP_UNLOCK(nd->ni_vp); 1216 error = linker_load_dependencies(lf); 1217 vn_lock(nd->ni_vp, LK_EXCLUSIVE | LK_RETRY); 1218 if (error) 1219 goto out; 1220 1221 /* External relocations */ 1222 error = relocate_file(ef); 1223 if (error) 1224 goto out; 1225 1226 /* Notify MD code that a module is being loaded. */ 1227 error = elf_cpu_load_file(lf); 1228 if (error) 1229 goto out; 1230 1231 #if defined(__i386__) || defined(__amd64__) 1232 /* Now ifuncs. */ 1233 error = link_elf_reloc_local(lf, true); 1234 if (error != 0) 1235 goto out; 1236 #endif 1237 1238 link_elf_protect(ef); 1239 link_elf_invoke_cbs(lf->ctors_addr, lf->ctors_size); 1240 *result = lf; 1241 1242 out: 1243 VOP_UNLOCK(nd->ni_vp); 1244 vn_close(nd->ni_vp, FREAD, td->td_ucred, td); 1245 free(nd, M_TEMP); 1246 if (error && lf) 1247 linker_file_unload(lf, LINKER_UNLOAD_FORCE); 1248 free(hdr, M_LINKER); 1249 1250 return error; 1251 } 1252 1253 static void 1254 link_elf_unload_file(linker_file_t file) 1255 { 1256 elf_file_t ef = (elf_file_t) file; 1257 u_int i; 1258 1259 link_elf_invoke_cbs(file->dtors_addr, file->dtors_size); 1260 1261 /* Notify MD code that a module is being unloaded. */ 1262 elf_cpu_unload_file(file); 1263 1264 if (ef->progtab) { 1265 for (i = 0; i < ef->nprogtab; i++) { 1266 if (ef->progtab[i].size == 0) 1267 continue; 1268 if (ef->progtab[i].name == NULL) 1269 continue; 1270 if (!strcmp(ef->progtab[i].name, DPCPU_SETNAME)) 1271 dpcpu_free(ef->progtab[i].addr, 1272 ef->progtab[i].size); 1273 #ifdef VIMAGE 1274 else if (!strcmp(ef->progtab[i].name, VNET_SETNAME)) 1275 vnet_data_free(ef->progtab[i].addr, 1276 ef->progtab[i].size); 1277 #endif 1278 } 1279 } 1280 if (ef->preloaded) { 1281 free(ef->reltab, M_LINKER); 1282 free(ef->relatab, M_LINKER); 1283 free(ef->progtab, M_LINKER); 1284 free(ef->ctftab, M_LINKER); 1285 free(ef->ctfoff, M_LINKER); 1286 free(ef->typoff, M_LINKER); 1287 if (file->pathname != NULL) 1288 preload_delete_name(file->pathname); 1289 return; 1290 } 1291 1292 for (i = 0; i < ef->nreltab; i++) 1293 free(ef->reltab[i].rel, M_LINKER); 1294 for (i = 0; i < ef->nrelatab; i++) 1295 free(ef->relatab[i].rela, M_LINKER); 1296 free(ef->reltab, M_LINKER); 1297 free(ef->relatab, M_LINKER); 1298 free(ef->progtab, M_LINKER); 1299 1300 if (ef->object != NULL) 1301 vm_map_remove(kernel_map, (vm_offset_t)ef->address, 1302 (vm_offset_t)ef->address + ptoa(ef->object->size)); 1303 free(ef->e_shdr, M_LINKER); 1304 free(ef->ddbsymtab, M_LINKER); 1305 free(ef->ddbstrtab, M_LINKER); 1306 free(ef->shstrtab, M_LINKER); 1307 free(ef->ctftab, M_LINKER); 1308 free(ef->ctfoff, M_LINKER); 1309 free(ef->typoff, M_LINKER); 1310 } 1311 1312 static const char * 1313 symbol_name(elf_file_t ef, Elf_Size r_info) 1314 { 1315 const Elf_Sym *ref; 1316 1317 if (ELF_R_SYM(r_info)) { 1318 ref = ef->ddbsymtab + ELF_R_SYM(r_info); 1319 return ef->ddbstrtab + ref->st_name; 1320 } else 1321 return NULL; 1322 } 1323 1324 static Elf_Addr 1325 findbase(elf_file_t ef, int sec) 1326 { 1327 int i; 1328 Elf_Addr base = 0; 1329 1330 for (i = 0; i < ef->nprogtab; i++) { 1331 if (sec == ef->progtab[i].sec) { 1332 base = (Elf_Addr)ef->progtab[i].addr; 1333 break; 1334 } 1335 } 1336 return base; 1337 } 1338 1339 static int 1340 relocate_file1(elf_file_t ef, bool ifuncs) 1341 { 1342 const Elf_Rel *rellim; 1343 const Elf_Rel *rel; 1344 const Elf_Rela *relalim; 1345 const Elf_Rela *rela; 1346 const char *symname; 1347 const Elf_Sym *sym; 1348 int i; 1349 Elf_Size symidx; 1350 Elf_Addr base; 1351 1352 /* Perform relocations without addend if there are any: */ 1353 for (i = 0; i < ef->nreltab; i++) { 1354 rel = ef->reltab[i].rel; 1355 if (rel == NULL) { 1356 link_elf_error(ef->lf.filename, "lost a reltab!"); 1357 return (ENOEXEC); 1358 } 1359 rellim = rel + ef->reltab[i].nrel; 1360 base = findbase(ef, ef->reltab[i].sec); 1361 if (base == 0) { 1362 link_elf_error(ef->lf.filename, "lost base for reltab"); 1363 return (ENOEXEC); 1364 } 1365 for ( ; rel < rellim; rel++) { 1366 symidx = ELF_R_SYM(rel->r_info); 1367 if (symidx >= ef->ddbsymcnt) 1368 continue; 1369 sym = ef->ddbsymtab + symidx; 1370 /* Local relocs are already done */ 1371 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) 1372 continue; 1373 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC || 1374 elf_is_ifunc_reloc(rel->r_info)) != ifuncs) 1375 continue; 1376 if (elf_reloc(&ef->lf, base, rel, ELF_RELOC_REL, 1377 elf_obj_lookup)) { 1378 symname = symbol_name(ef, rel->r_info); 1379 printf("link_elf_obj: symbol %s undefined\n", 1380 symname); 1381 return (ENOENT); 1382 } 1383 } 1384 } 1385 1386 /* Perform relocations with addend if there are any: */ 1387 for (i = 0; i < ef->nrelatab; i++) { 1388 rela = ef->relatab[i].rela; 1389 if (rela == NULL) { 1390 link_elf_error(ef->lf.filename, "lost a relatab!"); 1391 return (ENOEXEC); 1392 } 1393 relalim = rela + ef->relatab[i].nrela; 1394 base = findbase(ef, ef->relatab[i].sec); 1395 if (base == 0) { 1396 link_elf_error(ef->lf.filename, 1397 "lost base for relatab"); 1398 return (ENOEXEC); 1399 } 1400 for ( ; rela < relalim; rela++) { 1401 symidx = ELF_R_SYM(rela->r_info); 1402 if (symidx >= ef->ddbsymcnt) 1403 continue; 1404 sym = ef->ddbsymtab + symidx; 1405 /* Local relocs are already done */ 1406 if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) 1407 continue; 1408 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC || 1409 elf_is_ifunc_reloc(rela->r_info)) != ifuncs) 1410 continue; 1411 if (elf_reloc(&ef->lf, base, rela, ELF_RELOC_RELA, 1412 elf_obj_lookup)) { 1413 symname = symbol_name(ef, rela->r_info); 1414 printf("link_elf_obj: symbol %s undefined\n", 1415 symname); 1416 return (ENOENT); 1417 } 1418 } 1419 } 1420 1421 /* 1422 * Only clean SHN_FBSD_CACHED for successful return. If we 1423 * modified symbol table for the object but found an 1424 * unresolved symbol, there is no reason to roll back. 1425 */ 1426 elf_obj_cleanup_globals_cache(ef); 1427 1428 return (0); 1429 } 1430 1431 static int 1432 relocate_file(elf_file_t ef) 1433 { 1434 int error; 1435 1436 error = relocate_file1(ef, false); 1437 if (error == 0) 1438 error = relocate_file1(ef, true); 1439 return (error); 1440 } 1441 1442 static int 1443 link_elf_lookup_symbol1(linker_file_t lf, const char *name, c_linker_sym_t *sym, 1444 bool see_local) 1445 { 1446 elf_file_t ef = (elf_file_t)lf; 1447 const Elf_Sym *symp; 1448 const char *strp; 1449 int i; 1450 1451 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1452 strp = ef->ddbstrtab + symp->st_name; 1453 if (symp->st_shndx != SHN_UNDEF && strcmp(name, strp) == 0) { 1454 if (see_local || 1455 ELF_ST_BIND(symp->st_info) == STB_GLOBAL) { 1456 *sym = (c_linker_sym_t) symp; 1457 return (0); 1458 } 1459 return (ENOENT); 1460 } 1461 } 1462 return (ENOENT); 1463 } 1464 1465 static int 1466 link_elf_lookup_symbol(linker_file_t lf, const char *name, c_linker_sym_t *sym) 1467 { 1468 return (link_elf_lookup_symbol1(lf, name, sym, 1469 link_elf_obj_leak_locals)); 1470 } 1471 1472 static int 1473 link_elf_lookup_debug_symbol(linker_file_t lf, const char *name, 1474 c_linker_sym_t *sym) 1475 { 1476 return (link_elf_lookup_symbol1(lf, name, sym, true)); 1477 } 1478 1479 static int 1480 link_elf_symbol_values1(linker_file_t lf, c_linker_sym_t sym, 1481 linker_symval_t *symval, bool see_local) 1482 { 1483 elf_file_t ef; 1484 const Elf_Sym *es; 1485 caddr_t val; 1486 1487 ef = (elf_file_t) lf; 1488 es = (const Elf_Sym*) sym; 1489 val = (caddr_t)es->st_value; 1490 if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) { 1491 if (!see_local && ELF_ST_BIND(es->st_info) == STB_LOCAL) 1492 return (ENOENT); 1493 symval->name = ef->ddbstrtab + es->st_name; 1494 val = (caddr_t)es->st_value; 1495 if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC) 1496 val = ((caddr_t (*)(void))val)(); 1497 symval->value = val; 1498 symval->size = es->st_size; 1499 return (0); 1500 } 1501 return (ENOENT); 1502 } 1503 1504 static int 1505 link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1506 linker_symval_t *symval) 1507 { 1508 return (link_elf_symbol_values1(lf, sym, symval, 1509 link_elf_obj_leak_locals)); 1510 } 1511 1512 static int 1513 link_elf_debug_symbol_values(linker_file_t lf, c_linker_sym_t sym, 1514 linker_symval_t *symval) 1515 { 1516 return (link_elf_symbol_values1(lf, sym, symval, true)); 1517 } 1518 1519 static int 1520 link_elf_search_symbol(linker_file_t lf, caddr_t value, 1521 c_linker_sym_t *sym, long *diffp) 1522 { 1523 elf_file_t ef = (elf_file_t)lf; 1524 u_long off = (uintptr_t)(void *)value; 1525 u_long diff = off; 1526 u_long st_value; 1527 const Elf_Sym *es; 1528 const Elf_Sym *best = NULL; 1529 int i; 1530 1531 for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) { 1532 if (es->st_name == 0) 1533 continue; 1534 st_value = es->st_value; 1535 if (off >= st_value) { 1536 if (off - st_value < diff) { 1537 diff = off - st_value; 1538 best = es; 1539 if (diff == 0) 1540 break; 1541 } else if (off - st_value == diff) { 1542 best = es; 1543 } 1544 } 1545 } 1546 if (best == NULL) 1547 *diffp = off; 1548 else 1549 *diffp = diff; 1550 *sym = (c_linker_sym_t) best; 1551 1552 return (0); 1553 } 1554 1555 /* 1556 * Look up a linker set on an ELF system. 1557 */ 1558 static int 1559 link_elf_lookup_set(linker_file_t lf, const char *name, 1560 void ***startp, void ***stopp, int *countp) 1561 { 1562 elf_file_t ef = (elf_file_t)lf; 1563 void **start, **stop; 1564 int i, count; 1565 1566 /* Relative to section number */ 1567 for (i = 0; i < ef->nprogtab; i++) { 1568 if ((strncmp(ef->progtab[i].name, "set_", 4) == 0) && 1569 strcmp(ef->progtab[i].name + 4, name) == 0) { 1570 start = (void **)ef->progtab[i].addr; 1571 stop = (void **)((char *)ef->progtab[i].addr + 1572 ef->progtab[i].size); 1573 count = stop - start; 1574 if (startp) 1575 *startp = start; 1576 if (stopp) 1577 *stopp = stop; 1578 if (countp) 1579 *countp = count; 1580 return (0); 1581 } 1582 } 1583 return (ESRCH); 1584 } 1585 1586 static int 1587 link_elf_each_function_name(linker_file_t file, 1588 int (*callback)(const char *, void *), void *opaque) 1589 { 1590 elf_file_t ef = (elf_file_t)file; 1591 const Elf_Sym *symp; 1592 int i, error; 1593 1594 /* Exhaustive search */ 1595 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1596 if (symp->st_value != 0 && 1597 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1598 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1599 error = callback(ef->ddbstrtab + symp->st_name, opaque); 1600 if (error) 1601 return (error); 1602 } 1603 } 1604 return (0); 1605 } 1606 1607 static int 1608 link_elf_each_function_nameval(linker_file_t file, 1609 linker_function_nameval_callback_t callback, void *opaque) 1610 { 1611 linker_symval_t symval; 1612 elf_file_t ef = (elf_file_t)file; 1613 const Elf_Sym *symp; 1614 int i, error; 1615 1616 /* Exhaustive search */ 1617 for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) { 1618 if (symp->st_value != 0 && 1619 (ELF_ST_TYPE(symp->st_info) == STT_FUNC || 1620 ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) { 1621 error = link_elf_debug_symbol_values(file, 1622 (c_linker_sym_t)symp, &symval); 1623 if (error == 0) 1624 error = callback(file, i, &symval, opaque); 1625 if (error != 0) 1626 return (error); 1627 } 1628 } 1629 return (0); 1630 } 1631 1632 static void 1633 elf_obj_cleanup_globals_cache(elf_file_t ef) 1634 { 1635 Elf_Sym *sym; 1636 Elf_Size i; 1637 1638 for (i = 0; i < ef->ddbsymcnt; i++) { 1639 sym = ef->ddbsymtab + i; 1640 if (sym->st_shndx == SHN_FBSD_CACHED) { 1641 sym->st_shndx = SHN_UNDEF; 1642 sym->st_value = 0; 1643 } 1644 } 1645 } 1646 1647 /* 1648 * Symbol lookup function that can be used when the symbol index is known (ie 1649 * in relocations). It uses the symbol index instead of doing a fully fledged 1650 * hash table based lookup when such is valid. For example for local symbols. 1651 * This is not only more efficient, it's also more correct. It's not always 1652 * the case that the symbol can be found through the hash table. 1653 */ 1654 static int 1655 elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res) 1656 { 1657 elf_file_t ef = (elf_file_t)lf; 1658 Elf_Sym *sym; 1659 const char *symbol; 1660 Elf_Addr res1; 1661 1662 /* Don't even try to lookup the symbol if the index is bogus. */ 1663 if (symidx >= ef->ddbsymcnt) { 1664 *res = 0; 1665 return (EINVAL); 1666 } 1667 1668 sym = ef->ddbsymtab + symidx; 1669 1670 /* Quick answer if there is a definition included. */ 1671 if (sym->st_shndx != SHN_UNDEF) { 1672 res1 = (Elf_Addr)sym->st_value; 1673 if (ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC) 1674 res1 = ((Elf_Addr (*)(void))res1)(); 1675 *res = res1; 1676 return (0); 1677 } 1678 1679 /* If we get here, then it is undefined and needs a lookup. */ 1680 switch (ELF_ST_BIND(sym->st_info)) { 1681 case STB_LOCAL: 1682 /* Local, but undefined? huh? */ 1683 *res = 0; 1684 return (EINVAL); 1685 1686 case STB_GLOBAL: 1687 case STB_WEAK: 1688 /* Relative to Data or Function name */ 1689 symbol = ef->ddbstrtab + sym->st_name; 1690 1691 /* Force a lookup failure if the symbol name is bogus. */ 1692 if (*symbol == 0) { 1693 *res = 0; 1694 return (EINVAL); 1695 } 1696 res1 = (Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps); 1697 1698 /* 1699 * Cache global lookups during module relocation. The failure 1700 * case is particularly expensive for callers, who must scan 1701 * through the entire globals table doing strcmp(). Cache to 1702 * avoid doing such work repeatedly. 1703 * 1704 * After relocation is complete, undefined globals will be 1705 * restored to SHN_UNDEF in elf_obj_cleanup_globals_cache(), 1706 * above. 1707 */ 1708 if (res1 != 0) { 1709 sym->st_shndx = SHN_FBSD_CACHED; 1710 sym->st_value = res1; 1711 *res = res1; 1712 return (0); 1713 } else if (ELF_ST_BIND(sym->st_info) == STB_WEAK) { 1714 sym->st_value = 0; 1715 *res = 0; 1716 return (0); 1717 } 1718 return (EINVAL); 1719 1720 default: 1721 return (EINVAL); 1722 } 1723 } 1724 1725 static void 1726 link_elf_fix_link_set(elf_file_t ef) 1727 { 1728 static const char startn[] = "__start_"; 1729 static const char stopn[] = "__stop_"; 1730 Elf_Sym *sym; 1731 const char *sym_name, *linkset_name; 1732 Elf_Addr startp, stopp; 1733 Elf_Size symidx; 1734 int start, i; 1735 1736 startp = stopp = 0; 1737 for (symidx = 1 /* zero entry is special */; 1738 symidx < ef->ddbsymcnt; symidx++) { 1739 sym = ef->ddbsymtab + symidx; 1740 if (sym->st_shndx != SHN_UNDEF) 1741 continue; 1742 1743 sym_name = ef->ddbstrtab + sym->st_name; 1744 if (strncmp(sym_name, startn, sizeof(startn) - 1) == 0) { 1745 start = 1; 1746 linkset_name = sym_name + sizeof(startn) - 1; 1747 } 1748 else if (strncmp(sym_name, stopn, sizeof(stopn) - 1) == 0) { 1749 start = 0; 1750 linkset_name = sym_name + sizeof(stopn) - 1; 1751 } 1752 else 1753 continue; 1754 1755 for (i = 0; i < ef->nprogtab; i++) { 1756 if (strcmp(ef->progtab[i].name, linkset_name) == 0) { 1757 startp = (Elf_Addr)ef->progtab[i].addr; 1758 stopp = (Elf_Addr)(startp + ef->progtab[i].size); 1759 break; 1760 } 1761 } 1762 if (i == ef->nprogtab) 1763 continue; 1764 1765 sym->st_value = start ? startp : stopp; 1766 sym->st_shndx = i; 1767 } 1768 } 1769 1770 static int 1771 link_elf_reloc_local(linker_file_t lf, bool ifuncs) 1772 { 1773 elf_file_t ef = (elf_file_t)lf; 1774 const Elf_Rel *rellim; 1775 const Elf_Rel *rel; 1776 const Elf_Rela *relalim; 1777 const Elf_Rela *rela; 1778 const Elf_Sym *sym; 1779 Elf_Addr base; 1780 int i; 1781 Elf_Size symidx; 1782 1783 link_elf_fix_link_set(ef); 1784 1785 /* Perform relocations without addend if there are any: */ 1786 for (i = 0; i < ef->nreltab; i++) { 1787 rel = ef->reltab[i].rel; 1788 if (rel == NULL) { 1789 link_elf_error(ef->lf.filename, "lost a reltab"); 1790 return (ENOEXEC); 1791 } 1792 rellim = rel + ef->reltab[i].nrel; 1793 base = findbase(ef, ef->reltab[i].sec); 1794 if (base == 0) { 1795 link_elf_error(ef->lf.filename, "lost base for reltab"); 1796 return (ENOEXEC); 1797 } 1798 for ( ; rel < rellim; rel++) { 1799 symidx = ELF_R_SYM(rel->r_info); 1800 if (symidx >= ef->ddbsymcnt) 1801 continue; 1802 sym = ef->ddbsymtab + symidx; 1803 /* Only do local relocs */ 1804 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL) 1805 continue; 1806 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC || 1807 elf_is_ifunc_reloc(rel->r_info)) != ifuncs) 1808 continue; 1809 if (elf_reloc_local(lf, base, rel, ELF_RELOC_REL, 1810 elf_obj_lookup) != 0) 1811 return (ENOEXEC); 1812 } 1813 } 1814 1815 /* Perform relocations with addend if there are any: */ 1816 for (i = 0; i < ef->nrelatab; i++) { 1817 rela = ef->relatab[i].rela; 1818 if (rela == NULL) { 1819 link_elf_error(ef->lf.filename, "lost a relatab!"); 1820 return (ENOEXEC); 1821 } 1822 relalim = rela + ef->relatab[i].nrela; 1823 base = findbase(ef, ef->relatab[i].sec); 1824 if (base == 0) { 1825 link_elf_error(ef->lf.filename, "lost base for reltab"); 1826 return (ENOEXEC); 1827 } 1828 for ( ; rela < relalim; rela++) { 1829 symidx = ELF_R_SYM(rela->r_info); 1830 if (symidx >= ef->ddbsymcnt) 1831 continue; 1832 sym = ef->ddbsymtab + symidx; 1833 /* Only do local relocs */ 1834 if (ELF_ST_BIND(sym->st_info) != STB_LOCAL) 1835 continue; 1836 if ((ELF_ST_TYPE(sym->st_info) == STT_GNU_IFUNC || 1837 elf_is_ifunc_reloc(rela->r_info)) != ifuncs) 1838 continue; 1839 if (elf_reloc_local(lf, base, rela, ELF_RELOC_RELA, 1840 elf_obj_lookup) != 0) 1841 return (ENOEXEC); 1842 } 1843 } 1844 return (0); 1845 } 1846 1847 static long 1848 link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab) 1849 { 1850 elf_file_t ef = (elf_file_t)lf; 1851 1852 *symtab = ef->ddbsymtab; 1853 if (*symtab == NULL) 1854 return (0); 1855 return (ef->ddbsymcnt); 1856 } 1857 1858 static long 1859 link_elf_strtab_get(linker_file_t lf, caddr_t *strtab) 1860 { 1861 elf_file_t ef = (elf_file_t)lf; 1862 1863 *strtab = ef->ddbstrtab; 1864 if (*strtab == NULL) 1865 return (0); 1866 return (ef->ddbstrcnt); 1867 } 1868 1869 #ifdef VIMAGE 1870 static void 1871 link_elf_propagate_vnets(linker_file_t lf) 1872 { 1873 elf_file_t ef = (elf_file_t) lf; 1874 1875 if (ef->progtab) { 1876 for (int i = 0; i < ef->nprogtab; i++) { 1877 if (ef->progtab[i].size == 0) 1878 continue; 1879 if (ef->progtab[i].name == NULL) 1880 continue; 1881 if (strcmp(ef->progtab[i].name, VNET_SETNAME) == 0) { 1882 vnet_data_copy(ef->progtab[i].addr, 1883 ef->progtab[i].size); 1884 break; 1885 } 1886 } 1887 } 1888 } 1889 #endif 1890