216 static int	relocate_file1(elf_file_t ef, elf_lookup_fn lookup,
395 	elf_file_t ef = (elf_file_t)lf;  in link_elf_link_common_finish()  local
407 	ef->gdb.l_addr = lf->address;  in link_elf_link_common_finish()
410 	ef->gdb.l_name = newfilename;  in link_elf_link_common_finish()
411 	ef->gdb.l_ld = ef->dynamic;  in link_elf_link_common_finish()
412 	link_elf_add_gdb(&ef->gdb);  in link_elf_link_common_finish()
454 	elf_file_t ef;  in link_elf_init()  local
468 	ef = (elf_file_t) linker_kernel_file;  in link_elf_init()
469 	ef->preloaded = 1;  in link_elf_init()
472 	ef->address = (caddr_t) (__startkernel - KERNBASE);  in link_elf_init()
474 	ef->address = 0;  in link_elf_init()
477 	ef->object = NULL;  in link_elf_init()
479 	ef->dynamic = dp;  in link_elf_init()
482 		parse_dynamic(ef);  in link_elf_init()
493 		ef->modptr = preload_kmdp;  in link_elf_init()
505 			linker_kernel_file->ctors_addr = ef->address +  in link_elf_init()
510 	(void)link_elf_preload_parse_symbols(ef);  in link_elf_init()
530 link_elf_preload_parse_symbols(elf_file_t ef)  in link_elf_preload_parse_symbols()  argument
539 	if (ef->modptr == NULL)  in link_elf_preload_parse_symbols()
541 	pointer = preload_search_info(ef->modptr,  in link_elf_preload_parse_symbols()
546 	pointer = preload_search_info(ef->modptr,  in link_elf_preload_parse_symbols()
574 	ef->ddbsymtab = symtab;  in link_elf_preload_parse_symbols()
575 	ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);  in link_elf_preload_parse_symbols()
576 	ef->ddbstrtab = strtab;  in link_elf_preload_parse_symbols()
577 	ef->ddbstrcnt = strcnt;  in link_elf_preload_parse_symbols()
583 parse_dynamic(elf_file_t ef)  in parse_dynamic()  argument
588 	for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {  in parse_dynamic()
594 			    (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
595 			ef->nbuckets = hashtab[0];  in parse_dynamic()
596 			ef->nchains = hashtab[1];  in parse_dynamic()
597 			ef->buckets = hashtab + 2;  in parse_dynamic()
598 			ef->chains = ef->buckets + ef->nbuckets;  in parse_dynamic()
602 			ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
605 			ef->strsz = dp->d_un.d_val;  in parse_dynamic()
608 			ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
615 			ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
618 			ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
621 			ef->relsize = dp->d_un.d_val;  in parse_dynamic()
628 			ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
631 			ef->pltrelsize = dp->d_un.d_val;  in parse_dynamic()
634 			ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
637 			ef->relasize = dp->d_un.d_val;  in parse_dynamic()
657 		ef->pltrela = (const Elf_Rela *)ef->pltrel;  in parse_dynamic()
658 		ef->pltrel = NULL;  in parse_dynamic()
659 		ef->pltrelasize = ef->pltrelsize;  in parse_dynamic()
660 		ef->pltrelsize = 0;  in parse_dynamic()
663 	ef->ddbsymtab = ef->symtab;  in parse_dynamic()
664 	ef->ddbsymcnt = ef->nchains;  in parse_dynamic()
665 	ef->ddbstrtab = ef->strtab;  in parse_dynamic()
666 	ef->ddbstrcnt = ef->strsz;  in parse_dynamic()
668 	return elf_cpu_parse_dynamic(ef->address, ef->dynamic);  in parse_dynamic()
673 parse_dpcpu(elf_file_t ef)  in parse_dpcpu()  argument
680 	ef->pcpu_start = 0;  in parse_dpcpu()
681 	ef->pcpu_stop = 0;  in parse_dpcpu()
682 	error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start,  in parse_dpcpu()
683 	    (void ***)&ef->pcpu_stop, NULL);  in parse_dpcpu()
687 	size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start;  in parse_dpcpu()
695 		    "linker script\n", ef->lf.pathname);  in parse_dpcpu()
700 	pad = *(uint32_t *)((uintptr_t)ef->pcpu_stop - sizeof(pad));  in parse_dpcpu()
704 		    ef->lf.pathname, pad, LS_PADDING);  in parse_dpcpu()
716 	ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size);  in parse_dpcpu()
717 	if (ef->pcpu_base == 0) {  in parse_dpcpu()
720 		    __func__, size, ef->lf.pathname);  in parse_dpcpu()
723 	memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size);  in parse_dpcpu()
724 	dpcpu_copy((void *)ef->pcpu_base, size);  in parse_dpcpu()
725 	elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop,  in parse_dpcpu()
726 	    ef->pcpu_base);  in parse_dpcpu()
733 parse_vnet(elf_file_t ef)  in parse_vnet()  argument
740 	ef->vnet_start = 0;  in parse_vnet()
741 	ef->vnet_stop = 0;  in parse_vnet()
742 	ef->vnet_base = 0;  in parse_vnet()
743 	error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start,  in parse_vnet()
744 	    (void ***)&ef->vnet_stop, NULL);  in parse_vnet()
748 	size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start;  in parse_vnet()
756 		    "linker script\n", ef->lf.pathname);  in parse_vnet()
761 	pad = *(uint32_t *)((uintptr_t)ef->vnet_stop - sizeof(pad));  in parse_vnet()
765 		    ef->lf.pathname, pad, LS_PADDING);  in parse_vnet()
777 	ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size);  in parse_vnet()
778 	if (ef->vnet_base == 0) {  in parse_vnet()
781 		    __func__, size, ef->lf.pathname);  in parse_vnet()
784 	memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size);  in parse_vnet()
785 	vnet_save_init((void *)ef->vnet_base, size);  in parse_vnet()
786 	elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop,  in parse_vnet()
787 	    ef->vnet_base);  in parse_vnet()
799 preload_protect(elf_file_t ef, vm_prot_t prot)  in preload_protect()  argument
808 	hdr = (Elf_Ehdr *)ef->address;  in preload_protect()
809 	phdr = (Elf_Phdr *)(ef->address + hdr->e_phoff);  in preload_protect()
820 		error = pmap_change_prot((vm_offset_t)ef->address +  in preload_protect()
883 	elf_file_t ef;  in link_elf_link_preload()  local
907 	ef = (elf_file_t) lf;  in link_elf_link_preload()
908 	ef->preloaded = 1;  in link_elf_link_preload()
909 	ef->modptr = modptr;  in link_elf_link_preload()
910 	ef->address = *(caddr_t *)baseptr;  in link_elf_link_preload()
912 	ef->object = NULL;  in link_elf_link_preload()
914 	dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr;  in link_elf_link_preload()
915 	ef->dynamic = (Elf_Dyn *)dp;  in link_elf_link_preload()
916 	lf->address = ef->address;  in link_elf_link_preload()
924 		lf->ctors_addr = ef->address + *ctors_addrp;  in link_elf_link_preload()
932 	error = parse_dynamic(ef);  in link_elf_link_preload()
934 		error = parse_dpcpu(ef);  in link_elf_link_preload()
937 		error = parse_vnet(ef);  in link_elf_link_preload()
940 		error = preload_protect(ef, VM_PROT_ALL);  in link_elf_link_preload()
953 	elf_file_t ef;  in link_elf_link_preload_finish()  local
956 	ef = (elf_file_t) lf;  in link_elf_link_preload_finish()
957 	error = relocate_file(ef);  in link_elf_link_preload_finish()
959 		error = preload_protect(ef, VM_PROT_NONE);  in link_elf_link_preload_finish()
962 	(void)link_elf_preload_parse_symbols(ef);  in link_elf_link_preload_finish()
988 	elf_file_t ef;  in link_elf_load_file()  local
1134 	ef = (elf_file_t) lf;  in link_elf_load_file()
1136 	ef->object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL,  in link_elf_load_file()
1138 	if (ef->object == NULL) {  in link_elf_load_file()
1150 	error = vm_map_find(kernel_map, ef->object, 0,  in link_elf_load_file()
1154 		vm_object_deallocate(ef->object);  in link_elf_load_file()
1155 		ef->object = NULL;  in link_elf_load_file()
1161 	ef->address = mapbase;  in link_elf_load_file()
1199 	ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);  in link_elf_load_file()
1201 	lf->address = ef->address;  in link_elf_load_file()
1204 	error = parse_dynamic(ef);  in link_elf_load_file()
1207 	error = parse_dpcpu(ef);  in link_elf_load_file()
1211 	error = parse_vnet(ef);  in link_elf_load_file()
1222 	error = relocate_file(ef);  in link_elf_load_file()
1299 	ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK);  in link_elf_load_file()
1301 	ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK);  in link_elf_load_file()
1304 	    ef->symbase, symcnt, shdr[symtabindex].sh_offset,  in link_elf_load_file()
1310 	    ef->strbase, strcnt, shdr[symstrindex].sh_offset,  in link_elf_load_file()
1316 	ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);  in link_elf_load_file()
1317 	ef->ddbsymtab = (const Elf_Sym *)ef->symbase;  in link_elf_load_file()
1318 	ef->ddbstrcnt = strcnt;  in link_elf_load_file()
1319 	ef->ddbstrtab = ef->strbase;  in link_elf_load_file()
1348 	elf_file_t ef;  in elf_relocaddr()  local
1353 	ef = (elf_file_t)lf;  in elf_relocaddr()
1354 	if (x >= ef->pcpu_start && x < ef->pcpu_stop)  in elf_relocaddr()
1355 		return ((x - ef->pcpu_start) + ef->pcpu_base);  in elf_relocaddr()
1357 	if (x >= ef->vnet_start && x < ef->vnet_stop)  in elf_relocaddr()
1358 		return ((x - ef->vnet_start) + ef->vnet_base);  in elf_relocaddr()
1366 	elf_file_t ef = (elf_file_t) file;  in link_elf_unload_file()  local
1368 	if (ef->pcpu_base != 0) {  in link_elf_unload_file()
1369 		dpcpu_free((void *)ef->pcpu_base,  in link_elf_unload_file()
1370 		    ef->pcpu_stop - ef->pcpu_start);  in link_elf_unload_file()
1371 		elf_set_delete(&set_pcpu_list, ef->pcpu_start);  in link_elf_unload_file()
1374 	if (ef->vnet_base != 0) {  in link_elf_unload_file()
1375 		vnet_data_free((void *)ef->vnet_base,  in link_elf_unload_file()
1376 		    ef->vnet_stop - ef->vnet_start);  in link_elf_unload_file()
1377 		elf_set_delete(&set_vnet_list, ef->vnet_start);  in link_elf_unload_file()
1381 	if (ef->gdb.l_ld != NULL) {  in link_elf_unload_file()
1383 		free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER);  in link_elf_unload_file()
1384 		link_elf_delete_gdb(&ef->gdb);  in link_elf_unload_file()
1392 	if (ef->preloaded) {  in link_elf_unload_file()
1398 	if (ef->object != NULL) {  in link_elf_unload_file()
1399 		vm_map_remove(kernel_map, (vm_offset_t) ef->address,  in link_elf_unload_file()
1400 		    (vm_offset_t) ef->address  in link_elf_unload_file()
1401 		    + (ef->object->size << PAGE_SHIFT));  in link_elf_unload_file()
1404 	free(ef->address, M_LINKER);  in link_elf_unload_file()
1406 	free(ef->symbase, M_LINKER);  in link_elf_unload_file()
1407 	free(ef->strbase, M_LINKER);  in link_elf_unload_file()
1408 	free(ef->ctftab, M_LINKER);  in link_elf_unload_file()
1409 	free(ef->ctfoff, M_LINKER);  in link_elf_unload_file()
1410 	free(ef->typoff, M_LINKER);  in link_elf_unload_file()
1422 symbol_name(elf_file_t ef, Elf_Size r_info)  in symbol_name()  argument
1427 		ref = ef->symtab + ELF_R_SYM(r_info);  in symbol_name()
1428 		return (ef->strtab + ref->st_name);  in symbol_name()
1434 symbol_type(elf_file_t ef, Elf_Size r_info)  in symbol_type()  argument
1439 		ref = ef->symtab + ELF_R_SYM(r_info);  in symbol_type()
1446 relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc,  in relocate_file1()  argument
1457 		if ((symbol_type(ef, (iter)->r_info) ==			\  in relocate_file1()
1461 		if (reloc(&ef->lf, (Elf_Addr)ef->address,		\  in relocate_file1()
1463 			symname = symbol_name(ef, (iter)->r_info);	\  in relocate_file1()
1471 	APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL);  in relocate_file1()
1473 	APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA);  in relocate_file1()
1475 	APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL);  in relocate_file1()
1476 	APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA);  in relocate_file1()
1485 relocate_file(elf_file_t ef)  in relocate_file()  argument
1489 	error = relocate_file1(ef, elf_lookup, elf_reloc, false);  in relocate_file()
1491 		error = relocate_file1(ef, elf_lookup, elf_reloc, true);  in relocate_file()
1516 	elf_file_t ef = (elf_file_t) lf;  in link_elf_lookup_symbol1()  local
1523 	if (ef->buckets == NULL || ef->nbuckets == 0) {  in link_elf_lookup_symbol1()
1530 	symnum = ef->buckets[hash % ef->nbuckets];  in link_elf_lookup_symbol1()
1533 		if (symnum >= ef->nchains) {  in link_elf_lookup_symbol1()
1538 		symp = ef->symtab + symnum;  in link_elf_lookup_symbol1()
1544 		strp = ef->strtab + symp->st_name;  in link_elf_lookup_symbol1()
1560 		symnum = ef->chains[symnum];  in link_elf_lookup_symbol1()
1578 	elf_file_t ef = (elf_file_t)lf;  in link_elf_lookup_debug_symbol()  local
1586 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_lookup_debug_symbol()
1587 		strp = ef->ddbstrtab + symp->st_name;  in link_elf_lookup_debug_symbol()
1607 	elf_file_t ef = (elf_file_t)lf;  in link_elf_lookup_debug_symbol_ctf()  local
1612 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_lookup_debug_symbol_ctf()
1613 		strp = ef->ddbstrtab + symp->st_name;  in link_elf_lookup_debug_symbol_ctf()
1628 	return (i < ef->ddbsymcnt ? link_elf_ctf_get_ddb(lf, lc) : ENOENT);  in link_elf_lookup_debug_symbol_ctf()
1635 	elf_file_t ef;  in link_elf_symbol_values1()  local
1639 	ef = (elf_file_t)lf;  in link_elf_symbol_values1()
1641 	if (es >= ef->symtab && es < ef->symtab + ef->nchains) {  in link_elf_symbol_values1()
1644 		symval->name = ef->strtab + es->st_name;  in link_elf_symbol_values1()
1645 		val = (caddr_t)ef->address + es->st_value;  in link_elf_symbol_values1()
1668 	elf_file_t ef = (elf_file_t)lf;  in link_elf_debug_symbol_values()  local
1674 	if (ef->symtab == ef->ddbsymtab)  in link_elf_debug_symbol_values()
1677 	if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {  in link_elf_debug_symbol_values()
1678 		symval->name = ef->ddbstrtab + es->st_name;  in link_elf_debug_symbol_values()
1679 		val = (caddr_t)ef->address + es->st_value;  in link_elf_debug_symbol_values()
1693 	elf_file_t ef = (elf_file_t)lf;  in link_elf_search_symbol()  local
1701 	for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {  in link_elf_search_symbol()
1704 		st_value = es->st_value + (uintptr_t) (void *) ef->address;  in link_elf_search_symbol()
1785 	elf_file_t ef = (elf_file_t)file;  in link_elf_each_function_name()  local
1790 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_each_function_name()
1794 			error = callback(ef->ddbstrtab + symp->st_name, opaque);  in link_elf_each_function_name()
1807 	elf_file_t ef = (elf_file_t)file;  in link_elf_each_function_nameval()  local
1812 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_each_function_nameval()
1830 	elf_file_t ef = (elf_file_t)lf;  in elf_get_sym()  local
1832 	if (symidx >= ef->nchains)  in elf_get_sym()
1834 	return (ef->symtab + symidx);  in elf_get_sym()
1840 	elf_file_t ef = (elf_file_t)lf;  in elf_get_symname()  local
1843 	if (symidx >= ef->nchains)  in elf_get_symname()
1845 	sym = ef->symtab + symidx;  in elf_get_symname()
1846 	return (ef->strtab + sym->st_name);  in elf_get_symname()
1859 	elf_file_t ef = (elf_file_t)lf;  in elf_lookup()  local
1865 	if (symidx >= ef->nchains) {  in elf_lookup()
1870 	sym = ef->symtab + symidx;  in elf_lookup()
1882 		*res = ((Elf_Addr)ef->address + sym->st_value);  in elf_lookup()
1893 	symbol = ef->strtab + sym->st_name;  in elf_lookup()
1924 	elf_file_t ef = (elf_file_t)lf;  in link_elf_reloc_local()  local
1927 	if ((rel = ef->rel) != NULL) {  in link_elf_reloc_local()
1928 		rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);  in link_elf_reloc_local()
1930 			elf_reloc_local(lf, (Elf_Addr)ef->address, rel,  in link_elf_reloc_local()
1937 	if ((rela = ef->rela) != NULL) {  in link_elf_reloc_local()
1939 		    ((const char *)ef->rela + ef->relasize);  in link_elf_reloc_local()
1941 			elf_reloc_local(lf, (Elf_Addr)ef->address, rela,  in link_elf_reloc_local()
1951 	elf_file_t ef = (elf_file_t)lf;  in link_elf_symtab_get()  local
1953 	*symtab = ef->ddbsymtab;  in link_elf_symtab_get()
1958 	return (ef->ddbsymcnt);  in link_elf_symtab_get()
1964 	elf_file_t ef = (elf_file_t)lf;  in link_elf_strtab_get()  local
1966 	*strtab = ef->ddbstrtab;  in link_elf_strtab_get()
1971 	return (ef->ddbstrcnt);  in link_elf_strtab_get()
1978 	elf_file_t ef = (elf_file_t)lf;  in link_elf_propagate_vnets()  local
1981 	if (ef->vnet_base != 0) {  in link_elf_propagate_vnets()
1982 		size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start;  in link_elf_propagate_vnets()
1983 		vnet_data_copy((void *)ef->vnet_base, size);  in link_elf_propagate_vnets()
1998 	elf_file_t ef;  in elf_lookup_ifunc()  local
2002 	ef = (elf_file_t)lf;  in elf_lookup_ifunc()
2003 	symp = ef->symtab + symidx;  in elf_lookup_ifunc()
2005 		val = (caddr_t)ef->address + symp->st_value;  in elf_lookup_ifunc()
2016 	elf_file_t ef;  in link_elf_ireloc()  local
2019 	ef = &eff;  in link_elf_ireloc()
2021 	bzero_early(ef, sizeof(*ef));  in link_elf_ireloc()
2023 	ef->modptr = preload_kmdp;  in link_elf_ireloc()
2024 	ef->dynamic = (Elf_Dyn *)&_DYNAMIC;  in link_elf_ireloc()
2027 	ef->address = (caddr_t) (__startkernel - KERNBASE);  in link_elf_ireloc()
2029 	ef->address = 0;  in link_elf_ireloc()
2031 	parse_dynamic(ef);  in link_elf_ireloc()
2033 	link_elf_preload_parse_symbols(ef);  in link_elf_ireloc()
2034 	relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true);  in link_elf_ireloc()
2042 	elf_file_t ef;  in link_elf_late_ireloc()  local
2046 	ef = (elf_file_t)linker_kernel_file;  in link_elf_late_ireloc()
2048 	relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true);  in link_elf_late_ireloc()