216 static int	relocate_file1(elf_file_t ef, elf_lookup_fn lookup,
395 	elf_file_t ef = (elf_file_t)lf;  in link_elf_link_common_finish()  local
407 	ef->gdb.l_addr = lf->address;  in link_elf_link_common_finish()
410 	ef->gdb.l_name = newfilename;  in link_elf_link_common_finish()
411 	ef->gdb.l_ld = ef->dynamic;  in link_elf_link_common_finish()
412 	link_elf_add_gdb(&ef->gdb);  in link_elf_link_common_finish()
454 	elf_file_t ef;  in link_elf_init()  local
468 	ef = (elf_file_t) linker_kernel_file;  in link_elf_init()
469 	ef->preloaded = 1;  in link_elf_init()
472 	ef->address = (caddr_t) (__startkernel - KERNBASE);  in link_elf_init()
474 	ef->address = 0;  in link_elf_init()
477 	ef->object = NULL;  in link_elf_init()
479 	ef->dynamic = dp;  in link_elf_init()
482 		parse_dynamic(ef);  in link_elf_init()
493 		ef->modptr = preload_kmdp;  in link_elf_init()
505 			linker_kernel_file->ctors_addr = ef->address +  in link_elf_init()
510 	(void)link_elf_preload_parse_symbols(ef);  in link_elf_init()
521 	ef->pcpu_start = DPCPU_START;  in link_elf_init()
522 	ef->pcpu_stop = DPCPU_STOP;  in link_elf_init()
523 	ef->pcpu_base = DPCPU_START;  in link_elf_init()
527 	ef->vnet_start = VNET_START;  in link_elf_init()
528 	ef->vnet_stop = VNET_STOP;  in link_elf_init()
529 	ef->vnet_base = VNET_START;  in link_elf_init()
536 link_elf_preload_parse_symbols(elf_file_t ef)  in link_elf_preload_parse_symbols()  argument
545 	if (ef->modptr == NULL)  in link_elf_preload_parse_symbols()
547 	pointer = preload_search_info(ef->modptr,  in link_elf_preload_parse_symbols()
552 	pointer = preload_search_info(ef->modptr,  in link_elf_preload_parse_symbols()
580 	ef->ddbsymtab = symtab;  in link_elf_preload_parse_symbols()
581 	ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);  in link_elf_preload_parse_symbols()
582 	ef->ddbstrtab = strtab;  in link_elf_preload_parse_symbols()
583 	ef->ddbstrcnt = strcnt;  in link_elf_preload_parse_symbols()
589 parse_dynamic(elf_file_t ef)  in parse_dynamic()  argument
594 	for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {  in parse_dynamic()
600 			    (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
601 			ef->nbuckets = hashtab[0];  in parse_dynamic()
602 			ef->nchains = hashtab[1];  in parse_dynamic()
603 			ef->buckets = hashtab + 2;  in parse_dynamic()
604 			ef->chains = ef->buckets + ef->nbuckets;  in parse_dynamic()
608 			ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
611 			ef->strsz = dp->d_un.d_val;  in parse_dynamic()
614 			ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
621 			ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
624 			ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
627 			ef->relsize = dp->d_un.d_val;  in parse_dynamic()
634 			ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
637 			ef->pltrelsize = dp->d_un.d_val;  in parse_dynamic()
640 			ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr);  in parse_dynamic()
643 			ef->relasize = dp->d_un.d_val;  in parse_dynamic()
663 		ef->pltrela = (const Elf_Rela *)ef->pltrel;  in parse_dynamic()
664 		ef->pltrel = NULL;  in parse_dynamic()
665 		ef->pltrelasize = ef->pltrelsize;  in parse_dynamic()
666 		ef->pltrelsize = 0;  in parse_dynamic()
669 	ef->ddbsymtab = ef->symtab;  in parse_dynamic()
670 	ef->ddbsymcnt = ef->nchains;  in parse_dynamic()
671 	ef->ddbstrtab = ef->strtab;  in parse_dynamic()
672 	ef->ddbstrcnt = ef->strsz;  in parse_dynamic()
674 	return elf_cpu_parse_dynamic(ef->address, ef->dynamic);  in parse_dynamic()
679 parse_dpcpu(elf_file_t ef)  in parse_dpcpu()  argument
686 	ef->pcpu_start = 0;  in parse_dpcpu()
687 	ef->pcpu_stop = 0;  in parse_dpcpu()
688 	error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start,  in parse_dpcpu()
689 	    (void ***)&ef->pcpu_stop, NULL);  in parse_dpcpu()
693 	size = (uintptr_t)ef->pcpu_stop - (uintptr_t)ef->pcpu_start;  in parse_dpcpu()
701 		    "linker script\n", ef->lf.pathname);  in parse_dpcpu()
706 	pad = *(uint32_t *)((uintptr_t)ef->pcpu_stop - sizeof(pad));  in parse_dpcpu()
710 		    ef->lf.pathname, pad, LS_PADDING);  in parse_dpcpu()
722 	ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(size);  in parse_dpcpu()
723 	if (ef->pcpu_base == 0) {  in parse_dpcpu()
726 		    __func__, size, ef->lf.pathname);  in parse_dpcpu()
729 	memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, size);  in parse_dpcpu()
730 	dpcpu_copy((void *)ef->pcpu_base, size);  in parse_dpcpu()
731 	elf_set_add(&set_pcpu_list, ef->pcpu_start, ef->pcpu_stop,  in parse_dpcpu()
732 	    ef->pcpu_base);  in parse_dpcpu()
739 parse_vnet(elf_file_t ef)  in parse_vnet()  argument
746 	ef->vnet_start = 0;  in parse_vnet()
747 	ef->vnet_stop = 0;  in parse_vnet()
748 	ef->vnet_base = 0;  in parse_vnet()
749 	error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start,  in parse_vnet()
750 	    (void ***)&ef->vnet_stop, NULL);  in parse_vnet()
754 	size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start;  in parse_vnet()
762 		    "linker script\n", ef->lf.pathname);  in parse_vnet()
767 	pad = *(uint32_t *)((uintptr_t)ef->vnet_stop - sizeof(pad));  in parse_vnet()
771 		    ef->lf.pathname, pad, LS_PADDING);  in parse_vnet()
783 	ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(size);  in parse_vnet()
784 	if (ef->vnet_base == 0) {  in parse_vnet()
787 		    __func__, size, ef->lf.pathname);  in parse_vnet()
790 	memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, size);  in parse_vnet()
791 	vnet_save_init((void *)ef->vnet_base, size);  in parse_vnet()
792 	elf_set_add(&set_vnet_list, ef->vnet_start, ef->vnet_stop,  in parse_vnet()
793 	    ef->vnet_base);  in parse_vnet()
805 preload_protect(elf_file_t ef, vm_prot_t prot)  in preload_protect()  argument
814 	hdr = (Elf_Ehdr *)ef->address;  in preload_protect()
815 	phdr = (Elf_Phdr *)(ef->address + hdr->e_phoff);  in preload_protect()
826 		error = pmap_change_prot((vm_offset_t)ef->address +  in preload_protect()
889 	elf_file_t ef;  in link_elf_link_preload()  local
913 	ef = (elf_file_t) lf;  in link_elf_link_preload()
914 	ef->preloaded = 1;  in link_elf_link_preload()
915 	ef->modptr = modptr;  in link_elf_link_preload()
916 	ef->address = *(caddr_t *)baseptr;  in link_elf_link_preload()
918 	ef->object = NULL;  in link_elf_link_preload()
920 	dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr;  in link_elf_link_preload()
921 	ef->dynamic = (Elf_Dyn *)dp;  in link_elf_link_preload()
922 	lf->address = ef->address;  in link_elf_link_preload()
930 		lf->ctors_addr = ef->address + *ctors_addrp;  in link_elf_link_preload()
938 	error = parse_dynamic(ef);  in link_elf_link_preload()
940 		error = parse_dpcpu(ef);  in link_elf_link_preload()
943 		error = parse_vnet(ef);  in link_elf_link_preload()
946 		error = preload_protect(ef, VM_PROT_ALL);  in link_elf_link_preload()
959 	elf_file_t ef;  in link_elf_link_preload_finish()  local
962 	ef = (elf_file_t) lf;  in link_elf_link_preload_finish()
963 	error = relocate_file(ef);  in link_elf_link_preload_finish()
965 		error = preload_protect(ef, VM_PROT_NONE);  in link_elf_link_preload_finish()
968 	(void)link_elf_preload_parse_symbols(ef);  in link_elf_link_preload_finish()
994 	elf_file_t ef;  in link_elf_load_file()  local
1140 	ef = (elf_file_t) lf;  in link_elf_load_file()
1142 	ef->object = vm_pager_allocate(OBJT_PHYS, NULL, mapsize, VM_PROT_ALL,  in link_elf_load_file()
1144 	if (ef->object == NULL) {  in link_elf_load_file()
1156 	error = vm_map_find(kernel_map, ef->object, 0,  in link_elf_load_file()
1160 		vm_object_deallocate(ef->object);  in link_elf_load_file()
1161 		ef->object = NULL;  in link_elf_load_file()
1167 	ef->address = mapbase;  in link_elf_load_file()
1205 	ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);  in link_elf_load_file()
1207 	lf->address = ef->address;  in link_elf_load_file()
1210 	error = parse_dynamic(ef);  in link_elf_load_file()
1213 	error = parse_dpcpu(ef);  in link_elf_load_file()
1217 	error = parse_vnet(ef);  in link_elf_load_file()
1228 	error = relocate_file(ef);  in link_elf_load_file()
1305 	ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK);  in link_elf_load_file()
1307 	ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK);  in link_elf_load_file()
1310 	    ef->symbase, symcnt, shdr[symtabindex].sh_offset,  in link_elf_load_file()
1316 	    ef->strbase, strcnt, shdr[symstrindex].sh_offset,  in link_elf_load_file()
1322 	ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);  in link_elf_load_file()
1323 	ef->ddbsymtab = (const Elf_Sym *)ef->symbase;  in link_elf_load_file()
1324 	ef->ddbstrcnt = strcnt;  in link_elf_load_file()
1325 	ef->ddbstrtab = ef->strbase;  in link_elf_load_file()
1354 	elf_file_t ef;  in elf_relocaddr()  local
1359 	ef = (elf_file_t)lf;  in elf_relocaddr()
1360 	if (x >= ef->pcpu_start && x < ef->pcpu_stop)  in elf_relocaddr()
1361 		return ((x - ef->pcpu_start) + ef->pcpu_base);  in elf_relocaddr()
1363 	if (x >= ef->vnet_start && x < ef->vnet_stop)  in elf_relocaddr()
1364 		return ((x - ef->vnet_start) + ef->vnet_base);  in elf_relocaddr()
1372 	elf_file_t ef = (elf_file_t) file;  in link_elf_unload_file()  local
1374 	if (ef->pcpu_base != 0) {  in link_elf_unload_file()
1375 		dpcpu_free((void *)ef->pcpu_base,  in link_elf_unload_file()
1376 		    ef->pcpu_stop - ef->pcpu_start);  in link_elf_unload_file()
1377 		elf_set_delete(&set_pcpu_list, ef->pcpu_start);  in link_elf_unload_file()
1380 	if (ef->vnet_base != 0) {  in link_elf_unload_file()
1381 		vnet_data_free((void *)ef->vnet_base,  in link_elf_unload_file()
1382 		    ef->vnet_stop - ef->vnet_start);  in link_elf_unload_file()
1383 		elf_set_delete(&set_vnet_list, ef->vnet_start);  in link_elf_unload_file()
1387 	if (ef->gdb.l_ld != NULL) {  in link_elf_unload_file()
1389 		free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER);  in link_elf_unload_file()
1390 		link_elf_delete_gdb(&ef->gdb);  in link_elf_unload_file()
1398 	if (ef->preloaded) {  in link_elf_unload_file()
1404 	if (ef->object != NULL) {  in link_elf_unload_file()
1405 		vm_map_remove(kernel_map, (vm_offset_t) ef->address,  in link_elf_unload_file()
1406 		    (vm_offset_t) ef->address  in link_elf_unload_file()
1407 		    + (ef->object->size << PAGE_SHIFT));  in link_elf_unload_file()
1410 	free(ef->address, M_LINKER);  in link_elf_unload_file()
1412 	free(ef->symbase, M_LINKER);  in link_elf_unload_file()
1413 	free(ef->strbase, M_LINKER);  in link_elf_unload_file()
1414 	free(ef->ctftab, M_LINKER);  in link_elf_unload_file()
1415 	free(ef->ctfoff, M_LINKER);  in link_elf_unload_file()
1416 	free(ef->typoff, M_LINKER);  in link_elf_unload_file()
1428 symbol_name(elf_file_t ef, Elf_Size r_info)  in symbol_name()  argument
1433 		ref = ef->symtab + ELF_R_SYM(r_info);  in symbol_name()
1434 		return (ef->strtab + ref->st_name);  in symbol_name()
1440 symbol_type(elf_file_t ef, Elf_Size r_info)  in symbol_type()  argument
1445 		ref = ef->symtab + ELF_R_SYM(r_info);  in symbol_type()
1452 relocate_file1(elf_file_t ef, elf_lookup_fn lookup, elf_reloc_fn reloc,  in relocate_file1()  argument
1463 		if ((symbol_type(ef, (iter)->r_info) ==			\  in relocate_file1()
1467 		if (reloc(&ef->lf, (Elf_Addr)ef->address,		\  in relocate_file1()
1469 			symname = symbol_name(ef, (iter)->r_info);	\  in relocate_file1()
1477 	APPLY_RELOCS(rel, ef->rel, ef->relsize, ELF_RELOC_REL);  in relocate_file1()
1478 	TSENTER2("ef->rela");  in relocate_file1()
1479 	APPLY_RELOCS(rela, ef->rela, ef->relasize, ELF_RELOC_RELA);  in relocate_file1()
1480 	TSEXIT2("ef->rela");  in relocate_file1()
1481 	APPLY_RELOCS(rel, ef->pltrel, ef->pltrelsize, ELF_RELOC_REL);  in relocate_file1()
1482 	APPLY_RELOCS(rela, ef->pltrela, ef->pltrelasize, ELF_RELOC_RELA);  in relocate_file1()
1491 relocate_file(elf_file_t ef)  in relocate_file()  argument
1495 	error = relocate_file1(ef, elf_lookup, elf_reloc, false);  in relocate_file()
1497 		error = relocate_file1(ef, elf_lookup, elf_reloc, true);  in relocate_file()
1522 	elf_file_t ef = (elf_file_t) lf;  in link_elf_lookup_symbol1()  local
1529 	if (ef->buckets == NULL || ef->nbuckets == 0) {  in link_elf_lookup_symbol1()
1536 	symnum = ef->buckets[hash % ef->nbuckets];  in link_elf_lookup_symbol1()
1539 		if (symnum >= ef->nchains) {  in link_elf_lookup_symbol1()
1544 		symp = ef->symtab + symnum;  in link_elf_lookup_symbol1()
1550 		strp = ef->strtab + symp->st_name;  in link_elf_lookup_symbol1()
1566 		symnum = ef->chains[symnum];  in link_elf_lookup_symbol1()
1584 	elf_file_t ef = (elf_file_t)lf;  in link_elf_lookup_debug_symbol()  local
1592 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_lookup_debug_symbol()
1593 		strp = ef->ddbstrtab + symp->st_name;  in link_elf_lookup_debug_symbol()
1613 	elf_file_t ef = (elf_file_t)lf;  in link_elf_lookup_debug_symbol_ctf()  local
1618 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_lookup_debug_symbol_ctf()
1619 		strp = ef->ddbstrtab + symp->st_name;  in link_elf_lookup_debug_symbol_ctf()
1634 	return (i < ef->ddbsymcnt ? link_elf_ctf_get_ddb(lf, lc) : ENOENT);  in link_elf_lookup_debug_symbol_ctf()
1641 	elf_file_t ef;  in link_elf_ifunc_symbol_value()  local
1647 	ef = (elf_file_t)lf;  in link_elf_ifunc_symbol_value()
1653 		*valp = (caddr_t)ef->address + es->st_value;  in link_elf_ifunc_symbol_value()
1665 	elf_file_t ef;  in link_elf_symbol_values1()  local
1670 	ef = (elf_file_t)lf;  in link_elf_symbol_values1()
1672 	if (es >= ef->symtab && es < ef->symtab + ef->nchains) {  in link_elf_symbol_values1()
1675 		symval->name = ef->strtab + es->st_name;  in link_elf_symbol_values1()
1676 		val = (caddr_t)ef->address + es->st_value;  in link_elf_symbol_values1()
1701 	elf_file_t ef = (elf_file_t)lf;  in link_elf_debug_symbol_values()  local
1708 	if (ef->symtab == ef->ddbsymtab)  in link_elf_debug_symbol_values()
1711 	if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {  in link_elf_debug_symbol_values()
1712 		symval->name = ef->ddbstrtab + es->st_name;  in link_elf_debug_symbol_values()
1713 		val = (caddr_t)ef->address + es->st_value;  in link_elf_debug_symbol_values()
1729 	elf_file_t ef = (elf_file_t)lf;  in link_elf_search_symbol()  local
1737 	for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {  in link_elf_search_symbol()
1740 		st_value = es->st_value + (uintptr_t) (void *) ef->address;  in link_elf_search_symbol()
1821 	elf_file_t ef = (elf_file_t)file;  in link_elf_each_function_name()  local
1826 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_each_function_name()
1830 			error = callback(ef->ddbstrtab + symp->st_name, opaque);  in link_elf_each_function_name()
1843 	elf_file_t ef = (elf_file_t)file;  in link_elf_each_function_nameval()  local
1848 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {  in link_elf_each_function_nameval()
1866 	elf_file_t ef = (elf_file_t)lf;  in elf_get_sym()  local
1868 	if (symidx >= ef->nchains)  in elf_get_sym()
1870 	return (ef->symtab + symidx);  in elf_get_sym()
1876 	elf_file_t ef = (elf_file_t)lf;  in elf_get_symname()  local
1879 	if (symidx >= ef->nchains)  in elf_get_symname()
1881 	sym = ef->symtab + symidx;  in elf_get_symname()
1882 	return (ef->strtab + sym->st_name);  in elf_get_symname()
1895 	elf_file_t ef = (elf_file_t)lf;  in elf_lookup()  local
1901 	if (symidx >= ef->nchains) {  in elf_lookup()
1906 	sym = ef->symtab + symidx;  in elf_lookup()
1918 		*res = ((Elf_Addr)ef->address + sym->st_value);  in elf_lookup()
1929 	symbol = ef->strtab + sym->st_name;  in elf_lookup()
1960 	elf_file_t ef = (elf_file_t)lf;  in link_elf_reloc_local()  local
1963 	if ((rel = ef->rel) != NULL) {  in link_elf_reloc_local()
1964 		rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);  in link_elf_reloc_local()
1966 			elf_reloc_local(lf, (Elf_Addr)ef->address, rel,  in link_elf_reloc_local()
1973 	if ((rela = ef->rela) != NULL) {  in link_elf_reloc_local()
1975 		    ((const char *)ef->rela + ef->relasize);  in link_elf_reloc_local()
1977 			elf_reloc_local(lf, (Elf_Addr)ef->address, rela,  in link_elf_reloc_local()
1987 	elf_file_t ef = (elf_file_t)lf;  in link_elf_symtab_get()  local
1989 	*symtab = ef->ddbsymtab;  in link_elf_symtab_get()
1994 	return (ef->ddbsymcnt);  in link_elf_symtab_get()
2000 	elf_file_t ef = (elf_file_t)lf;  in link_elf_strtab_get()  local
2002 	*strtab = ef->ddbstrtab;  in link_elf_strtab_get()
2007 	return (ef->ddbstrcnt);  in link_elf_strtab_get()
2014 	elf_file_t ef = (elf_file_t)lf;  in link_elf_propagate_vnets()  local
2017 	if (ef->vnet_base != 0) {  in link_elf_propagate_vnets()
2018 		size = (uintptr_t)ef->vnet_stop - (uintptr_t)ef->vnet_start;  in link_elf_propagate_vnets()
2019 		vnet_data_copy((void *)ef->vnet_base, size);  in link_elf_propagate_vnets()
2034 	elf_file_t ef;  in elf_lookup_ifunc()  local
2038 	ef = (elf_file_t)lf;  in elf_lookup_ifunc()
2039 	symp = ef->symtab + symidx;  in elf_lookup_ifunc()
2041 		val = (caddr_t)ef->address + symp->st_value;  in elf_lookup_ifunc()
2052 	elf_file_t ef;  in link_elf_ireloc()  local
2055 	ef = &eff;  in link_elf_ireloc()
2057 	bzero_early(ef, sizeof(*ef));  in link_elf_ireloc()
2059 	ef->modptr = preload_kmdp;  in link_elf_ireloc()
2060 	ef->dynamic = (Elf_Dyn *)&_DYNAMIC;  in link_elf_ireloc()
2063 	ef->address = (caddr_t) (__startkernel - KERNBASE);  in link_elf_ireloc()
2065 	ef->address = 0;  in link_elf_ireloc()
2067 	parse_dynamic(ef);  in link_elf_ireloc()
2069 	link_elf_preload_parse_symbols(ef);  in link_elf_ireloc()
2070 	relocate_file1(ef, elf_lookup_ifunc, elf_reloc, true);  in link_elf_ireloc()
2078 	elf_file_t ef;  in link_elf_late_ireloc()  local
2082 	ef = (elf_file_t)linker_kernel_file;  in link_elf_late_ireloc()
2084 	relocate_file1(ef, elf_lookup_ifunc, elf_reloc_late, true);  in link_elf_late_ireloc()