xref: /illumos-gate/usr/src/uts/common/exec/elf/elf.c (revision 6233c8a8e800fadd57c41fea4cec9f7db9bd5ba1)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	   All Rights Reserved	*/
/*
 * Copyright 2019, Joyent, Inc.
 * Copyright 2023 Oxide Computer Company
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/thread.h>
#include <sys/sysmacros.h>
#include <sys/signal.h>
#include <sys/cred.h>
#include <sys/user.h>
#include <sys/errno.h>
#include <sys/vnode.h>
#include <sys/mman.h>
#include <sys/kmem.h>
#include <sys/proc.h>
#include <sys/pathname.h>
#include <sys/policy.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/elf.h>
#include <sys/vmsystm.h>
#include <sys/debug.h>
#include <sys/auxv.h>
#include <sys/exec.h>
#include <sys/prsystm.h>
#include <vm/as.h>
#include <vm/rm.h>
#include <vm/seg.h>
#include <vm/seg_vn.h>
#include <sys/modctl.h>
#include <sys/systeminfo.h>
#include <sys/vmparam.h>
#include <sys/machelf.h>
#include <sys/shm_impl.h>
#include <sys/archsystm.h>
#include <sys/fasttrap.h>
#include <sys/brand.h>
#include "elf_impl.h"
#include <sys/sdt.h>
#include <sys/siginfo.h>
#include <sys/random.h>

#include <core_shstrtab.h>

#if defined(__x86)
#include <sys/comm_page_util.h>
#include <sys/fp.h>
#endif /* defined(__x86) */


extern int at_flags;
extern volatile size_t aslr_max_brk_skew;

#define	ORIGIN_STR	"ORIGIN"
#define	ORIGIN_STR_SIZE	6

static int getelfhead(vnode_t *, cred_t *, Ehdr *, uint_t *, uint_t *,
    uint_t *);
static int getelfphdr(vnode_t *, cred_t *, const Ehdr *, uint_t, caddr_t *,
    size_t *);
static int getelfshdr(vnode_t *, cred_t *, const Ehdr *, uint_t, uint_t,
    caddr_t *, size_t *, caddr_t *, size_t *);
static size_t elfsize(const Ehdr *, uint_t, const caddr_t, uintptr_t *);
static int mapelfexec(vnode_t *, Ehdr *, uint_t, caddr_t, Phdr **, Phdr **,
    Phdr **, Phdr **, Phdr *, caddr_t *, caddr_t *, intptr_t *, uintptr_t *,
    size_t, size_t *, size_t *);


#ifdef _ELF32_COMPAT
/* Link against the non-compat instances when compiling the 32-bit version. */
extern size_t elf_datasz_max;
extern size_t elf_zeropg_sz;
extern void elf_ctx_resize_scratch(elf_core_ctx_t *, size_t);
extern uint_t elf_nphdr_max;
extern uint_t elf_nshdr_max;
extern size_t elf_shstrtab_max;
#else
size_t elf_datasz_max = 1 * 1024 * 1024;
size_t elf_zeropg_sz = 4 * 1024;
uint_t elf_nphdr_max = 1000;
uint_t elf_nshdr_max = 10000;
size_t elf_shstrtab_max = 100 * 1024;
#endif

static int
dtrace_safe_phdr(Phdr *phdrp, struct uarg *args, uintptr_t base)
{
	ASSERT(phdrp->p_type == PT_SUNWDTRACE);

	/*
	 * See the comment in fasttrap.h for information on how to safely
	 * update this program header.
	 */
	if (phdrp->p_memsz < PT_SUNWDTRACE_SIZE ||
	    (phdrp->p_flags & (PF_R | PF_W | PF_X)) != (PF_R | PF_W | PF_X))
		return (-1);

	args->thrptr = phdrp->p_vaddr + base;

	return (0);
}

static int
handle_secflag_dt(proc_t *p, uint_t dt, uint_t val)
{
	uint_t flag;

	switch (dt) {
	case DT_SUNW_ASLR:
		flag = PROC_SEC_ASLR;
		break;
	default:
		return (EINVAL);
	}

	if (val == 0) {
		if (secflag_isset(p->p_secflags.psf_lower, flag))
			return (EPERM);
		if ((secpolicy_psecflags(CRED(), p, p) != 0) &&
		    secflag_isset(p->p_secflags.psf_inherit, flag))
			return (EPERM);

		secflag_clear(&p->p_secflags.psf_effective, flag);
	} else {
		if (!secflag_isset(p->p_secflags.psf_upper, flag))
			return (EPERM);

		if ((secpolicy_psecflags(CRED(), p, p) != 0) &&
		    !secflag_isset(p->p_secflags.psf_inherit, flag))
			return (EPERM);

		secflag_set(&p->p_secflags.psf_effective, flag);
	}

	return (0);
}

#ifndef _ELF32_COMPAT
void
elf_ctx_resize_scratch(elf_core_ctx_t *ctx, size_t sz)
{
	size_t target = MIN(sz, elf_datasz_max);

	if (target > ctx->ecc_bufsz) {
		if (ctx->ecc_buf != NULL) {
			kmem_free(ctx->ecc_buf, ctx->ecc_bufsz);
		}
		ctx->ecc_buf = kmem_alloc(target, KM_SLEEP);
		ctx->ecc_bufsz = target;
	}
}
#endif /* _ELF32_COMPAT */

/*
 * Map in the executable pointed to by vp. Returns 0 on success.
 */
int
mapexec_brand(vnode_t *vp, uarg_t *args, Ehdr *ehdr, Addr *uphdr_vaddr,
    intptr_t *voffset, caddr_t exec_file, int *interp, caddr_t *bssbase,
    caddr_t *brkbase, size_t *brksize, uintptr_t *lddatap)
{
	size_t		len, phdrsize;
	struct vattr	vat;
	caddr_t		phdrbase = NULL;
	uint_t		nshdrs, shstrndx, nphdrs;
	int		error = 0;
	Phdr		*uphdr = NULL;
	Phdr		*junk = NULL;
	Phdr		*dynphdr = NULL;
	Phdr		*dtrphdr = NULL;
	uintptr_t	lddata, minaddr;
	size_t		execsz;

	if (lddatap != NULL)
		*lddatap = 0;

	if (error = execpermissions(vp, &vat, args)) {
		uprintf("%s: Cannot execute %s\n", exec_file, args->pathname);
		return (error);
	}

	if ((error = getelfhead(vp, CRED(), ehdr, &nshdrs, &shstrndx,
	    &nphdrs)) != 0 ||
	    (error = getelfphdr(vp, CRED(), ehdr, nphdrs, &phdrbase,
	    &phdrsize)) != 0) {
		uprintf("%s: Cannot read %s\n", exec_file, args->pathname);
		return (error);
	}

	if ((len = elfsize(ehdr, nphdrs, phdrbase, &lddata)) == 0) {
		uprintf("%s: Nothing to load in %s", exec_file, args->pathname);
		kmem_free(phdrbase, phdrsize);
		return (ENOEXEC);
	}
	if (lddatap != NULL)
		*lddatap = lddata;

	if (error = mapelfexec(vp, ehdr, nphdrs, phdrbase, &uphdr, &dynphdr,
	    &junk, &dtrphdr, NULL, bssbase, brkbase, voffset, &minaddr,
	    len, &execsz, brksize)) {
		uprintf("%s: Cannot map %s\n", exec_file, args->pathname);
		if (uphdr != NULL && uphdr->p_flags == 0)
			kmem_free(uphdr, sizeof (Phdr));
		kmem_free(phdrbase, phdrsize);
		return (error);
	}

	/*
	 * Inform our caller if the executable needs an interpreter.
	 */
	*interp = (dynphdr == NULL) ? 0 : 1;

	/*
	 * If this is a statically linked executable, voffset should indicate
	 * the address of the executable itself (it normally holds the address
	 * of the interpreter).
	 */
	if (ehdr->e_type == ET_EXEC && *interp == 0)
		*voffset = minaddr;

	if (uphdr != NULL) {
		*uphdr_vaddr = uphdr->p_vaddr;

		if (uphdr->p_flags == 0)
			kmem_free(uphdr, sizeof (Phdr));
	} else {
		*uphdr_vaddr = (Addr)-1;
	}

	kmem_free(phdrbase, phdrsize);
	return (error);
}

int
elfexec(vnode_t *vp, execa_t *uap, uarg_t *args, intpdata_t *idatap,
    int level, size_t *execsz, int setid, caddr_t exec_file, cred_t *cred,
    int brand_action)
{
	caddr_t		phdrbase = NULL;
	caddr_t		bssbase = 0;
	caddr_t		brkbase = 0;
	size_t		brksize = 0;
	size_t		dlnsize;
	aux_entry_t	*aux;
	int		error;
	ssize_t		resid;
	int		fd = -1;
	intptr_t	voffset;
	Phdr		*intphdr = NULL;
	Phdr		*dynamicphdr = NULL;
	Phdr		*stphdr = NULL;
	Phdr		*uphdr = NULL;
	Phdr		*junk = NULL;
	size_t		len;
	size_t		postfixsize = 0;
	size_t		i;
	Phdr		*phdrp;
	Phdr		*dataphdrp = NULL;
	Phdr		*dtrphdr;
	Phdr		*capphdr = NULL;
	Cap		*cap = NULL;
	size_t		capsize;
	int		hasu = 0;
	int		hasauxv = 0;
	int		hasintp = 0;
	int		branded = 0;
	boolean_t	dynuphdr = B_FALSE;

	struct proc *p = ttoproc(curthread);
	struct user *up = PTOU(p);
	struct bigwad {
		Ehdr	ehdr;
		aux_entry_t	elfargs[__KERN_NAUXV_IMPL];
		char		dl_name[MAXPATHLEN];
		char		pathbuf[MAXPATHLEN];
		struct vattr	vattr;
		struct execenv	exenv;
	} *bigwad;	/* kmem_alloc this behemoth so we don't blow stack */
	Ehdr		*ehdrp;
	uint_t		nshdrs, shstrndx, nphdrs;
	size_t		phdrsize;
	char		*dlnp;
	char		*pathbufp;
	rlim64_t	limit;
	rlim64_t	roundlimit;

	ASSERT(p->p_model == DATAMODEL_ILP32 || p->p_model == DATAMODEL_LP64);

	bigwad = kmem_alloc(sizeof (struct bigwad), KM_SLEEP);
	ehdrp = &bigwad->ehdr;
	dlnp = bigwad->dl_name;
	pathbufp = bigwad->pathbuf;

	/*
	 * Obtain ELF and program header information.
	 */
	if ((error = getelfhead(vp, CRED(), ehdrp, &nshdrs, &shstrndx,
	    &nphdrs)) != 0 ||
	    (error = getelfphdr(vp, CRED(), ehdrp, nphdrs, &phdrbase,
	    &phdrsize)) != 0)
		goto out;

	/*
	 * Prevent executing an ELF file that has no entry point.
	 */
	if (ehdrp->e_entry == 0) {
		uprintf("%s: Bad entry point\n", exec_file);
		goto bad;
	}

	/*
	 * Put data model that we're exec-ing to into the args passed to
	 * exec_args(), so it will know what it is copying to on new stack.
	 * Now that we know whether we are exec-ing a 32-bit or 64-bit
	 * executable, we can set execsz with the appropriate NCARGS.
	 */
#ifdef	_LP64
	if (ehdrp->e_ident[EI_CLASS] == ELFCLASS32) {
		args->to_model = DATAMODEL_ILP32;
		*execsz = btopr(SINCR) + btopr(SSIZE) + btopr(NCARGS32-1);
	} else {
		args->to_model = DATAMODEL_LP64;
		args->stk_prot &= ~PROT_EXEC;
#if defined(__x86)
		args->dat_prot &= ~PROT_EXEC;
#endif
		*execsz = btopr(SINCR) + btopr(SSIZE) + btopr(NCARGS64-1);
	}
#else	/* _LP64 */
	args->to_model = DATAMODEL_ILP32;
	*execsz = btopr(SINCR) + btopr(SSIZE) + btopr(NCARGS-1);
#endif	/* _LP64 */

	/*
	 * We delay invoking the brand callback until we've figured out
	 * what kind of elf binary we're trying to run, 32-bit or 64-bit.
	 * We do this because now the brand library can just check
	 * args->to_model to see if the target is 32-bit or 64-bit without
	 * having do duplicate all the code above.
	 *
	 * The level checks associated with brand handling below are used to
	 * prevent a loop since the brand elfexec function typically comes back
	 * through this function. We must check <= here since the nested
	 * handling in the #! interpreter code will increment the level before
	 * calling gexec to run the final elfexec interpreter.
	 */
	if ((level <= INTP_MAXDEPTH) &&
	    (brand_action != EBA_NATIVE) && (PROC_IS_BRANDED(p))) {
		error = BROP(p)->b_elfexec(vp, uap, args,
		    idatap, level + 1, execsz, setid, exec_file, cred,
		    brand_action);
		goto out;
	}

	/*
	 * Determine aux size now so that stack can be built
	 * in one shot (except actual copyout of aux image),
	 * determine any non-default stack protections,
	 * and still have this code be machine independent.
	 */
	const uint_t hsize = ehdrp->e_phentsize;
	phdrp = (Phdr *)phdrbase;
	for (i = nphdrs; i > 0; i--) {
		switch (phdrp->p_type) {
		case PT_INTERP:
			hasauxv = hasintp = 1;
			break;
		case PT_PHDR:
			hasu = 1;
			break;
		case PT_SUNWSTACK:
			args->stk_prot = PROT_USER;
			if (phdrp->p_flags & PF_R)
				args->stk_prot |= PROT_READ;
			if (phdrp->p_flags & PF_W)
				args->stk_prot |= PROT_WRITE;
			if (phdrp->p_flags & PF_X)
				args->stk_prot |= PROT_EXEC;
			break;
		case PT_LOAD:
			dataphdrp = phdrp;
			break;
		case PT_SUNWCAP:
			capphdr = phdrp;
			break;
		case PT_DYNAMIC:
			dynamicphdr = phdrp;
			break;
		}
		phdrp = (Phdr *)((caddr_t)phdrp + hsize);
	}

	if (ehdrp->e_type != ET_EXEC) {
		dataphdrp = NULL;
		hasauxv = 1;
	}

	/* Copy BSS permissions to args->dat_prot */
	if (dataphdrp != NULL) {
		args->dat_prot = PROT_USER;
		if (dataphdrp->p_flags & PF_R)
			args->dat_prot |= PROT_READ;
		if (dataphdrp->p_flags & PF_W)
			args->dat_prot |= PROT_WRITE;
		if (dataphdrp->p_flags & PF_X)
			args->dat_prot |= PROT_EXEC;
	}

	/*
	 * If a auxvector will be required - reserve the space for
	 * it now.  This may be increased by exec_args if there are
	 * ISA-specific types (included in __KERN_NAUXV_IMPL).
	 */
	if (hasauxv) {
		/*
		 * If a AUX vector is being built - the base AUX
		 * entries are:
		 *
		 *	AT_BASE
		 *	AT_FLAGS
		 *	AT_PAGESZ
		 *	AT_SUN_AUXFLAGS
		 *	AT_SUN_HWCAP
		 *	AT_SUN_HWCAP2
		 *	AT_SUN_HWCAP3
		 *	AT_SUN_PLATFORM (added in stk_copyout)
		 *	AT_SUN_EXECNAME (added in stk_copyout)
		 *	AT_NULL
		 *
		 * total == 10
		 */
		if (hasintp && hasu) {
			/*
			 * Has PT_INTERP & PT_PHDR - the auxvectors that
			 * will be built are:
			 *
			 *	AT_PHDR
			 *	AT_PHENT
			 *	AT_PHNUM
			 *	AT_ENTRY
			 *	AT_LDDATA
			 *
			 * total = 5
			 */
			args->auxsize = (10 + 5) * sizeof (aux_entry_t);
		} else if (hasintp) {
			/*
			 * Has PT_INTERP but no PT_PHDR
			 *
			 *	AT_EXECFD
			 *	AT_LDDATA
			 *
			 * total = 2
			 */
			args->auxsize = (10 + 2) * sizeof (aux_entry_t);
		} else {
			args->auxsize = 10 * sizeof (aux_entry_t);
		}
	} else {
		args->auxsize = 0;
	}

	/*
	 * If this binary is using an emulator, we need to add an
	 * AT_SUN_EMULATOR aux entry.
	 */
	if (args->emulator != NULL)
		args->auxsize += sizeof (aux_entry_t);

	/*
	 * On supported kernels (x86_64) make room in the auxv for the
	 * AT_SUN_COMMPAGE entry.  This will go unpopulated on i86xpv systems
	 * which do not provide such functionality.
	 *
	 * Additionally cover the floating point information AT_SUN_FPSIZE and
	 * AT_SUN_FPTYPE.
	 */
#if defined(__amd64)
	args->auxsize += 3 * sizeof (aux_entry_t);
#endif /* defined(__amd64) */

	if ((brand_action != EBA_NATIVE) && (PROC_IS_BRANDED(p))) {
		branded = 1;
		/*
		 * We will be adding 4 entries to the aux vectors.  One for
		 * the the brandname and 3 for the brand specific aux vectors.
		 */
		args->auxsize += 4 * sizeof (aux_entry_t);
	}

	/* If the binary has an explicit ASLR flag, it must be honoured */
	if ((dynamicphdr != NULL) && (dynamicphdr->p_filesz > 0)) {
		const size_t dynfilesz = dynamicphdr->p_filesz;
		const size_t dynoffset = dynamicphdr->p_offset;
		Dyn *dyn, *dp;

		if (dynoffset > MAXOFFSET_T ||
		    dynfilesz > MAXOFFSET_T ||
		    dynoffset + dynfilesz > MAXOFFSET_T) {
			uprintf("%s: cannot read full .dynamic section\n",
			    exec_file);
			error = EINVAL;
			goto out;
		}

#define	DYN_STRIDE	100
		for (i = 0; i < dynfilesz; i += sizeof (*dyn) * DYN_STRIDE) {
			const size_t remdyns = (dynfilesz - i) / sizeof (*dyn);
			const size_t ndyns = MIN(DYN_STRIDE, remdyns);
			const size_t dynsize = ndyns * sizeof (*dyn);

			dyn = kmem_alloc(dynsize, KM_SLEEP);

			if ((error = vn_rdwr(UIO_READ, vp, (caddr_t)dyn,
			    (ssize_t)dynsize, (offset_t)(dynoffset + i),
			    UIO_SYSSPACE, 0, (rlim64_t)0,
			    CRED(), NULL)) != 0) {
				uprintf("%s: cannot read .dynamic section\n",
				    exec_file);
				goto out;
			}

			for (dp = dyn; dp < (dyn + ndyns); dp++) {
				if (dp->d_tag == DT_SUNW_ASLR) {
					if ((error = handle_secflag_dt(p,
					    DT_SUNW_ASLR,
					    dp->d_un.d_val)) != 0) {
						uprintf("%s: error setting "
						    "security-flag from "
						    "DT_SUNW_ASLR: %d\n",
						    exec_file, error);
						goto out;
					}
				}
			}

			kmem_free(dyn, dynsize);
		}
	}

	/* Hardware/Software capabilities */
	if (capphdr != NULL &&
	    (capsize = capphdr->p_filesz) > 0 &&
	    capsize <= 16 * sizeof (*cap)) {
		const uint_t ncaps = capsize / sizeof (*cap);
		Cap *cp;

		cap = kmem_alloc(capsize, KM_SLEEP);
		if ((error = vn_rdwr(UIO_READ, vp, (caddr_t)cap,
		    (ssize_t)capsize, (offset_t)capphdr->p_offset,
		    UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), NULL)) != 0) {
			uprintf("%s: Cannot read capabilities section\n",
			    exec_file);
			goto out;
		}
		for (cp = cap; cp < cap + ncaps; cp++) {
			if (cp->c_tag == CA_SUNW_SF_1 &&
			    (cp->c_un.c_val & SF1_SUNW_ADDR32)) {
				if (args->to_model == DATAMODEL_LP64)
					args->addr32 = 1;
				break;
			}
		}
	}

	aux = bigwad->elfargs;
	/*
	 * Move args to the user's stack.
	 * This can fill in the AT_SUN_PLATFORM and AT_SUN_EXECNAME aux entries.
	 */
	if ((error = exec_args(uap, args, idatap, (void **)&aux)) != 0) {
		if (error == -1) {
			error = ENOEXEC;
			goto bad;
		}
		goto out;
	}
	/* we're single threaded after this point */

	/*
	 * If this is an ET_DYN executable (shared object),
	 * determine its memory size so that mapelfexec() can load it.
	 */
	if (ehdrp->e_type == ET_DYN)
		len = elfsize(ehdrp, nphdrs, phdrbase, NULL);
	else
		len = 0;

	dtrphdr = NULL;

	error = mapelfexec(vp, ehdrp, nphdrs, phdrbase, &uphdr, &intphdr,
	    &stphdr, &dtrphdr, dataphdrp, &bssbase, &brkbase, &voffset, NULL,
	    len, execsz, &brksize);

	/*
	 * Our uphdr has been dynamically allocated if (and only if) its
	 * program header flags are clear.  To avoid leaks, this must be
	 * checked regardless of whether mapelfexec() emitted an error.
	 */
	dynuphdr = (uphdr != NULL && uphdr->p_flags == 0);

	if (error != 0)
		goto bad;

	if (uphdr != NULL && intphdr == NULL)
		goto bad;

	if (dtrphdr != NULL && dtrace_safe_phdr(dtrphdr, args, voffset) != 0) {
		uprintf("%s: Bad DTrace phdr in %s\n", exec_file, exec_file);
		goto bad;
	}

	if (intphdr != NULL) {
		size_t		len;
		uintptr_t	lddata;
		char		*p;
		struct vnode	*nvp;

		dlnsize = intphdr->p_filesz;

		/*
		 * Make sure none of the component pieces of dlnsize result in
		 * an oversized or zeroed result.
		 */
		if (intphdr->p_filesz > MAXPATHLEN || dlnsize > MAXPATHLEN ||
		    dlnsize == 0 || dlnsize < intphdr->p_filesz) {
			goto bad;
		}

		/*
		 * Read in "interpreter" pathname.
		 */
		if ((error = vn_rdwr(UIO_READ, vp, dlnp,
		    (ssize_t)intphdr->p_filesz, (offset_t)intphdr->p_offset,
		    UIO_SYSSPACE, 0, (rlim64_t)0, CRED(), &resid)) != 0) {
			uprintf("%s: Cannot obtain interpreter pathname\n",
			    exec_file);
			goto bad;
		}

		if (resid != 0 || dlnp[dlnsize - 1] != '\0')
			goto bad;

		/*
		 * Search for '$ORIGIN' token in interpreter path.
		 * If found, expand it.
		 */
		for (p = dlnp; p = strchr(p, '$'); ) {
			uint_t	len, curlen;
			char	*_ptr;

			if (strncmp(++p, ORIGIN_STR, ORIGIN_STR_SIZE))
				continue;

			/*
			 * We don't support $ORIGIN on setid programs to close
			 * a potential attack vector.
			 */
			if ((setid & EXECSETID_SETID) != 0) {
				error = ENOEXEC;
				goto bad;
			}

			curlen = 0;
			len = p - dlnp - 1;
			if (len) {
				bcopy(dlnp, pathbufp, len);
				curlen += len;
			}
			if (_ptr = strrchr(args->pathname, '/')) {
				len = _ptr - args->pathname;
				if ((curlen + len) > MAXPATHLEN)
					break;

				bcopy(args->pathname, &pathbufp[curlen], len);
				curlen += len;
			} else {
				/*
				 * executable is a basename found in the
				 * current directory.  So - just substitue
				 * '.' for ORIGIN.
				 */
				pathbufp[curlen] = '.';
				curlen++;
			}
			p += ORIGIN_STR_SIZE;
			len = strlen(p);

			if ((curlen + len) > MAXPATHLEN)
				break;
			bcopy(p, &pathbufp[curlen], len);
			curlen += len;
			pathbufp[curlen++] = '\0';
			bcopy(pathbufp, dlnp, curlen);
		}

		/*
		 * /usr/lib/ld.so.1 is known to be a symlink to /lib/ld.so.1
		 * (and /usr/lib/64/ld.so.1 is a symlink to /lib/64/ld.so.1).
		 * Just in case /usr is not mounted, change it now.
		 */
		if (strcmp(dlnp, USR_LIB_RTLD) == 0)
			dlnp += 4;
		error = lookupname(dlnp, UIO_SYSSPACE, FOLLOW, NULLVPP, &nvp);
		if (error && dlnp != bigwad->dl_name) {
			/* new kernel, old user-level */
			error = lookupname(dlnp -= 4, UIO_SYSSPACE, FOLLOW,
			    NULLVPP, &nvp);
		}
		if (error) {
			uprintf("%s: Cannot find %s\n", exec_file, dlnp);
			goto bad;
		}

		/*
		 * Setup the "aux" vector.
		 */
		if (uphdr) {
			if (ehdrp->e_type == ET_DYN) {
				/* don't use the first page */
				bigwad->exenv.ex_brkbase = (caddr_t)PAGESIZE;
				bigwad->exenv.ex_bssbase = (caddr_t)PAGESIZE;
			} else {
				bigwad->exenv.ex_bssbase = bssbase;
				bigwad->exenv.ex_brkbase = brkbase;
			}
			bigwad->exenv.ex_brksize = brksize;
			bigwad->exenv.ex_magic = elfmagic;
			bigwad->exenv.ex_vp = vp;
			setexecenv(&bigwad->exenv);

			ADDAUX(aux, AT_PHDR, uphdr->p_vaddr + voffset)
			ADDAUX(aux, AT_PHENT, ehdrp->e_phentsize)
			ADDAUX(aux, AT_PHNUM, nphdrs)
			ADDAUX(aux, AT_ENTRY, ehdrp->e_entry + voffset)
		} else {
			if ((error = execopen(&vp, &fd)) != 0) {
				VN_RELE(nvp);
				goto bad;
			}

			ADDAUX(aux, AT_EXECFD, fd)
		}

		if ((error = execpermissions(nvp, &bigwad->vattr, args)) != 0) {
			VN_RELE(nvp);
			uprintf("%s: Cannot execute %s\n", exec_file, dlnp);
			goto bad;
		}

		/*
		 * Now obtain the ELF header along with the entire program
		 * header contained in "nvp".
		 */
		kmem_free(phdrbase, phdrsize);
		phdrbase = NULL;
		if ((error = getelfhead(nvp, CRED(), ehdrp, &nshdrs,
		    &shstrndx, &nphdrs)) != 0 ||
		    (error = getelfphdr(nvp, CRED(), ehdrp, nphdrs, &phdrbase,
		    &phdrsize)) != 0) {
			VN_RELE(nvp);
			uprintf("%s: Cannot read %s\n", exec_file, dlnp);
			goto bad;
		}

		/*
		 * Determine memory size of the "interpreter's" loadable
		 * sections.  This size is then used to obtain the virtual
		 * address of a hole, in the user's address space, large
		 * enough to map the "interpreter".
		 */
		if ((len = elfsize(ehdrp, nphdrs, phdrbase, &lddata)) == 0) {
			VN_RELE(nvp);
			uprintf("%s: Nothing to load in %s\n", exec_file, dlnp);
			goto bad;
		}

		dtrphdr = NULL;

		error = mapelfexec(nvp, ehdrp, nphdrs, phdrbase, NULL, &junk,
		    &junk, &dtrphdr, NULL, NULL, NULL, &voffset, NULL, len,
		    execsz, NULL);

		if (error || junk != NULL) {
			VN_RELE(nvp);
			uprintf("%s: Cannot map %s\n", exec_file, dlnp);
			goto bad;
		}

		/*
		 * We use the DTrace program header to initialize the
		 * architecture-specific user per-LWP location. The dtrace
		 * fasttrap provider requires ready access to per-LWP scratch
		 * space. We assume that there is only one such program header
		 * in the interpreter.
		 */
		if (dtrphdr != NULL &&
		    dtrace_safe_phdr(dtrphdr, args, voffset) != 0) {
			VN_RELE(nvp);
			uprintf("%s: Bad DTrace phdr in %s\n", exec_file, dlnp);
			goto bad;
		}

		VN_RELE(nvp);
		ADDAUX(aux, AT_SUN_LDDATA, voffset + lddata)
	}

	if (hasauxv) {
		int auxf = AF_SUN_HWCAPVERIFY;
#if defined(__amd64)
		size_t fpsize;
		int fptype;
#endif /* defined(__amd64) */

		/*
		 * Note: AT_SUN_PLATFORM and AT_SUN_EXECNAME were filled in via
		 * exec_args()
		 */
		ADDAUX(aux, AT_BASE, voffset)
		ADDAUX(aux, AT_FLAGS, at_flags)
		ADDAUX(aux, AT_PAGESZ, PAGESIZE)
		/*
		 * Linker flags. (security)
		 * p_flag not yet set at this time.
		 * We rely on gexec() to provide us with the information.
		 * If the application is set-uid but this is not reflected
		 * in a mismatch between real/effective uids/gids, then
		 * don't treat this as a set-uid exec.  So we care about
		 * the EXECSETID_UGIDS flag but not the ...SETID flag.
		 */
		if ((setid &= ~EXECSETID_SETID) != 0)
			auxf |= AF_SUN_SETUGID;

		/*
		 * If we're running a native process from within a branded
		 * zone under pfexec then we clear the AF_SUN_SETUGID flag so
		 * that the native ld.so.1 is able to link with the native
		 * libraries instead of using the brand libraries that are
		 * installed in the zone.  We only do this for processes
		 * which we trust because we see they are already running
		 * under pfexec (where uid != euid).  This prevents a
		 * malicious user within the zone from crafting a wrapper to
		 * run native suid commands with unsecure libraries interposed.
		 */
		if ((brand_action == EBA_NATIVE) && (PROC_IS_BRANDED(p) &&
		    (setid &= ~EXECSETID_SETID) != 0))
			auxf &= ~AF_SUN_SETUGID;

		/*
		 * Record the user addr of the auxflags aux vector entry
		 * since brands may optionally want to manipulate this field.
		 */
		args->auxp_auxflags =
		    (char *)((char *)args->stackend +
		    ((char *)&aux->a_type -
		    (char *)bigwad->elfargs));
		ADDAUX(aux, AT_SUN_AUXFLAGS, auxf);

		/*
		 * Hardware capability flag word (performance hints)
		 * Used for choosing faster library routines.
		 * (Potentially different between 32-bit and 64-bit ABIs)
		 */
		if (args->to_model == DATAMODEL_NATIVE) {
			ADDAUX(aux, AT_SUN_HWCAP, auxv_hwcap)
			ADDAUX(aux, AT_SUN_HWCAP2, auxv_hwcap_2)
			ADDAUX(aux, AT_SUN_HWCAP3, auxv_hwcap_3)
		} else {
			ADDAUX(aux, AT_SUN_HWCAP, auxv_hwcap32)
			ADDAUX(aux, AT_SUN_HWCAP2, auxv_hwcap32_2)
			ADDAUX(aux, AT_SUN_HWCAP3, auxv_hwcap32_3)
		}

		if (branded) {
			/*
			 * Reserve space for the brand-private aux vectors,
			 * and record the user addr of that space.
			 */
			args->auxp_brand =
			    (char *)((char *)args->stackend +
			    ((char *)&aux->a_type -
			    (char *)bigwad->elfargs));
			ADDAUX(aux, AT_SUN_BRAND_AUX1, 0)
			ADDAUX(aux, AT_SUN_BRAND_AUX2, 0)
			ADDAUX(aux, AT_SUN_BRAND_AUX3, 0)
		}

		/*
		 * Add the comm page auxv entry, mapping it in if needed. Also
		 * take care of the FPU entries.
		 */
#if defined(__amd64)
		if (args->commpage != (uintptr_t)NULL ||
		    (args->commpage = (uintptr_t)comm_page_mapin()) !=
		    (uintptr_t)NULL) {
			ADDAUX(aux, AT_SUN_COMMPAGE, args->commpage)
		} else {
			/*
			 * If the comm page cannot be mapped, pad out the auxv
			 * to satisfy later size checks.
			 */
			ADDAUX(aux, AT_NULL, 0)
		}

		fptype = AT_386_FPINFO_NONE;
		fpu_auxv_info(&fptype, &fpsize);
		if (fptype != AT_386_FPINFO_NONE) {
			ADDAUX(aux, AT_SUN_FPTYPE, fptype)
			ADDAUX(aux, AT_SUN_FPSIZE, fpsize)
		} else {
			ADDAUX(aux, AT_NULL, 0)
			ADDAUX(aux, AT_NULL, 0)
		}
#endif /* defined(__amd64) */

		ADDAUX(aux, AT_NULL, 0)
		postfixsize = (uintptr_t)aux - (uintptr_t)bigwad->elfargs;

		/*
		 * We make assumptions above when we determine how many aux
		 * vector entries we will be adding. However, if we have an
		 * invalid elf file, it is possible that mapelfexec might
		 * behave differently (but not return an error), in which case
		 * the number of aux entries we actually add will be different.
		 * We detect that now and error out.
		 */
		if (postfixsize != args->auxsize) {
			DTRACE_PROBE2(elfexec_badaux, size_t, postfixsize,
			    size_t, args->auxsize);
			goto bad;
		}
		ASSERT(postfixsize <= __KERN_NAUXV_IMPL * sizeof (aux_entry_t));
	}

	/*
	 * For the 64-bit kernel, the limit is big enough that rounding it up
	 * to a page can overflow the 64-bit limit, so we check for btopr()
	 * overflowing here by comparing it with the unrounded limit in pages.
	 * If it hasn't overflowed, compare the exec size with the rounded up
	 * limit in pages.  Otherwise, just compare with the unrounded limit.
	 */
	limit = btop(p->p_vmem_ctl);
	roundlimit = btopr(p->p_vmem_ctl);
	if ((roundlimit > limit && *execsz > roundlimit) ||
	    (roundlimit < limit && *execsz > limit)) {
		mutex_enter(&p->p_lock);
		(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p,
		    RCA_SAFE);
		mutex_exit(&p->p_lock);
		error = ENOMEM;
		goto bad;
	}

	bzero(up->u_auxv, sizeof (up->u_auxv));
	up->u_commpagep = args->commpage;
	if (postfixsize) {
		size_t num_auxv;

		/*
		 * Copy the aux vector to the user stack.
		 */
		error = execpoststack(args, bigwad->elfargs, postfixsize);
		if (error)
			goto bad;

		/*
		 * Copy auxv to the process's user structure for use by /proc.
		 * If this is a branded process, the brand's exec routine will
		 * copy it's private entries to the user structure later. It
		 * relies on the fact that the blank entries are at the end.
		 */
		num_auxv = postfixsize / sizeof (aux_entry_t);
		ASSERT(num_auxv <= sizeof (up->u_auxv) / sizeof (auxv_t));
		aux = bigwad->elfargs;
		for (i = 0; i < num_auxv; i++) {
			up->u_auxv[i].a_type = aux[i].a_type;
			up->u_auxv[i].a_un.a_val = (aux_val_t)aux[i].a_un.a_val;
		}
	}

	/*
	 * Pass back the starting address so we can set the program counter.
	 */
	args->entry = (uintptr_t)(ehdrp->e_entry + voffset);

	if (!uphdr) {
		if (ehdrp->e_type == ET_DYN) {
			/*
			 * If we are executing a shared library which doesn't
			 * have a interpreter (probably ld.so.1) then
			 * we don't set the brkbase now.  Instead we
			 * delay it's setting until the first call
			 * via grow.c::brk().  This permits ld.so.1 to
			 * initialize brkbase to the tail of the executable it
			 * loads (which is where it needs to be).
			 */
			bigwad->exenv.ex_brkbase = (caddr_t)0;
			bigwad->exenv.ex_bssbase = (caddr_t)0;
			bigwad->exenv.ex_brksize = 0;
		} else {
			bigwad->exenv.ex_brkbase = brkbase;
			bigwad->exenv.ex_bssbase = bssbase;
			bigwad->exenv.ex_brksize = brksize;
		}
		bigwad->exenv.ex_magic = elfmagic;
		bigwad->exenv.ex_vp = vp;
		setexecenv(&bigwad->exenv);
	}

	ASSERT(error == 0);
	goto out;

bad:
	if (fd != -1)		/* did we open the a.out yet */
		(void) execclose(fd);

	psignal(p, SIGKILL);

	if (error == 0)
		error = ENOEXEC;
out:
	if (dynuphdr)
		kmem_free(uphdr, sizeof (Phdr));
	if (phdrbase != NULL)
		kmem_free(phdrbase, phdrsize);
	if (cap != NULL)
		kmem_free(cap, capsize);
	kmem_free(bigwad, sizeof (struct bigwad));
	return (error);
}

/*
 * Compute the memory size requirement for the ELF file.
 */
static size_t
elfsize(const Ehdr *ehdrp, uint_t nphdrs, const caddr_t phdrbase,
    uintptr_t *lddata)
{
	const Phdr *phdrp = (Phdr *)phdrbase;
	const uint_t hsize = ehdrp->e_phentsize;
	boolean_t dfirst = B_TRUE;
	uintptr_t loaddr = UINTPTR_MAX;
	uintptr_t hiaddr = 0;
	uint_t i;

	for (i = nphdrs; i > 0; i--) {
		if (phdrp->p_type == PT_LOAD) {
			const uintptr_t lo = phdrp->p_vaddr;
			const uintptr_t hi = lo + phdrp->p_memsz;

			loaddr = MIN(lo, loaddr);
			hiaddr = MAX(hi, hiaddr);

			/*
			 * save the address of the first data segment
			 * of a object - used for the AT_SUNW_LDDATA
			 * aux entry.
			 */
			if ((lddata != NULL) && dfirst &&
			    (phdrp->p_flags & PF_W)) {
				*lddata = lo;
				dfirst = B_FALSE;
			}
		}
		phdrp = (Phdr *)((caddr_t)phdrp + hsize);
	}

	if (hiaddr <= loaddr) {
		/* No non-zero PT_LOAD segment found */
		return (0);
	}

	return (roundup(hiaddr - (loaddr & PAGEMASK), PAGESIZE));
}

/*
 * Read in the ELF header and program header table.
 * SUSV3 requires:
 *	ENOEXEC	File format is not recognized
 *	EINVAL	Format recognized but execution not supported
 */
static int
getelfhead(vnode_t *vp, cred_t *credp, Ehdr *ehdr, uint_t *nshdrs,
    uint_t *shstrndx, uint_t *nphdrs)
{
	int error;
	ssize_t resid;

	/*
	 * We got here by the first two bytes in ident,
	 * now read the entire ELF header.
	 */
	if ((error = vn_rdwr(UIO_READ, vp, (caddr_t)ehdr,
	    sizeof (Ehdr), (offset_t)0, UIO_SYSSPACE, 0,
	    (rlim64_t)0, credp, &resid)) != 0)
		return (error);

	/*
	 * Since a separate version is compiled for handling 32-bit and
	 * 64-bit ELF executables on a 64-bit kernel, the 64-bit version
	 * doesn't need to be able to deal with 32-bit ELF files.
	 */
	if (resid != 0 ||
	    ehdr->e_ident[EI_MAG2] != ELFMAG2 ||
	    ehdr->e_ident[EI_MAG3] != ELFMAG3)
		return (ENOEXEC);

	if ((ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) ||
#if defined(_ILP32) || defined(_ELF32_COMPAT)
	    ehdr->e_ident[EI_CLASS] != ELFCLASS32 ||
#else
	    ehdr->e_ident[EI_CLASS] != ELFCLASS64 ||
#endif
	    !elfheadcheck(ehdr->e_ident[EI_DATA], ehdr->e_machine,
	    ehdr->e_flags))
		return (EINVAL);

	*nshdrs = ehdr->e_shnum;
	*shstrndx = ehdr->e_shstrndx;
	*nphdrs = ehdr->e_phnum;

	/*
	 * If e_shnum, e_shstrndx, or e_phnum is its sentinel value, we need
	 * to read in the section header at index zero to access the true
	 * values for those fields.
	 */
	if ((*nshdrs == 0 && ehdr->e_shoff != 0) ||
	    *shstrndx == SHN_XINDEX || *nphdrs == PN_XNUM) {
		Shdr shdr;

		if (ehdr->e_shoff == 0)
			return (EINVAL);

		if ((error = vn_rdwr(UIO_READ, vp, (caddr_t)&shdr,
		    sizeof (shdr), (offset_t)ehdr->e_shoff, UIO_SYSSPACE, 0,
		    (rlim64_t)0, credp, NULL)) != 0) {
			return (error);
		}

		if (*nshdrs == 0)
			*nshdrs = shdr.sh_size;
		if (*shstrndx == SHN_XINDEX)
			*shstrndx = shdr.sh_link;
		if (*nphdrs == PN_XNUM && shdr.sh_info != 0)
			*nphdrs = shdr.sh_info;
	}

	return (0);
}

/*
 * We use members through p_flags on 32-bit files and p_memsz on 64-bit files,
 * so e_phentsize must be at least large enough to include those members.
 */
#if !defined(_LP64) || defined(_ELF32_COMPAT)
#define	MINPHENTSZ	(offsetof(Phdr, p_flags) + \
			sizeof (((Phdr *)NULL)->p_flags))
#else
#define	MINPHENTSZ	(offsetof(Phdr, p_memsz) + \
			sizeof (((Phdr *)NULL)->p_memsz))
#endif

static int
getelfphdr(vnode_t *vp, cred_t *credp, const Ehdr *ehdr, uint_t nphdrs,
    caddr_t *phbasep, size_t *phsizep)
{
	int err;

	/*
	 * Ensure that e_phentsize is large enough for required fields to be
	 * accessible and will maintain 8-byte alignment.
	 */
	if (ehdr->e_phentsize < MINPHENTSZ || (ehdr->e_phentsize & 3))
		return (EINVAL);

	*phsizep = nphdrs * ehdr->e_phentsize;

	if (*phsizep > sizeof (Phdr) * elf_nphdr_max) {
		if ((*phbasep = kmem_alloc(*phsizep, KM_NOSLEEP)) == NULL)
			return (ENOMEM);
	} else {
		*phbasep = kmem_alloc(*phsizep, KM_SLEEP);
	}

	if ((err = vn_rdwr(UIO_READ, vp, *phbasep, (ssize_t)*phsizep,
	    (offset_t)ehdr->e_phoff, UIO_SYSSPACE, 0, (rlim64_t)0,
	    credp, NULL)) != 0) {
		kmem_free(*phbasep, *phsizep);
		*phbasep = NULL;
		return (err);
	}

	return (0);
}

#define	MINSHDRSZ	(offsetof(Shdr, sh_entsize) + \
			sizeof (((Shdr *)NULL)->sh_entsize))

static int
getelfshdr(vnode_t *vp, cred_t *credp, const Ehdr *ehdr, uint_t nshdrs,
    uint_t shstrndx, caddr_t *shbasep, size_t *shsizep, char **shstrbasep,
    size_t *shstrsizep)
{
	int err;
	Shdr *shdr;

	/*
	 * Since we're going to be using e_shentsize to iterate down the
	 * array of section headers, it must be 8-byte aligned or else
	 * a we might cause a misaligned access. We use all members through
	 * sh_entsize (on both 32- and 64-bit ELF files) so e_shentsize
	 * must be at least large enough to include that member. The index
	 * of the string table section must also be valid.
	 */
	if (ehdr->e_shentsize < MINSHDRSZ || (ehdr->e_shentsize & 3) ||
	    nshdrs == 0 || shstrndx >= nshdrs) {
		return (EINVAL);
	}

	*shsizep = nshdrs * ehdr->e_shentsize;

	if (*shsizep > sizeof (Shdr) * elf_nshdr_max) {
		if ((*shbasep = kmem_alloc(*shsizep, KM_NOSLEEP)) == NULL)
			return (ENOMEM);
	} else {
		*shbasep = kmem_alloc(*shsizep, KM_SLEEP);
	}

	if ((err = vn_rdwr(UIO_READ, vp, *shbasep, (ssize_t)*shsizep,
	    (offset_t)ehdr->e_shoff, UIO_SYSSPACE, 0, (rlim64_t)0,
	    credp, NULL)) != 0) {
		kmem_free(*shbasep, *shsizep);
		return (err);
	}

	/*
	 * Grab the section string table.  Walking through the shdrs is
	 * pointless if their names cannot be interrogated.
	 */
	shdr = (Shdr *)(*shbasep + shstrndx * ehdr->e_shentsize);
	if ((*shstrsizep = shdr->sh_size) == 0) {
		kmem_free(*shbasep, *shsizep);
		return (EINVAL);
	}

	if (*shstrsizep > elf_shstrtab_max) {
		if ((*shstrbasep = kmem_alloc(*shstrsizep,
		    KM_NOSLEEP)) == NULL) {
			kmem_free(*shbasep, *shsizep);
			return (ENOMEM);
		}
	} else {
		*shstrbasep = kmem_alloc(*shstrsizep, KM_SLEEP);
	}

	if ((err = vn_rdwr(UIO_READ, vp, *shstrbasep, (ssize_t)*shstrsizep,
	    (offset_t)shdr->sh_offset, UIO_SYSSPACE, 0, (rlim64_t)0,
	    credp, NULL)) != 0) {
		kmem_free(*shbasep, *shsizep);
		kmem_free(*shstrbasep, *shstrsizep);
		return (err);
	}

	/*
	 * Make sure the strtab is null-terminated to make sure we
	 * don't run off the end of the table.
	 */
	(*shstrbasep)[*shstrsizep - 1] = '\0';

	return (0);
}

int
elfreadhdr(vnode_t *vp, cred_t *credp, Ehdr *ehdrp, uint_t *nphdrs,
    caddr_t *phbasep, size_t *phsizep)
{
	int error;
	uint_t nshdrs, shstrndx;

	if ((error = getelfhead(vp, credp, ehdrp, &nshdrs, &shstrndx,
	    nphdrs)) != 0 ||
	    (error = getelfphdr(vp, credp, ehdrp, *nphdrs, phbasep,
	    phsizep)) != 0) {
		return (error);
	}
	return (0);
}

static int
mapelfexec(
	vnode_t *vp,
	Ehdr *ehdr,
	uint_t nphdrs,
	caddr_t phdrbase,
	Phdr **uphdr,
	Phdr **intphdr,
	Phdr **stphdr,
	Phdr **dtphdr,
	Phdr *dataphdrp,
	caddr_t *bssbase,
	caddr_t *brkbase,
	intptr_t *voffset,
	uintptr_t *minaddrp,
	size_t len,
	size_t *execsz,
	size_t *brksize)
{
	Phdr *phdr;
	int error, page, prot;
	caddr_t addr = NULL;
	caddr_t minaddr = (caddr_t)UINTPTR_MAX;
	uint_t i;
	size_t zfodsz, memsz;
	boolean_t ptload = B_FALSE;
	off_t offset;
	const uint_t hsize = ehdr->e_phentsize;
	extern int use_brk_lpg;

	if (ehdr->e_type == ET_DYN) {
		secflagset_t flags = 0;
		/*
		 * Obtain the virtual address of a hole in the
		 * address space to map the "interpreter".
		 */
		if (secflag_enabled(curproc, PROC_SEC_ASLR))
			flags |= _MAP_RANDOMIZE;

		map_addr(&addr, len, (offset_t)0, 1, flags);
		if (addr == NULL)
			return (ENOMEM);
		*voffset = (intptr_t)addr;

		/*
		 * Calculate the minimum vaddr so it can be subtracted out.
		 * According to the ELF specification, since PT_LOAD sections
		 * must be sorted by increasing p_vaddr values, this is
		 * guaranteed to be the first PT_LOAD section.
		 */
		phdr = (Phdr *)phdrbase;
		for (i = nphdrs; i > 0; i--) {
			if (phdr->p_type == PT_LOAD) {
				*voffset -= (uintptr_t)phdr->p_vaddr;
				break;
			}
			phdr = (Phdr *)((caddr_t)phdr + hsize);
		}

	} else {
		*voffset = 0;
	}

	phdr = (Phdr *)phdrbase;
	for (i = nphdrs; i > 0; i--) {
		switch (phdr->p_type) {
		case PT_LOAD:
			ptload = B_TRUE;
			prot = PROT_USER;
			if (phdr->p_flags & PF_R)
				prot |= PROT_READ;
			if (phdr->p_flags & PF_W)
				prot |= PROT_WRITE;
			if (phdr->p_flags & PF_X)
				prot |= PROT_EXEC;

			addr = (caddr_t)((uintptr_t)phdr->p_vaddr + *voffset);

			if (*intphdr != NULL && uphdr != NULL &&
			    *uphdr == NULL) {
				/*
				 * The PT_PHDR program header is, strictly
				 * speaking, optional.  If we find that this
				 * is missing, we will determine the location
				 * of the program headers based on the address
				 * of the lowest PT_LOAD segment (namely, this
				 * one):  we subtract the p_offset to get to
				 * the ELF header and then add back the program
				 * header offset to get to the program headers.
				 * We then cons up a Phdr that corresponds to
				 * the (missing) PT_PHDR, setting the flags
				 * to 0 to denote that this is artificial and
				 * should (must) be freed by the caller.
				 */
				Phdr *cons;

				cons = kmem_zalloc(sizeof (Phdr), KM_SLEEP);

				cons->p_flags = 0;
				cons->p_type = PT_PHDR;
				cons->p_vaddr = ((uintptr_t)addr -
				    phdr->p_offset) + ehdr->e_phoff;

				*uphdr = cons;
			}

			/*
			 * The ELF spec dictates that p_filesz may not be
			 * larger than p_memsz in PT_LOAD segments.
			 */
			if (phdr->p_filesz > phdr->p_memsz) {
				error = EINVAL;
				goto bad;
			}

			/*
			 * Keep track of the segment with the lowest starting
			 * address.
			 */
			if (addr < minaddr)
				minaddr = addr;

			zfodsz = (size_t)phdr->p_memsz - phdr->p_filesz;

			offset = phdr->p_offset;
			if (((uintptr_t)offset & PAGEOFFSET) ==
			    ((uintptr_t)addr & PAGEOFFSET) &&
			    (!(vp->v_flag & VNOMAP))) {
				page = 1;
			} else {
				page = 0;
			}

			/*
			 * Set the heap pagesize for OOB when the bss size
			 * is known and use_brk_lpg is not 0.
			 */
			if (brksize != NULL && use_brk_lpg &&
			    zfodsz != 0 && phdr == dataphdrp &&
			    (prot & PROT_WRITE)) {
				const size_t tlen = P2NPHASE((uintptr_t)addr +
				    phdr->p_filesz, PAGESIZE);

				if (zfodsz > tlen) {
					const caddr_t taddr = addr +
					    phdr->p_filesz + tlen;

					/*
					 * Since a hole in the AS large enough
					 * for this object as calculated by
					 * elfsize() is available, we do not
					 * need to fear overflow for 'taddr'.
					 */
					curproc->p_brkpageszc =
					    page_szc(map_pgsz(MAPPGSZ_HEAP,
					    curproc, taddr, zfodsz - tlen, 0));
				}
			}

			if (curproc->p_brkpageszc != 0 && phdr == dataphdrp &&
			    (prot & PROT_WRITE)) {
				uint_t	szc = curproc->p_brkpageszc;
				size_t pgsz = page_get_pagesize(szc);
				caddr_t ebss = addr + phdr->p_memsz;
				/*
				 * If we need extra space to keep the BSS an
				 * integral number of pages in size, some of
				 * that space may fall beyond p_brkbase, so we
				 * need to set p_brksize to account for it
				 * being (logically) part of the brk.
				 */
				size_t extra_zfodsz;

				ASSERT(pgsz > PAGESIZE);

				extra_zfodsz = P2NPHASE((uintptr_t)ebss, pgsz);

				if (error = execmap(vp, addr, phdr->p_filesz,
				    zfodsz + extra_zfodsz, phdr->p_offset,
				    prot, page, szc))
					goto bad;
				if (brksize != NULL)
					*brksize = extra_zfodsz;
			} else {
				if (error = execmap(vp, addr, phdr->p_filesz,
				    zfodsz, phdr->p_offset, prot, page, 0))
					goto bad;
			}

			if (bssbase != NULL && addr >= *bssbase &&
			    phdr == dataphdrp) {
				*bssbase = addr + phdr->p_filesz;
			}
			if (brkbase != NULL && addr >= *brkbase) {
				*brkbase = addr + phdr->p_memsz;
			}

			memsz = btopr(phdr->p_memsz);
			if ((*execsz + memsz) < *execsz) {
				error = ENOMEM;
				goto bad;
			}
			*execsz += memsz;
			break;

		case PT_INTERP:
			if (ptload)
				goto bad;
			*intphdr = phdr;
			break;

		case PT_SHLIB:
			*stphdr = phdr;
			break;

		case PT_PHDR:
			if (ptload || phdr->p_flags == 0)
				goto bad;

			if (uphdr != NULL)
				*uphdr = phdr;

			break;

		case PT_NULL:
		case PT_DYNAMIC:
		case PT_NOTE:
			break;

		case PT_SUNWDTRACE:
			if (dtphdr != NULL)
				*dtphdr = phdr;
			break;

		default:
			break;
		}
		phdr = (Phdr *)((caddr_t)phdr + hsize);
	}

	if (minaddrp != NULL) {
		ASSERT(minaddr != (caddr_t)UINTPTR_MAX);
		*minaddrp = (uintptr_t)minaddr;
	}

	if (brkbase != NULL && secflag_enabled(curproc, PROC_SEC_ASLR)) {
		size_t off;
		uintptr_t base = (uintptr_t)*brkbase;
		uintptr_t oend = base + *brksize;

		ASSERT(ISP2(aslr_max_brk_skew));

		(void) random_get_pseudo_bytes((uint8_t *)&off, sizeof (off));
		base += P2PHASE(off, aslr_max_brk_skew);
		base = P2ROUNDUP(base, PAGESIZE);
		*brkbase = (caddr_t)base;
		/*
		 * Above, we set *brksize to account for the possibility we
		 * had to grow the 'brk' in padding out the BSS to a page
		 * boundary.
		 *
		 * We now need to adjust that based on where we now are
		 * actually putting the brk.
		 */
		if (oend > base)
			*brksize = oend - base;
		else
			*brksize = 0;
	}

	return (0);
bad:
	if (error == 0)
		error = EINVAL;
	return (error);
}

int
elfnote(vnode_t *vp, offset_t *offsetp, int type, int descsz, void *desc,
    rlim64_t rlimit, cred_t *credp)
{
	Note note;
	int error;

	bzero(&note, sizeof (note));
	bcopy("CORE", note.name, 4);
	note.nhdr.n_type = type;
	/*
	 * The System V ABI states that n_namesz must be the length of the
	 * string that follows the Nhdr structure including the terminating
	 * null. The ABI also specifies that sufficient padding should be
	 * included so that the description that follows the name string
	 * begins on a 4- or 8-byte boundary for 32- and 64-bit binaries
	 * respectively. However, since this change was not made correctly
	 * at the time of the 64-bit port, both 32- and 64-bit binaries
	 * descriptions are only guaranteed to begin on a 4-byte boundary.
	 */
	note.nhdr.n_namesz = 5;
	note.nhdr.n_descsz = roundup(descsz, sizeof (Word));

	if (error = core_write(vp, UIO_SYSSPACE, *offsetp, &note,
	    sizeof (note), rlimit, credp))
		return (error);

	*offsetp += sizeof (note);

	if (error = core_write(vp, UIO_SYSSPACE, *offsetp, desc,
	    note.nhdr.n_descsz, rlimit, credp))
		return (error);

	*offsetp += note.nhdr.n_descsz;
	return (0);
}

/*
 * Copy the section data from one vnode to the section of another vnode.
 */
static void
elf_copy_scn(elf_core_ctx_t *ctx, const Shdr *src, vnode_t *src_vp, Shdr *dst)
{
	size_t n = src->sh_size;
	u_offset_t off = 0;
	const u_offset_t soff = src->sh_offset;
	const u_offset_t doff = ctx->ecc_doffset;
	void *buf = ctx->ecc_buf;
	vnode_t *dst_vp = ctx->ecc_vp;
	cred_t *credp = ctx->ecc_credp;

	/* Protect the copy loop below from overflow on the offsets */
	if (n > OFF_MAX || (n + soff) > OFF_MAX || (n + doff) > OFF_MAX ||
	    (n + soff) < n || (n + doff) < n) {
		dst->sh_size = 0;
		dst->sh_offset = 0;
		return;
	}

	while (n != 0) {
		const size_t len = MIN(ctx->ecc_bufsz, n);
		ssize_t resid;

		if (vn_rdwr(UIO_READ, src_vp, buf, (ssize_t)len,
		    (offset_t)(soff + off),
		    UIO_SYSSPACE, 0, (rlim64_t)0, credp, &resid) != 0 ||
		    resid >= len || resid < 0 ||
		    core_write(dst_vp, UIO_SYSSPACE, (offset_t)(doff + off),
		    buf, len - resid, ctx->ecc_rlimit, credp) != 0) {
			dst->sh_size = 0;
			dst->sh_offset = 0;
			return;
		}

		ASSERT(n >= len - resid);

		n -= len - resid;
		off += len - resid;
	}

	ctx->ecc_doffset += src->sh_size;
}

/*
 * Walk sections for a given ELF object, counting (or copying) those of
 * interest (CTF, symtab, strtab, .debug_*).
 */
static int
elf_process_obj_scns(elf_core_ctx_t *ctx, vnode_t *mvp, caddr_t saddr,
    Shdr *v, uint_t idx, const uint_t remain, shstrtab_t *shstrtab,
    uint_t *countp)
{
	Ehdr ehdr;
	const core_content_t content = ctx->ecc_content;
	cred_t *credp = ctx->ecc_credp;
	Shdr *ctf = NULL, *symtab = NULL, *strtab = NULL;
	uintptr_t off = 0;
	uint_t nshdrs, shstrndx, nphdrs, count, extra;
	u_offset_t *doffp = &ctx->ecc_doffset;
	boolean_t ctf_link = B_FALSE;
	caddr_t shbase;
	size_t shsize, shstrsize;
	char *shstrbase;
	int error = 0;
	const boolean_t justcounting = (v == NULL);

	/*
	 * remain must be less than UINT_MAX so we can check for count
	 * exceeding it.
	 */
	ASSERT3U(remain, <, UINT_MAX);

	*countp = count = 0;

	if ((content &
	    (CC_CONTENT_CTF | CC_CONTENT_SYMTAB | CC_CONTENT_DEBUG)) == 0) {
		return (0);
	}

	if (getelfhead(mvp, credp, &ehdr, &nshdrs, &shstrndx, &nphdrs) != 0 ||
	    getelfshdr(mvp, credp, &ehdr, nshdrs, shstrndx, &shbase, &shsize,
	    &shstrbase, &shstrsize) != 0) {
		return (0);
	}

	/* Starting at index 1 skips SHT_NULL which is expected at index 0 */
	off = ehdr.e_shentsize;
	for (uint_t i = 1; i < nshdrs; i++, off += ehdr.e_shentsize) {
		Shdr *shdr, *symchk = NULL, *strchk;
		const char *name;

		shdr = (Shdr *)(shbase + off);
		if (shdr->sh_name >= shstrsize || shdr->sh_type == SHT_NULL)
			continue;

		name = shstrbase + shdr->sh_name;

		if (ctf == NULL && (content & CC_CONTENT_CTF) != 0 &&
		    strcmp(name, shstrtab_data[STR_CTF]) == 0) {
			ctf = shdr;
			if (ctf->sh_link != 0 && ctf->sh_link < nshdrs) {
				/* check linked symtab below */
				symchk = (Shdr *)(shbase +
				    shdr->sh_link * ehdr.e_shentsize);
				ctf_link = B_TRUE;
			} else {
				continue;
			}
		} else if (symtab == NULL &&
		    (content & CC_CONTENT_SYMTAB) != 0 &&
		    strcmp(name, shstrtab_data[STR_SYMTAB]) == 0) {
			symchk = shdr;
		} else if ((content & CC_CONTENT_DEBUG) != 0 &&
		    strncmp(name, ".debug_", strlen(".debug_")) == 0) {
			/*
			 * The design of the above check is intentional. In
			 * particular, we want to capture any sections that
			 * begin with '.debug_' for a few reasons:
			 *
			 * 1) Various revisions to the DWARF spec end up
			 * changing the set of section headers that exist. This
			 * ensures that we don't need to change the kernel to
			 * get a new version.
			 *
			 * 2) Other software uses .debug_ sections for things
			 * which aren't DWARF. This allows them to be captured
			 * as well.
			 */
			count++;

			if (count > remain) {
				error = ENOMEM;
				goto done;
			}

			if (justcounting)
				continue;

			elf_ctx_resize_scratch(ctx, shdr->sh_size);

			if (!shstrtab_ndx(shstrtab, name, &v[idx].sh_name)) {
				error = ENOMEM;
				goto done;
			}

			v[idx].sh_addr = (Addr)(uintptr_t)saddr;
			v[idx].sh_type = shdr->sh_type;
			v[idx].sh_addralign = shdr->sh_addralign;
			*doffp = roundup(*doffp, v[idx].sh_addralign);
			v[idx].sh_offset = *doffp;
			v[idx].sh_size = shdr->sh_size;
			v[idx].sh_link = 0;
			v[idx].sh_entsize = shdr->sh_entsize;
			v[idx].sh_info = shdr->sh_info;

			elf_copy_scn(ctx, shdr, mvp, &v[idx]);
			idx++;

			continue;
		} else {
			continue;
		}

		ASSERT(symchk != NULL);
		if ((symchk->sh_type != SHT_DYNSYM &&
		    symchk->sh_type != SHT_SYMTAB) ||
		    symchk->sh_link == 0 || symchk->sh_link >= nshdrs) {
			ctf_link = B_FALSE;
			continue;
		}
		strchk = (Shdr *)(shbase + symchk->sh_link * ehdr.e_shentsize);
		if (strchk->sh_type != SHT_STRTAB) {
			ctf_link = B_FALSE;
			continue;
		}
		symtab = symchk;
		strtab = strchk;

		if (symtab != NULL && ctf != NULL &&
		    (content & CC_CONTENT_DEBUG) == 0) {
			/* No other shdrs are of interest at this point */
			break;
		}
	}

	extra = 0;
	if (ctf != NULL)
		extra += 1;
	if (symtab != NULL)
		extra += 2;

	if (remain < extra || count > remain - extra) {
		error = ENOMEM;
		goto done;
	}

	count += extra;

	if (justcounting)
		goto done;

	/* output CTF section */
	if (ctf != NULL) {
		elf_ctx_resize_scratch(ctx, ctf->sh_size);

		if (!shstrtab_ndx(shstrtab,
		    shstrtab_data[STR_CTF], &v[idx].sh_name)) {
			error = ENOMEM;
			goto done;
		}
		v[idx].sh_addr = (Addr)(uintptr_t)saddr;
		v[idx].sh_type = SHT_PROGBITS;
		v[idx].sh_addralign = 4;
		*doffp = roundup(*doffp, v[idx].sh_addralign);
		v[idx].sh_offset = *doffp;
		v[idx].sh_size = ctf->sh_size;

		if (ctf_link) {
			/*
			 * The linked symtab (and strtab) will be output
			 * immediately after this CTF section.  Its shdr index
			 * directly follows this one.
			 */
			v[idx].sh_link = idx + 1;
			ASSERT(symtab != NULL);
		} else {
			v[idx].sh_link = 0;
		}
		elf_copy_scn(ctx, ctf, mvp, &v[idx]);
		idx++;
	}

	/* output SYMTAB/STRTAB sections */
	if (symtab != NULL) {
		shstrtype_t symtab_type, strtab_type;
		uint_t symtab_name, strtab_name;

		elf_ctx_resize_scratch(ctx,
		    MAX(symtab->sh_size, strtab->sh_size));

		if (symtab->sh_type == SHT_DYNSYM) {
			symtab_type = STR_DYNSYM;
			strtab_type = STR_DYNSTR;
		} else {
			symtab_type = STR_SYMTAB;
			strtab_type = STR_STRTAB;
		}

		if (!shstrtab_ndx(shstrtab,
		    shstrtab_data[symtab_type], &symtab_name)) {
			error = ENOMEM;
			goto done;
		}
		if (!shstrtab_ndx(shstrtab,
		    shstrtab_data[strtab_type], &strtab_name)) {
			error = ENOMEM;
			goto done;
		}

		v[idx].sh_name = symtab_name;
		v[idx].sh_type = symtab->sh_type;
		v[idx].sh_addr = symtab->sh_addr;
		if (ehdr.e_type == ET_DYN || v[idx].sh_addr == 0)
			v[idx].sh_addr += (Addr)(uintptr_t)saddr;
		v[idx].sh_addralign = symtab->sh_addralign;
		*doffp = roundup(*doffp, v[idx].sh_addralign);
		v[idx].sh_offset = *doffp;
		v[idx].sh_size = symtab->sh_size;
		v[idx].sh_link = idx + 1;
		v[idx].sh_entsize = symtab->sh_entsize;
		v[idx].sh_info = symtab->sh_info;

		elf_copy_scn(ctx, symtab, mvp, &v[idx]);
		idx++;

		v[idx].sh_name = strtab_name;
		v[idx].sh_type = SHT_STRTAB;
		v[idx].sh_flags = SHF_STRINGS;
		v[idx].sh_addr = strtab->sh_addr;
		if (ehdr.e_type == ET_DYN || v[idx].sh_addr == 0)
			v[idx].sh_addr += (Addr)(uintptr_t)saddr;
		v[idx].sh_addralign = strtab->sh_addralign;
		*doffp = roundup(*doffp, v[idx].sh_addralign);
		v[idx].sh_offset = *doffp;
		v[idx].sh_size = strtab->sh_size;

		elf_copy_scn(ctx, strtab, mvp, &v[idx]);
		idx++;
	}

done:
	kmem_free(shstrbase, shstrsize);
	kmem_free(shbase, shsize);

	if (error == 0)
		*countp = count;

	return (error);
}

/*
 * Walk mappings in process address space, examining those which correspond to
 * loaded objects.  It is called twice from elfcore: Once to simply count
 * relevant sections, and again later to copy those sections once an adequate
 * buffer has been allocated for the shdr details.
 */
static int
elf_process_scns(elf_core_ctx_t *ctx, Shdr *v, const uint_t nv, uint_t *nshdrsp)
{
	vnode_t *lastvp = NULL;
	struct seg *seg;
	uint_t remain, idx;
	shstrtab_t shstrtab;
	struct as *as = ctx->ecc_p->p_as;
	int error = 0;
	const boolean_t justcounting = (v == NULL);

	ASSERT(AS_WRITE_HELD(as));

	if (justcounting) {
		ASSERT(nv == 0);
		/*
		 * In the counting case, set remain to UINT_MAX so that we
		 * allow up to that many sections. Note that remain is
		 * decremented immediately below to account for the SHT_NULL
		 * section at index zero and so we do not end up passing
		 * UINT_MAX as the 'remain' value to elf_process_obj_scns().
		 * Once we've finished counting, we further check that there
		 * is at least one array slot available for shstrtab.
		 */
		remain = UINT_MAX;
	} else {
		ASSERT(nv != 0);
		remain = nv;

		if (!shstrtab_init(&shstrtab))
			return (ENOMEM);
	}

	/* Per the ELF spec, shdr index 0 is reserved. */
	idx = 1;
	remain--;
	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
		vnode_t *mvp;
		void *tmp = NULL;
		caddr_t saddr = seg->s_base, naddr, eaddr;
		size_t segsize;
		uint_t count, prot;

		/*
		 * Since we're just looking for text segments of load
		 * objects, we only care about the protection bits; we don't
		 * care about the actual size of the segment so we use the
		 * reserved size. If the segment's size is zero, there's
		 * something fishy going on so we ignore this segment.
		 */
		if (seg->s_ops != &segvn_ops ||
		    SEGOP_GETVP(seg, seg->s_base, &mvp) != 0 ||
		    mvp == lastvp || mvp == NULL || mvp->v_type != VREG ||
		    (segsize = pr_getsegsize(seg, 1)) == 0) {
			continue;
		}

		eaddr = saddr + segsize;
		prot = pr_getprot(seg, 1, &tmp, &saddr, &naddr, eaddr);
		pr_getprot_done(&tmp);

		/*
		 * Skip this segment unless the protection bits look like
		 * what we'd expect for a text segment.
		 */
		if ((prot & (PROT_WRITE | PROT_EXEC)) != PROT_EXEC)
			continue;

		error = elf_process_obj_scns(ctx, mvp, saddr, v, idx, remain,
		    &shstrtab, &count);
		if (error != 0)
			goto done;

		VERIFY3U(count, <=, remain);
		if (!justcounting) {
			VERIFY3U(idx + count, <=, nv);
		}

		remain -= count;
		idx += count;
		lastvp = mvp;
	}

	if (justcounting) {
		if (idx == 1) {
			/* No sections found */
			*nshdrsp = 0;
		} else if (remain < 1) {
			/* No space for the shrstrtab at the end */
			*nshdrsp = 0;
			return (ENOMEM);
		} else {
			/* Include room for the shrstrtab at the end */
			*nshdrsp = idx + 1;
		}
		return (0);
	}

	if (remain != 1) {
		cmn_err(CE_WARN, "elfcore: core dump failed for "
		    "process %d; address space is changing",
		    ctx->ecc_p->p_pid);
		error = EIO;
		goto done;
	}

	if (!shstrtab_ndx(&shstrtab, shstrtab_data[STR_SHSTRTAB],
	    &v[idx].sh_name)) {
		error = ENOMEM;
		goto done;
	}
	v[idx].sh_size = shstrtab_size(&shstrtab);
	v[idx].sh_addralign = 1;
	v[idx].sh_offset = ctx->ecc_doffset;
	v[idx].sh_flags = SHF_STRINGS;
	v[idx].sh_type = SHT_STRTAB;

	elf_ctx_resize_scratch(ctx, v[idx].sh_size);
	VERIFY3U(ctx->ecc_bufsz, >=, v[idx].sh_size);
	shstrtab_dump(&shstrtab, ctx->ecc_buf);

	error = core_write(ctx->ecc_vp, UIO_SYSSPACE, ctx->ecc_doffset,
	    ctx->ecc_buf, v[idx].sh_size, ctx->ecc_rlimit, ctx->ecc_credp);
	if (error == 0)
		ctx->ecc_doffset += v[idx].sh_size;

done:
	if (!justcounting)
		shstrtab_fini(&shstrtab);

	return (error);
}

int
elfcore(vnode_t *vp, proc_t *p, cred_t *credp, rlim64_t rlimit, int sig,
    core_content_t content)
{
	u_offset_t poffset, soffset, doffset;
	int error;
	uint_t i, nphdrs, nshdrs;
	struct seg *seg;
	struct as *as = p->p_as;
	void *bigwad, *zeropg = NULL;
	size_t bigsize, phdrsz, shdrsz;
	Ehdr *ehdr;
	Phdr *phdr;
	Shdr shdr0;
	caddr_t brkbase, stkbase;
	size_t brksize, stksize;
	boolean_t overflowed = B_FALSE, retried = B_FALSE;
	klwp_t *lwp = ttolwp(curthread);
	elf_core_ctx_t ctx = {
		.ecc_vp = vp,
		.ecc_p = p,
		.ecc_credp = credp,
		.ecc_rlimit = rlimit,
		.ecc_content = content,
		.ecc_doffset = 0,
		.ecc_buf = NULL,
		.ecc_bufsz = 0
	};

top:
	/*
	 * Make sure we have everything we need (registers, etc.).
	 * All other lwps have already stopped and are in an orderly state.
	 */
	ASSERT(p == ttoproc(curthread));
	prstop(0, 0);

	AS_LOCK_ENTER(as, RW_WRITER);
	nphdrs = prnsegs(as, 0) + 2;		/* two CORE note sections */

	/*
	 * Count the number of section headers we're going to need.
	 */
	nshdrs = error = 0;
	if (content & (CC_CONTENT_CTF | CC_CONTENT_SYMTAB | CC_CONTENT_DEBUG))
		error = elf_process_scns(&ctx, NULL, 0, &nshdrs);
	AS_LOCK_EXIT(as);

	if (error != 0)
		return (error);

	/*
	 * The core file contents may require zero section headers, but if
	 * we overflow the 16 bits allotted to the program header count in
	 * the ELF header, we'll need that program header at index zero.
	 */
	if (nshdrs == 0 && nphdrs >= PN_XNUM)
		nshdrs = 1;

	/*
	 * Allocate a buffer which is sized adequately to hold the ehdr, phdrs
	 * or shdrs needed to produce the core file.  It is used for the three
	 * tasks sequentially, not simultaneously, so it does not need space
	 * for all three data at once, only the largest one.
	 */
	VERIFY3U(nphdrs, >=, 2);
	phdrsz = nphdrs * sizeof (Phdr);
	shdrsz = nshdrs * sizeof (Shdr);
	bigsize = MAX(sizeof (Ehdr), MAX(phdrsz, shdrsz));
	bigwad = kmem_alloc(bigsize, KM_SLEEP);

	ehdr = (Ehdr *)bigwad;
	bzero(ehdr, sizeof (*ehdr));

	ehdr->e_ident[EI_MAG0] = ELFMAG0;
	ehdr->e_ident[EI_MAG1] = ELFMAG1;
	ehdr->e_ident[EI_MAG2] = ELFMAG2;
	ehdr->e_ident[EI_MAG3] = ELFMAG3;
	ehdr->e_ident[EI_CLASS] = ELFCLASS;
	ehdr->e_type = ET_CORE;

#if !defined(_LP64) || defined(_ELF32_COMPAT)

#if defined(__sparc)
	ehdr->e_ident[EI_DATA] = ELFDATA2MSB;
	ehdr->e_machine = EM_SPARC;
#elif defined(__i386_COMPAT)
	ehdr->e_ident[EI_DATA] = ELFDATA2LSB;
	ehdr->e_machine = EM_386;
#else
#error "no recognized machine type is defined"
#endif

#else	/* !defined(_LP64) || defined(_ELF32_COMPAT) */

#if defined(__sparc)
	ehdr->e_ident[EI_DATA] = ELFDATA2MSB;
	ehdr->e_machine = EM_SPARCV9;
#elif defined(__amd64)
	ehdr->e_ident[EI_DATA] = ELFDATA2LSB;
	ehdr->e_machine = EM_AMD64;
#else
#error "no recognized 64-bit machine type is defined"
#endif

#endif	/* !defined(_LP64) || defined(_ELF32_COMPAT) */

	poffset = sizeof (Ehdr);
	soffset = sizeof (Ehdr) + phdrsz;
	doffset = sizeof (Ehdr) + phdrsz + shdrsz;
	bzero(&shdr0, sizeof (shdr0));

	/*
	 * If the count of program headers or section headers or the index
	 * of the section string table can't fit in the mere 16 bits
	 * shortsightedly allotted to them in the ELF header, we use the
	 * extended formats and put the real values in the section header
	 * as index 0.
	 */
	if (nphdrs >= PN_XNUM) {
		ehdr->e_phnum = PN_XNUM;
		shdr0.sh_info = nphdrs;
	} else {
		ehdr->e_phnum = (unsigned short)nphdrs;
	}

	if (nshdrs > 0) {
		if (nshdrs >= SHN_LORESERVE) {
			ehdr->e_shnum = 0;
			shdr0.sh_size = nshdrs;
		} else {
			ehdr->e_shnum = (unsigned short)nshdrs;
		}

		if (nshdrs - 1 >= SHN_LORESERVE) {
			ehdr->e_shstrndx = SHN_XINDEX;
			shdr0.sh_link = nshdrs - 1;
		} else {
			ehdr->e_shstrndx = (unsigned short)(nshdrs - 1);
		}

		ehdr->e_shoff = soffset;
		ehdr->e_shentsize = sizeof (Shdr);
	}

	ehdr->e_ident[EI_VERSION] = EV_CURRENT;
	ehdr->e_version = EV_CURRENT;
	ehdr->e_ehsize = sizeof (Ehdr);
	ehdr->e_phoff = poffset;
	ehdr->e_phentsize = sizeof (Phdr);

	if (error = core_write(vp, UIO_SYSSPACE, (offset_t)0, ehdr,
	    sizeof (Ehdr), rlimit, credp)) {
		goto done;
	}

	phdr = (Phdr *)bigwad;
	bzero(phdr, phdrsz);

	setup_old_note_header(&phdr[0], p);
	phdr[0].p_offset = doffset = roundup(doffset, sizeof (Word));
	doffset += phdr[0].p_filesz;

	setup_note_header(&phdr[1], p);
	phdr[1].p_offset = doffset = roundup(doffset, sizeof (Word));
	doffset += phdr[1].p_filesz;

	mutex_enter(&p->p_lock);

	brkbase = p->p_brkbase;
	brksize = p->p_brksize;

	stkbase = p->p_usrstack - p->p_stksize;
	stksize = p->p_stksize;

	mutex_exit(&p->p_lock);

	AS_LOCK_ENTER(as, RW_WRITER);
	i = 2;
	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
		caddr_t eaddr = seg->s_base + pr_getsegsize(seg, 0);
		caddr_t saddr, naddr;
		void *tmp = NULL;
		extern struct seg_ops segspt_shmops;

		if ((seg->s_flags & S_HOLE) != 0) {
			continue;
		}

		for (saddr = seg->s_base; saddr < eaddr; saddr = naddr) {
			uint_t prot;
			size_t size;
			int type;
			vnode_t *mvp;

			prot = pr_getprot(seg, 0, &tmp, &saddr, &naddr, eaddr);
			prot &= PROT_READ | PROT_WRITE | PROT_EXEC;
			if ((size = (size_t)(naddr - saddr)) == 0) {
				ASSERT(tmp == NULL);
				continue;
			} else if (i == nphdrs) {
				pr_getprot_done(&tmp);
				overflowed = B_TRUE;
				break;
			}
			phdr[i].p_type = PT_LOAD;
			phdr[i].p_vaddr = (Addr)(uintptr_t)saddr;
			phdr[i].p_memsz = size;
			if (prot & PROT_READ)
				phdr[i].p_flags |= PF_R;
			if (prot & PROT_WRITE)
				phdr[i].p_flags |= PF_W;
			if (prot & PROT_EXEC)
				phdr[i].p_flags |= PF_X;

			/*
			 * Figure out which mappings to include in the core.
			 */
			type = SEGOP_GETTYPE(seg, saddr);

			if (saddr == stkbase && size == stksize) {
				if (!(content & CC_CONTENT_STACK))
					goto exclude;

			} else if (saddr == brkbase && size == brksize) {
				if (!(content & CC_CONTENT_HEAP))
					goto exclude;

			} else if (seg->s_ops == &segspt_shmops) {
				if (type & MAP_NORESERVE) {
					if (!(content & CC_CONTENT_DISM))
						goto exclude;
				} else {
					if (!(content & CC_CONTENT_ISM))
						goto exclude;
				}

			} else if (seg->s_ops != &segvn_ops) {
				goto exclude;

			} else if (type & MAP_SHARED) {
				if (shmgetid(p, saddr) != SHMID_NONE) {
					if (!(content & CC_CONTENT_SHM))
						goto exclude;

				} else if (SEGOP_GETVP(seg, seg->s_base,
				    &mvp) != 0 || mvp == NULL ||
				    mvp->v_type != VREG) {
					if (!(content & CC_CONTENT_SHANON))
						goto exclude;

				} else {
					if (!(content & CC_CONTENT_SHFILE))
						goto exclude;
				}

			} else if (SEGOP_GETVP(seg, seg->s_base, &mvp) != 0 ||
			    mvp == NULL || mvp->v_type != VREG) {
				if (!(content & CC_CONTENT_ANON))
					goto exclude;

			} else if (prot == (PROT_READ | PROT_EXEC)) {
				if (!(content & CC_CONTENT_TEXT))
					goto exclude;

			} else if (prot == PROT_READ) {
				if (!(content & CC_CONTENT_RODATA))
					goto exclude;

			} else {
				if (!(content & CC_CONTENT_DATA))
					goto exclude;
			}

			doffset = roundup(doffset, sizeof (Word));
			phdr[i].p_offset = doffset;
			phdr[i].p_filesz = size;
			doffset += size;
exclude:
			i++;
		}
		VERIFY(tmp == NULL);
		if (overflowed)
			break;
	}
	AS_LOCK_EXIT(as);

	if (overflowed || i != nphdrs) {
		if (!retried) {
			retried = B_TRUE;
			overflowed = B_FALSE;
			kmem_free(bigwad, bigsize);
			goto top;
		}
		cmn_err(CE_WARN, "elfcore: core dump failed for "
		    "process %d; address space is changing", p->p_pid);
		error = EIO;
		goto done;
	}

	if ((error = core_write(vp, UIO_SYSSPACE, poffset,
	    phdr, phdrsz, rlimit, credp)) != 0) {
		goto done;
	}

	if ((error = write_old_elfnotes(p, sig, vp, phdr[0].p_offset, rlimit,
	    credp)) != 0) {
		goto done;
	}
	if ((error = write_elfnotes(p, sig, vp, phdr[1].p_offset, rlimit,
	    credp, content)) != 0) {
		goto done;
	}

	for (i = 2; i < nphdrs; i++) {
		prkillinfo_t killinfo;
		sigqueue_t *sq;
		int sig, j;

		if (phdr[i].p_filesz == 0)
			continue;

		/*
		 * If we hit a region that was mapped PROT_NONE then we cannot
		 * continue dumping this normally as the kernel would be unable
		 * to read from the page and that would result in us failing to
		 * dump the page. As such, any region mapped PROT_NONE, we dump
		 * as a zero-filled page such that this is still represented in
		 * the map.
		 *
		 * If dumping out this segment fails, rather than failing
		 * the core dump entirely, we reset the size of the mapping
		 * to zero to indicate that the data is absent from the core
		 * file and or in the PF_SUNW_FAILURE flag to differentiate
		 * this from mappings that were excluded due to the core file
		 * content settings.
		 */
		if ((phdr[i].p_flags & (PF_R | PF_W | PF_X)) == 0) {
			size_t towrite = phdr[i].p_filesz;
			size_t curoff = 0;

			if (zeropg == NULL) {
				zeropg = kmem_zalloc(elf_zeropg_sz, KM_SLEEP);
			}

			error = 0;
			while (towrite != 0) {
				size_t len = MIN(towrite, elf_zeropg_sz);

				error = core_write(vp, UIO_SYSSPACE,
				    phdr[i].p_offset + curoff, zeropg, len,
				    rlimit, credp);
				if (error != 0)
					break;

				towrite -= len;
				curoff += len;
			}
		} else {
			error = core_seg(p, vp, phdr[i].p_offset,
			    (caddr_t)(uintptr_t)phdr[i].p_vaddr,
			    phdr[i].p_filesz, rlimit, credp);
		}
		if (error == 0)
			continue;

		if ((sig = lwp->lwp_cursig) == 0) {
			/*
			 * We failed due to something other than a signal.
			 * Since the space reserved for the segment is now
			 * unused, we stash the errno in the first four
			 * bytes. This undocumented interface will let us
			 * understand the nature of the failure.
			 */
			(void) core_write(vp, UIO_SYSSPACE, phdr[i].p_offset,
			    &error, sizeof (error), rlimit, credp);

			phdr[i].p_filesz = 0;
			phdr[i].p_flags |= PF_SUNW_FAILURE;
			if ((error = core_write(vp, UIO_SYSSPACE,
			    poffset + sizeof (Phdr) * i, &phdr[i],
			    sizeof (Phdr), rlimit, credp)) != 0)
				goto done;

			continue;
		}

		/*
		 * We took a signal.  We want to abort the dump entirely, but
		 * we also want to indicate what failed and why.  We therefore
		 * use the space reserved for the first failing segment to
		 * write our error (which, for purposes of compatability with
		 * older core dump readers, we set to EINTR) followed by any
		 * siginfo associated with the signal.
		 */
		bzero(&killinfo, sizeof (killinfo));
		killinfo.prk_error = EINTR;

		sq = sig == SIGKILL ? curproc->p_killsqp : lwp->lwp_curinfo;

		if (sq != NULL) {
			bcopy(&sq->sq_info, &killinfo.prk_info,
			    sizeof (sq->sq_info));
		} else {
			killinfo.prk_info.si_signo = lwp->lwp_cursig;
			killinfo.prk_info.si_code = SI_NOINFO;
		}

#if (defined(_SYSCALL32_IMPL) || defined(_LP64))
		/*
		 * If this is a 32-bit process, we need to translate from the
		 * native siginfo to the 32-bit variant.  (Core readers must
		 * always have the same data model as their target or must
		 * be aware of -- and compensate for -- data model differences.)
		 */
		if (curproc->p_model == DATAMODEL_ILP32) {
			siginfo32_t si32;

			siginfo_kto32((k_siginfo_t *)&killinfo.prk_info, &si32);
			bcopy(&si32, &killinfo.prk_info, sizeof (si32));
		}
#endif

		(void) core_write(vp, UIO_SYSSPACE, phdr[i].p_offset,
		    &killinfo, sizeof (killinfo), rlimit, credp);

		/*
		 * For the segment on which we took the signal, indicate that
		 * its data now refers to a siginfo.
		 */
		phdr[i].p_filesz = 0;
		phdr[i].p_flags |= PF_SUNW_FAILURE | PF_SUNW_KILLED |
		    PF_SUNW_SIGINFO;

		/*
		 * And for every other segment, indicate that its absence
		 * is due to a signal.
		 */
		for (j = i + 1; j < nphdrs; j++) {
			phdr[j].p_filesz = 0;
			phdr[j].p_flags |= PF_SUNW_FAILURE | PF_SUNW_KILLED;
		}

		/*
		 * Finally, write out our modified program headers.
		 */
		if ((error = core_write(vp, UIO_SYSSPACE,
		    poffset + sizeof (Phdr) * i, &phdr[i],
		    sizeof (Phdr) * (nphdrs - i), rlimit, credp)) != 0) {
			goto done;
		}

		break;
	}

	if (nshdrs > 0) {
		Shdr *shdr = (Shdr *)bigwad;

		bzero(shdr, shdrsz);
		if (nshdrs > 1) {
			ctx.ecc_doffset = doffset;
			AS_LOCK_ENTER(as, RW_WRITER);
			error = elf_process_scns(&ctx, shdr, nshdrs, NULL);
			AS_LOCK_EXIT(as);
			if (error != 0)
				goto done;
		}
		/* Copy any extended format data destined for the first shdr */
		bcopy(&shdr0, shdr, sizeof (shdr0));

		error = core_write(vp, UIO_SYSSPACE, soffset, shdr, shdrsz,
		    rlimit, credp);
	}

done:
	if (zeropg != NULL)
		kmem_free(zeropg, elf_zeropg_sz);
	if (ctx.ecc_bufsz != 0)
		kmem_free(ctx.ecc_buf, ctx.ecc_bufsz);
	kmem_free(bigwad, bigsize);
	return (error);
}

#ifndef	_ELF32_COMPAT

static struct execsw esw = {
#ifdef	_LP64
	elf64magicstr,
#else	/* _LP64 */
	elf32magicstr,
#endif	/* _LP64 */
	0,
	5,
	elfexec,
	elfcore
};

static struct modlexec modlexec = {
	&mod_execops, "exec module for elf", &esw
};

#ifdef	_LP64
extern int elf32exec(vnode_t *vp, execa_t *uap, uarg_t *args,
			intpdata_t *idatap, int level, size_t *execsz,
			int setid, caddr_t exec_file, cred_t *cred,
			int brand_action);
extern int elf32core(vnode_t *vp, proc_t *p, cred_t *credp,
			rlim64_t rlimit, int sig, core_content_t content);

static struct execsw esw32 = {
	elf32magicstr,
	0,
	5,
	elf32exec,
	elf32core
};

static struct modlexec modlexec32 = {
	&mod_execops, "32-bit exec module for elf", &esw32
};
#endif	/* _LP64 */

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modlexec,
#ifdef	_LP64
	(void *)&modlexec32,
#endif	/* _LP64 */
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

#endif	/* !_ELF32_COMPAT */