xref: /freebsd/sys/kern/kern_exec.c (revision 57f804675e65951d630a38d94c07be4a27ae4053)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 1993, David Greenman
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include "opt_capsicum.h"
33 #include "opt_hwpmc_hooks.h"
34 #include "opt_ktrace.h"
35 #include "opt_vm.h"
36 
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/acct.h>
40 #include <sys/capsicum.h>
41 #include <sys/eventhandler.h>
42 #include <sys/exec.h>
43 #include <sys/fcntl.h>
44 #include <sys/filedesc.h>
45 #include <sys/imgact.h>
46 #include <sys/imgact_elf.h>
47 #include <sys/kernel.h>
48 #include <sys/lock.h>
49 #include <sys/malloc.h>
50 #include <sys/mman.h>
51 #include <sys/mount.h>
52 #include <sys/mutex.h>
53 #include <sys/namei.h>
54 #include <sys/pioctl.h>
55 #include <sys/priv.h>
56 #include <sys/proc.h>
57 #include <sys/ptrace.h>
58 #include <sys/resourcevar.h>
59 #include <sys/rwlock.h>
60 #include <sys/sched.h>
61 #include <sys/sdt.h>
62 #include <sys/sf_buf.h>
63 #include <sys/shm.h>
64 #include <sys/signalvar.h>
65 #include <sys/smp.h>
66 #include <sys/stat.h>
67 #include <sys/syscallsubr.h>
68 #include <sys/sysctl.h>
69 #include <sys/sysent.h>
70 #include <sys/sysproto.h>
71 #include <sys/vnode.h>
72 #include <sys/wait.h>
73 #ifdef KTRACE
74 #include <sys/ktrace.h>
75 #endif
76 
77 #include <vm/vm.h>
78 #include <vm/vm_param.h>
79 #include <vm/pmap.h>
80 #include <vm/vm_page.h>
81 #include <vm/vm_map.h>
82 #include <vm/vm_kern.h>
83 #include <vm/vm_extern.h>
84 #include <vm/vm_object.h>
85 #include <vm/vm_pager.h>
86 
87 #ifdef	HWPMC_HOOKS
88 #include <sys/pmckern.h>
89 #endif
90 
91 #include <machine/reg.h>
92 
93 #include <security/audit/audit.h>
94 #include <security/mac/mac_framework.h>
95 
96 #ifdef KDTRACE_HOOKS
97 #include <sys/dtrace_bsd.h>
98 dtrace_execexit_func_t	dtrace_fasttrap_exec;
99 #endif
100 
101 SDT_PROVIDER_DECLARE(proc);
102 SDT_PROBE_DEFINE1(proc, , , exec, "char *");
103 SDT_PROBE_DEFINE1(proc, , , exec__failure, "int");
104 SDT_PROBE_DEFINE1(proc, , , exec__success, "char *");
105 
106 MALLOC_DEFINE(M_PARGS, "proc-args", "Process arguments");
107 
108 int coredump_pack_fileinfo = 1;
109 SYSCTL_INT(_kern, OID_AUTO, coredump_pack_fileinfo, CTLFLAG_RWTUN,
110     &coredump_pack_fileinfo, 0,
111     "Enable file path packing in 'procstat -f' coredump notes");
112 
113 int coredump_pack_vmmapinfo = 1;
114 SYSCTL_INT(_kern, OID_AUTO, coredump_pack_vmmapinfo, CTLFLAG_RWTUN,
115     &coredump_pack_vmmapinfo, 0,
116     "Enable file path packing in 'procstat -v' coredump notes");
117 
118 static int sysctl_kern_ps_strings(SYSCTL_HANDLER_ARGS);
119 static int sysctl_kern_usrstack(SYSCTL_HANDLER_ARGS);
120 static int sysctl_kern_stackprot(SYSCTL_HANDLER_ARGS);
121 static int do_execve(struct thread *td, struct image_args *args,
122     struct mac *mac_p);
123 
124 /* XXX This should be vm_size_t. */
125 SYSCTL_PROC(_kern, KERN_PS_STRINGS, ps_strings, CTLTYPE_ULONG|CTLFLAG_RD|
126     CTLFLAG_CAPRD|CTLFLAG_MPSAFE, NULL, 0, sysctl_kern_ps_strings, "LU", "");
127 
128 /* XXX This should be vm_size_t. */
129 SYSCTL_PROC(_kern, KERN_USRSTACK, usrstack, CTLTYPE_ULONG|CTLFLAG_RD|
130     CTLFLAG_CAPRD|CTLFLAG_MPSAFE, NULL, 0, sysctl_kern_usrstack, "LU", "");
131 
132 SYSCTL_PROC(_kern, OID_AUTO, stackprot, CTLTYPE_INT|CTLFLAG_RD|CTLFLAG_MPSAFE,
133     NULL, 0, sysctl_kern_stackprot, "I", "");
134 
135 u_long ps_arg_cache_limit = PAGE_SIZE / 16;
136 SYSCTL_ULONG(_kern, OID_AUTO, ps_arg_cache_limit, CTLFLAG_RW,
137     &ps_arg_cache_limit, 0, "");
138 
139 static int disallow_high_osrel;
140 SYSCTL_INT(_kern, OID_AUTO, disallow_high_osrel, CTLFLAG_RW,
141     &disallow_high_osrel, 0,
142     "Disallow execution of binaries built for higher version of the world");
143 
144 static int map_at_zero = 0;
145 SYSCTL_INT(_security_bsd, OID_AUTO, map_at_zero, CTLFLAG_RWTUN, &map_at_zero, 0,
146     "Permit processes to map an object at virtual address 0.");
147 
148 static int
149 sysctl_kern_ps_strings(SYSCTL_HANDLER_ARGS)
150 {
151 	struct proc *p;
152 	int error;
153 
154 	p = curproc;
155 #ifdef SCTL_MASK32
156 	if (req->flags & SCTL_MASK32) {
157 		unsigned int val;
158 		val = (unsigned int)p->p_sysent->sv_psstrings;
159 		error = SYSCTL_OUT(req, &val, sizeof(val));
160 	} else
161 #endif
162 		error = SYSCTL_OUT(req, &p->p_sysent->sv_psstrings,
163 		   sizeof(p->p_sysent->sv_psstrings));
164 	return error;
165 }
166 
167 static int
168 sysctl_kern_usrstack(SYSCTL_HANDLER_ARGS)
169 {
170 	struct proc *p;
171 	int error;
172 
173 	p = curproc;
174 #ifdef SCTL_MASK32
175 	if (req->flags & SCTL_MASK32) {
176 		unsigned int val;
177 		val = (unsigned int)p->p_sysent->sv_usrstack;
178 		error = SYSCTL_OUT(req, &val, sizeof(val));
179 	} else
180 #endif
181 		error = SYSCTL_OUT(req, &p->p_sysent->sv_usrstack,
182 		    sizeof(p->p_sysent->sv_usrstack));
183 	return error;
184 }
185 
186 static int
187 sysctl_kern_stackprot(SYSCTL_HANDLER_ARGS)
188 {
189 	struct proc *p;
190 
191 	p = curproc;
192 	return (SYSCTL_OUT(req, &p->p_sysent->sv_stackprot,
193 	    sizeof(p->p_sysent->sv_stackprot)));
194 }
195 
196 /*
197  * Each of the items is a pointer to a `const struct execsw', hence the
198  * double pointer here.
199  */
200 static const struct execsw **execsw;
201 
202 #ifndef _SYS_SYSPROTO_H_
203 struct execve_args {
204 	char    *fname;
205 	char    **argv;
206 	char    **envv;
207 };
208 #endif
209 
210 int
211 sys_execve(struct thread *td, struct execve_args *uap)
212 {
213 	struct image_args args;
214 	struct vmspace *oldvmspace;
215 	int error;
216 
217 	error = pre_execve(td, &oldvmspace);
218 	if (error != 0)
219 		return (error);
220 	error = exec_copyin_args(&args, uap->fname, UIO_USERSPACE,
221 	    uap->argv, uap->envv);
222 	if (error == 0)
223 		error = kern_execve(td, &args, NULL);
224 	post_execve(td, error, oldvmspace);
225 	return (error);
226 }
227 
228 #ifndef _SYS_SYSPROTO_H_
229 struct fexecve_args {
230 	int	fd;
231 	char	**argv;
232 	char	**envv;
233 }
234 #endif
235 int
236 sys_fexecve(struct thread *td, struct fexecve_args *uap)
237 {
238 	struct image_args args;
239 	struct vmspace *oldvmspace;
240 	int error;
241 
242 	error = pre_execve(td, &oldvmspace);
243 	if (error != 0)
244 		return (error);
245 	error = exec_copyin_args(&args, NULL, UIO_SYSSPACE,
246 	    uap->argv, uap->envv);
247 	if (error == 0) {
248 		args.fd = uap->fd;
249 		error = kern_execve(td, &args, NULL);
250 	}
251 	post_execve(td, error, oldvmspace);
252 	return (error);
253 }
254 
255 #ifndef _SYS_SYSPROTO_H_
256 struct __mac_execve_args {
257 	char	*fname;
258 	char	**argv;
259 	char	**envv;
260 	struct mac	*mac_p;
261 };
262 #endif
263 
264 int
265 sys___mac_execve(struct thread *td, struct __mac_execve_args *uap)
266 {
267 #ifdef MAC
268 	struct image_args args;
269 	struct vmspace *oldvmspace;
270 	int error;
271 
272 	error = pre_execve(td, &oldvmspace);
273 	if (error != 0)
274 		return (error);
275 	error = exec_copyin_args(&args, uap->fname, UIO_USERSPACE,
276 	    uap->argv, uap->envv);
277 	if (error == 0)
278 		error = kern_execve(td, &args, uap->mac_p);
279 	post_execve(td, error, oldvmspace);
280 	return (error);
281 #else
282 	return (ENOSYS);
283 #endif
284 }
285 
286 int
287 pre_execve(struct thread *td, struct vmspace **oldvmspace)
288 {
289 	struct proc *p;
290 	int error;
291 
292 	KASSERT(td == curthread, ("non-current thread %p", td));
293 	error = 0;
294 	p = td->td_proc;
295 	if ((p->p_flag & P_HADTHREADS) != 0) {
296 		PROC_LOCK(p);
297 		if (thread_single(p, SINGLE_BOUNDARY) != 0)
298 			error = ERESTART;
299 		PROC_UNLOCK(p);
300 	}
301 	KASSERT(error != 0 || (td->td_pflags & TDP_EXECVMSPC) == 0,
302 	    ("nested execve"));
303 	*oldvmspace = p->p_vmspace;
304 	return (error);
305 }
306 
307 void
308 post_execve(struct thread *td, int error, struct vmspace *oldvmspace)
309 {
310 	struct proc *p;
311 
312 	KASSERT(td == curthread, ("non-current thread %p", td));
313 	p = td->td_proc;
314 	if ((p->p_flag & P_HADTHREADS) != 0) {
315 		PROC_LOCK(p);
316 		/*
317 		 * If success, we upgrade to SINGLE_EXIT state to
318 		 * force other threads to suicide.
319 		 */
320 		if (error == EJUSTRETURN)
321 			thread_single(p, SINGLE_EXIT);
322 		else
323 			thread_single_end(p, SINGLE_BOUNDARY);
324 		PROC_UNLOCK(p);
325 	}
326 	if ((td->td_pflags & TDP_EXECVMSPC) != 0) {
327 		KASSERT(p->p_vmspace != oldvmspace,
328 		    ("oldvmspace still used"));
329 		vmspace_free(oldvmspace);
330 		td->td_pflags &= ~TDP_EXECVMSPC;
331 	}
332 }
333 
334 /*
335  * XXX: kern_execve has the astonishing property of not always returning to
336  * the caller.  If sufficiently bad things happen during the call to
337  * do_execve(), it can end up calling exit1(); as a result, callers must
338  * avoid doing anything which they might need to undo (e.g., allocating
339  * memory).
340  */
341 int
342 kern_execve(struct thread *td, struct image_args *args, struct mac *mac_p)
343 {
344 
345 	AUDIT_ARG_ARGV(args->begin_argv, args->argc,
346 	    exec_args_get_begin_envv(args) - args->begin_argv);
347 	AUDIT_ARG_ENVV(exec_args_get_begin_envv(args), args->envc,
348 	    args->endp - exec_args_get_begin_envv(args));
349 	return (do_execve(td, args, mac_p));
350 }
351 
352 /*
353  * In-kernel implementation of execve().  All arguments are assumed to be
354  * userspace pointers from the passed thread.
355  */
356 static int
357 do_execve(struct thread *td, struct image_args *args, struct mac *mac_p)
358 {
359 	struct proc *p = td->td_proc;
360 	struct nameidata nd;
361 	struct ucred *oldcred;
362 	struct uidinfo *euip = NULL;
363 	uintptr_t stack_base;
364 	struct image_params image_params, *imgp;
365 	struct vattr attr;
366 	int (*img_first)(struct image_params *);
367 	struct pargs *oldargs = NULL, *newargs = NULL;
368 	struct sigacts *oldsigacts = NULL, *newsigacts = NULL;
369 #ifdef KTRACE
370 	struct vnode *tracevp = NULL;
371 	struct ucred *tracecred = NULL;
372 #endif
373 	struct vnode *oldtextvp = NULL, *newtextvp;
374 	int credential_changing;
375 #ifdef MAC
376 	struct label *interpvplabel = NULL;
377 	int will_transition;
378 #endif
379 #ifdef HWPMC_HOOKS
380 	struct pmckern_procexec pe;
381 #endif
382 	int error, i, orig_osrel;
383 	uint32_t orig_fctl0;
384 	static const char fexecv_proc_title[] = "(fexecv)";
385 
386 	imgp = &image_params;
387 
388 	/*
389 	 * Lock the process and set the P_INEXEC flag to indicate that
390 	 * it should be left alone until we're done here.  This is
391 	 * necessary to avoid race conditions - e.g. in ptrace() -
392 	 * that might allow a local user to illicitly obtain elevated
393 	 * privileges.
394 	 */
395 	PROC_LOCK(p);
396 	KASSERT((p->p_flag & P_INEXEC) == 0,
397 	    ("%s(): process already has P_INEXEC flag", __func__));
398 	p->p_flag |= P_INEXEC;
399 	PROC_UNLOCK(p);
400 
401 	/*
402 	 * Initialize part of the common data
403 	 */
404 	bzero(imgp, sizeof(*imgp));
405 	imgp->proc = p;
406 	imgp->attr = &attr;
407 	imgp->args = args;
408 	oldcred = p->p_ucred;
409 	orig_osrel = p->p_osrel;
410 	orig_fctl0 = p->p_fctl0;
411 
412 #ifdef MAC
413 	error = mac_execve_enter(imgp, mac_p);
414 	if (error)
415 		goto exec_fail;
416 #endif
417 
418 	/*
419 	 * Translate the file name. namei() returns a vnode pointer
420 	 *	in ni_vp among other things.
421 	 *
422 	 * XXXAUDIT: It would be desirable to also audit the name of the
423 	 * interpreter if this is an interpreted binary.
424 	 */
425 	if (args->fname != NULL) {
426 		NDINIT(&nd, LOOKUP, ISOPEN | LOCKLEAF | LOCKSHARED | FOLLOW |
427 		    SAVENAME | AUDITVNODE1, UIO_SYSSPACE, args->fname, td);
428 	}
429 
430 	SDT_PROBE1(proc, , , exec, args->fname);
431 
432 interpret:
433 	if (args->fname != NULL) {
434 #ifdef CAPABILITY_MODE
435 		/*
436 		 * While capability mode can't reach this point via direct
437 		 * path arguments to execve(), we also don't allow
438 		 * interpreters to be used in capability mode (for now).
439 		 * Catch indirect lookups and return a permissions error.
440 		 */
441 		if (IN_CAPABILITY_MODE(td)) {
442 			error = ECAPMODE;
443 			goto exec_fail;
444 		}
445 #endif
446 		error = namei(&nd);
447 		if (error)
448 			goto exec_fail;
449 
450 		newtextvp = nd.ni_vp;
451 		imgp->vp = newtextvp;
452 	} else {
453 		AUDIT_ARG_FD(args->fd);
454 		/*
455 		 * Descriptors opened only with O_EXEC or O_RDONLY are allowed.
456 		 */
457 		error = fgetvp_exec(td, args->fd, &cap_fexecve_rights, &newtextvp);
458 		if (error)
459 			goto exec_fail;
460 		vn_lock(newtextvp, LK_SHARED | LK_RETRY);
461 		AUDIT_ARG_VNODE1(newtextvp);
462 		imgp->vp = newtextvp;
463 	}
464 
465 	/*
466 	 * Check file permissions.  Also 'opens' file and sets its vnode to
467 	 * text mode.
468 	 */
469 	error = exec_check_permissions(imgp);
470 	if (error)
471 		goto exec_fail_dealloc;
472 
473 	imgp->object = imgp->vp->v_object;
474 	if (imgp->object != NULL)
475 		vm_object_reference(imgp->object);
476 
477 	error = exec_map_first_page(imgp);
478 	if (error)
479 		goto exec_fail_dealloc;
480 
481 	imgp->proc->p_osrel = 0;
482 	imgp->proc->p_fctl0 = 0;
483 
484 	/*
485 	 * Implement image setuid/setgid.
486 	 *
487 	 * Determine new credentials before attempting image activators
488 	 * so that it can be used by process_exec handlers to determine
489 	 * credential/setid changes.
490 	 *
491 	 * Don't honor setuid/setgid if the filesystem prohibits it or if
492 	 * the process is being traced.
493 	 *
494 	 * We disable setuid/setgid/etc in capability mode on the basis
495 	 * that most setugid applications are not written with that
496 	 * environment in mind, and will therefore almost certainly operate
497 	 * incorrectly. In principle there's no reason that setugid
498 	 * applications might not be useful in capability mode, so we may want
499 	 * to reconsider this conservative design choice in the future.
500 	 *
501 	 * XXXMAC: For the time being, use NOSUID to also prohibit
502 	 * transitions on the file system.
503 	 */
504 	credential_changing = 0;
505 	credential_changing |= (attr.va_mode & S_ISUID) &&
506 	    oldcred->cr_uid != attr.va_uid;
507 	credential_changing |= (attr.va_mode & S_ISGID) &&
508 	    oldcred->cr_gid != attr.va_gid;
509 #ifdef MAC
510 	will_transition = mac_vnode_execve_will_transition(oldcred, imgp->vp,
511 	    interpvplabel, imgp);
512 	credential_changing |= will_transition;
513 #endif
514 
515 	/* Don't inherit PROC_PDEATHSIG_CTL value if setuid/setgid. */
516 	if (credential_changing)
517 		imgp->proc->p_pdeathsig = 0;
518 
519 	if (credential_changing &&
520 #ifdef CAPABILITY_MODE
521 	    ((oldcred->cr_flags & CRED_FLAG_CAPMODE) == 0) &&
522 #endif
523 	    (imgp->vp->v_mount->mnt_flag & MNT_NOSUID) == 0 &&
524 	    (p->p_flag & P_TRACED) == 0) {
525 		imgp->credential_setid = true;
526 		VOP_UNLOCK(imgp->vp);
527 		imgp->newcred = crdup(oldcred);
528 		if (attr.va_mode & S_ISUID) {
529 			euip = uifind(attr.va_uid);
530 			change_euid(imgp->newcred, euip);
531 		}
532 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
533 		if (attr.va_mode & S_ISGID)
534 			change_egid(imgp->newcred, attr.va_gid);
535 		/*
536 		 * Implement correct POSIX saved-id behavior.
537 		 *
538 		 * XXXMAC: Note that the current logic will save the
539 		 * uid and gid if a MAC domain transition occurs, even
540 		 * though maybe it shouldn't.
541 		 */
542 		change_svuid(imgp->newcred, imgp->newcred->cr_uid);
543 		change_svgid(imgp->newcred, imgp->newcred->cr_gid);
544 	} else {
545 		/*
546 		 * Implement correct POSIX saved-id behavior.
547 		 *
548 		 * XXX: It's not clear that the existing behavior is
549 		 * POSIX-compliant.  A number of sources indicate that the
550 		 * saved uid/gid should only be updated if the new ruid is
551 		 * not equal to the old ruid, or the new euid is not equal
552 		 * to the old euid and the new euid is not equal to the old
553 		 * ruid.  The FreeBSD code always updates the saved uid/gid.
554 		 * Also, this code uses the new (replaced) euid and egid as
555 		 * the source, which may or may not be the right ones to use.
556 		 */
557 		if (oldcred->cr_svuid != oldcred->cr_uid ||
558 		    oldcred->cr_svgid != oldcred->cr_gid) {
559 			VOP_UNLOCK(imgp->vp);
560 			imgp->newcred = crdup(oldcred);
561 			vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
562 			change_svuid(imgp->newcred, imgp->newcred->cr_uid);
563 			change_svgid(imgp->newcred, imgp->newcred->cr_gid);
564 		}
565 	}
566 	/* The new credentials are installed into the process later. */
567 
568 	/*
569 	 * Do the best to calculate the full path to the image file.
570 	 */
571 	if (args->fname != NULL && args->fname[0] == '/')
572 		imgp->execpath = args->fname;
573 	else {
574 		VOP_UNLOCK(imgp->vp);
575 		if (vn_fullpath(td, imgp->vp, &imgp->execpath,
576 		    &imgp->freepath) != 0)
577 			imgp->execpath = args->fname;
578 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
579 	}
580 
581 	/*
582 	 *	If the current process has a special image activator it
583 	 *	wants to try first, call it.   For example, emulating shell
584 	 *	scripts differently.
585 	 */
586 	error = -1;
587 	if ((img_first = imgp->proc->p_sysent->sv_imgact_try) != NULL)
588 		error = img_first(imgp);
589 
590 	/*
591 	 *	Loop through the list of image activators, calling each one.
592 	 *	An activator returns -1 if there is no match, 0 on success,
593 	 *	and an error otherwise.
594 	 */
595 	for (i = 0; error == -1 && execsw[i]; ++i) {
596 		if (execsw[i]->ex_imgact == NULL ||
597 		    execsw[i]->ex_imgact == img_first) {
598 			continue;
599 		}
600 		error = (*execsw[i]->ex_imgact)(imgp);
601 	}
602 
603 	if (error) {
604 		if (error == -1)
605 			error = ENOEXEC;
606 		goto exec_fail_dealloc;
607 	}
608 
609 	/*
610 	 * Special interpreter operation, cleanup and loop up to try to
611 	 * activate the interpreter.
612 	 */
613 	if (imgp->interpreted) {
614 		exec_unmap_first_page(imgp);
615 		/*
616 		 * The text reference needs to be removed for scripts.
617 		 * There is a short period before we determine that
618 		 * something is a script where text reference is active.
619 		 * The vnode lock is held over this entire period
620 		 * so nothing should illegitimately be blocked.
621 		 */
622 		MPASS(imgp->textset);
623 		VOP_UNSET_TEXT_CHECKED(newtextvp);
624 		imgp->textset = false;
625 		/* free name buffer and old vnode */
626 		if (args->fname != NULL)
627 			NDFREE(&nd, NDF_ONLY_PNBUF);
628 #ifdef MAC
629 		mac_execve_interpreter_enter(newtextvp, &interpvplabel);
630 #endif
631 		if (imgp->opened) {
632 			VOP_CLOSE(newtextvp, FREAD, td->td_ucred, td);
633 			imgp->opened = 0;
634 		}
635 		vput(newtextvp);
636 		vm_object_deallocate(imgp->object);
637 		imgp->object = NULL;
638 		imgp->credential_setid = false;
639 		if (imgp->newcred != NULL) {
640 			crfree(imgp->newcred);
641 			imgp->newcred = NULL;
642 		}
643 		imgp->execpath = NULL;
644 		free(imgp->freepath, M_TEMP);
645 		imgp->freepath = NULL;
646 		/* set new name to that of the interpreter */
647 		NDINIT(&nd, LOOKUP, ISOPEN | LOCKLEAF | FOLLOW | SAVENAME,
648 		    UIO_SYSSPACE, imgp->interpreter_name, td);
649 		args->fname = imgp->interpreter_name;
650 		goto interpret;
651 	}
652 
653 	/*
654 	 * NB: We unlock the vnode here because it is believed that none
655 	 * of the sv_copyout_strings/sv_fixup operations require the vnode.
656 	 */
657 	VOP_UNLOCK(imgp->vp);
658 
659 	if (disallow_high_osrel &&
660 	    P_OSREL_MAJOR(p->p_osrel) > P_OSREL_MAJOR(__FreeBSD_version)) {
661 		error = ENOEXEC;
662 		uprintf("Osrel %d for image %s too high\n", p->p_osrel,
663 		    imgp->execpath != NULL ? imgp->execpath : "<unresolved>");
664 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
665 		goto exec_fail_dealloc;
666 	}
667 
668 	/* ABI enforces the use of Capsicum. Switch into capabilities mode. */
669 	if (SV_PROC_FLAG(p, SV_CAPSICUM))
670 		sys_cap_enter(td, NULL);
671 
672 	/*
673 	 * Copy out strings (args and env) and initialize stack base.
674 	 */
675 	error = (*p->p_sysent->sv_copyout_strings)(imgp, &stack_base);
676 	if (error != 0) {
677 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
678 		goto exec_fail_dealloc;
679 	}
680 
681 	/*
682 	 * Stack setup.
683 	 */
684 	error = (*p->p_sysent->sv_fixup)(&stack_base, imgp);
685 	if (error != 0) {
686 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
687 		goto exec_fail_dealloc;
688 	}
689 
690 	if (args->fdp != NULL) {
691 		/* Install a brand new file descriptor table. */
692 		fdinstall_remapped(td, args->fdp);
693 		args->fdp = NULL;
694 	} else {
695 		/*
696 		 * Keep on using the existing file descriptor table. For
697 		 * security and other reasons, the file descriptor table
698 		 * cannot be shared after an exec.
699 		 */
700 		fdunshare(td);
701 		/* close files on exec */
702 		fdcloseexec(td);
703 	}
704 
705 	/*
706 	 * Malloc things before we need locks.
707 	 */
708 	i = exec_args_get_begin_envv(imgp->args) - imgp->args->begin_argv;
709 	/* Cache arguments if they fit inside our allowance */
710 	if (ps_arg_cache_limit >= i + sizeof(struct pargs)) {
711 		newargs = pargs_alloc(i);
712 		bcopy(imgp->args->begin_argv, newargs->ar_args, i);
713 	}
714 
715 	/*
716 	 * For security and other reasons, signal handlers cannot
717 	 * be shared after an exec. The new process gets a copy of the old
718 	 * handlers. In execsigs(), the new process will have its signals
719 	 * reset.
720 	 */
721 	if (sigacts_shared(p->p_sigacts)) {
722 		oldsigacts = p->p_sigacts;
723 		newsigacts = sigacts_alloc();
724 		sigacts_copy(newsigacts, oldsigacts);
725 	}
726 
727 	vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
728 
729 	PROC_LOCK(p);
730 	if (oldsigacts)
731 		p->p_sigacts = newsigacts;
732 	/* Stop profiling */
733 	stopprofclock(p);
734 
735 	/* reset caught signals */
736 	execsigs(p);
737 
738 	/* name this process - nameiexec(p, ndp) */
739 	bzero(p->p_comm, sizeof(p->p_comm));
740 	if (args->fname)
741 		bcopy(nd.ni_cnd.cn_nameptr, p->p_comm,
742 		    min(nd.ni_cnd.cn_namelen, MAXCOMLEN));
743 	else if (vn_commname(newtextvp, p->p_comm, sizeof(p->p_comm)) != 0)
744 		bcopy(fexecv_proc_title, p->p_comm, sizeof(fexecv_proc_title));
745 	bcopy(p->p_comm, td->td_name, sizeof(td->td_name));
746 #ifdef KTR
747 	sched_clear_tdname(td);
748 #endif
749 
750 	/*
751 	 * mark as execed, wakeup the process that vforked (if any) and tell
752 	 * it that it now has its own resources back
753 	 */
754 	p->p_flag |= P_EXEC;
755 	if ((p->p_flag2 & P2_NOTRACE_EXEC) == 0)
756 		p->p_flag2 &= ~P2_NOTRACE;
757 	if ((p->p_flag2 & P2_STKGAP_DISABLE_EXEC) == 0)
758 		p->p_flag2 &= ~P2_STKGAP_DISABLE;
759 	if (p->p_flag & P_PPWAIT) {
760 		p->p_flag &= ~(P_PPWAIT | P_PPTRACE);
761 		cv_broadcast(&p->p_pwait);
762 		/* STOPs are no longer ignored, arrange for AST */
763 		signotify(td);
764 	}
765 
766 	/*
767 	 * Implement image setuid/setgid installation.
768 	 */
769 	if (imgp->credential_setid) {
770 		/*
771 		 * Turn off syscall tracing for set-id programs, except for
772 		 * root.  Record any set-id flags first to make sure that
773 		 * we do not regain any tracing during a possible block.
774 		 */
775 		setsugid(p);
776 
777 #ifdef KTRACE
778 		if (p->p_tracecred != NULL &&
779 		    priv_check_cred(p->p_tracecred, PRIV_DEBUG_DIFFCRED))
780 			ktrprocexec(p, &tracecred, &tracevp);
781 #endif
782 		/*
783 		 * Close any file descriptors 0..2 that reference procfs,
784 		 * then make sure file descriptors 0..2 are in use.
785 		 *
786 		 * Both fdsetugidsafety() and fdcheckstd() may call functions
787 		 * taking sleepable locks, so temporarily drop our locks.
788 		 */
789 		PROC_UNLOCK(p);
790 		VOP_UNLOCK(imgp->vp);
791 		fdsetugidsafety(td);
792 		error = fdcheckstd(td);
793 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
794 		if (error != 0)
795 			goto exec_fail_dealloc;
796 		PROC_LOCK(p);
797 #ifdef MAC
798 		if (will_transition) {
799 			mac_vnode_execve_transition(oldcred, imgp->newcred,
800 			    imgp->vp, interpvplabel, imgp);
801 		}
802 #endif
803 	} else {
804 		if (oldcred->cr_uid == oldcred->cr_ruid &&
805 		    oldcred->cr_gid == oldcred->cr_rgid)
806 			p->p_flag &= ~P_SUGID;
807 	}
808 	/*
809 	 * Set the new credentials.
810 	 */
811 	if (imgp->newcred != NULL) {
812 		proc_set_cred(p, imgp->newcred);
813 		crfree(oldcred);
814 		oldcred = NULL;
815 	}
816 
817 	/*
818 	 * Store the vp for use in procfs.  This vnode was referenced by namei
819 	 * or fgetvp_exec.
820 	 */
821 	oldtextvp = p->p_textvp;
822 	p->p_textvp = newtextvp;
823 
824 #ifdef KDTRACE_HOOKS
825 	/*
826 	 * Tell the DTrace fasttrap provider about the exec if it
827 	 * has declared an interest.
828 	 */
829 	if (dtrace_fasttrap_exec)
830 		dtrace_fasttrap_exec(p);
831 #endif
832 
833 	/*
834 	 * Notify others that we exec'd, and clear the P_INEXEC flag
835 	 * as we're now a bona fide freshly-execed process.
836 	 */
837 	KNOTE_LOCKED(p->p_klist, NOTE_EXEC);
838 	p->p_flag &= ~P_INEXEC;
839 
840 	/* clear "fork but no exec" flag, as we _are_ execing */
841 	p->p_acflag &= ~AFORK;
842 
843 	/*
844 	 * Free any previous argument cache and replace it with
845 	 * the new argument cache, if any.
846 	 */
847 	oldargs = p->p_args;
848 	p->p_args = newargs;
849 	newargs = NULL;
850 
851 	PROC_UNLOCK(p);
852 
853 #ifdef	HWPMC_HOOKS
854 	/*
855 	 * Check if system-wide sampling is in effect or if the
856 	 * current process is using PMCs.  If so, do exec() time
857 	 * processing.  This processing needs to happen AFTER the
858 	 * P_INEXEC flag is cleared.
859 	 */
860 	if (PMC_SYSTEM_SAMPLING_ACTIVE() || PMC_PROC_IS_USING_PMCS(p)) {
861 		VOP_UNLOCK(imgp->vp);
862 		pe.pm_credentialschanged = credential_changing;
863 		pe.pm_entryaddr = imgp->entry_addr;
864 
865 		PMC_CALL_HOOK_X(td, PMC_FN_PROCESS_EXEC, (void *) &pe);
866 		vn_lock(imgp->vp, LK_SHARED | LK_RETRY);
867 	}
868 #endif
869 
870 	/* Set values passed into the program in registers. */
871 	(*p->p_sysent->sv_setregs)(td, imgp, stack_base);
872 
873 	VOP_MMAPPED(imgp->vp);
874 
875 	SDT_PROBE1(proc, , , exec__success, args->fname);
876 
877 exec_fail_dealloc:
878 	if (error != 0) {
879 		p->p_osrel = orig_osrel;
880 		p->p_fctl0 = orig_fctl0;
881 	}
882 
883 	if (imgp->firstpage != NULL)
884 		exec_unmap_first_page(imgp);
885 
886 	if (imgp->vp != NULL) {
887 		if (args->fname)
888 			NDFREE(&nd, NDF_ONLY_PNBUF);
889 		if (imgp->opened)
890 			VOP_CLOSE(imgp->vp, FREAD, td->td_ucred, td);
891 		if (imgp->textset)
892 			VOP_UNSET_TEXT_CHECKED(imgp->vp);
893 		if (error != 0)
894 			vput(imgp->vp);
895 		else
896 			VOP_UNLOCK(imgp->vp);
897 	}
898 
899 	if (imgp->object != NULL)
900 		vm_object_deallocate(imgp->object);
901 
902 	free(imgp->freepath, M_TEMP);
903 
904 	if (error == 0) {
905 		if (p->p_ptevents & PTRACE_EXEC) {
906 			PROC_LOCK(p);
907 			if (p->p_ptevents & PTRACE_EXEC)
908 				td->td_dbgflags |= TDB_EXEC;
909 			PROC_UNLOCK(p);
910 		}
911 
912 		/*
913 		 * Stop the process here if its stop event mask has
914 		 * the S_EXEC bit set.
915 		 */
916 		STOPEVENT(p, S_EXEC, 0);
917 	} else {
918 exec_fail:
919 		/* we're done here, clear P_INEXEC */
920 		PROC_LOCK(p);
921 		p->p_flag &= ~P_INEXEC;
922 		PROC_UNLOCK(p);
923 
924 		SDT_PROBE1(proc, , , exec__failure, error);
925 	}
926 
927 	if (imgp->newcred != NULL && oldcred != NULL)
928 		crfree(imgp->newcred);
929 
930 #ifdef MAC
931 	mac_execve_exit(imgp);
932 	mac_execve_interpreter_exit(interpvplabel);
933 #endif
934 	exec_free_args(args);
935 
936 	/*
937 	 * Handle deferred decrement of ref counts.
938 	 */
939 	if (oldtextvp != NULL)
940 		vrele(oldtextvp);
941 #ifdef KTRACE
942 	if (tracevp != NULL)
943 		vrele(tracevp);
944 	if (tracecred != NULL)
945 		crfree(tracecred);
946 #endif
947 	pargs_drop(oldargs);
948 	pargs_drop(newargs);
949 	if (oldsigacts != NULL)
950 		sigacts_free(oldsigacts);
951 	if (euip != NULL)
952 		uifree(euip);
953 
954 	if (error && imgp->vmspace_destroyed) {
955 		/* sorry, no more process anymore. exit gracefully */
956 		exit1(td, 0, SIGABRT);
957 		/* NOT REACHED */
958 	}
959 
960 #ifdef KTRACE
961 	if (error == 0)
962 		ktrprocctor(p);
963 #endif
964 
965 	/*
966 	 * We don't want cpu_set_syscall_retval() to overwrite any of
967 	 * the register values put in place by exec_setregs().
968 	 * Implementations of cpu_set_syscall_retval() will leave
969 	 * registers unmodified when returning EJUSTRETURN.
970 	 */
971 	return (error == 0 ? EJUSTRETURN : error);
972 }
973 
974 int
975 exec_map_first_page(struct image_params *imgp)
976 {
977 	vm_object_t object;
978 	vm_page_t m;
979 	int error;
980 
981 	if (imgp->firstpage != NULL)
982 		exec_unmap_first_page(imgp);
983 
984 	object = imgp->vp->v_object;
985 	if (object == NULL)
986 		return (EACCES);
987 #if VM_NRESERVLEVEL > 0
988 	if ((object->flags & OBJ_COLORED) == 0) {
989 		VM_OBJECT_WLOCK(object);
990 		vm_object_color(object, 0);
991 		VM_OBJECT_WUNLOCK(object);
992 	}
993 #endif
994 	error = vm_page_grab_valid_unlocked(&m, object, 0,
995 	    VM_ALLOC_COUNT(VM_INITIAL_PAGEIN) |
996             VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED);
997 
998 	if (error != VM_PAGER_OK)
999 		return (EIO);
1000 	imgp->firstpage = sf_buf_alloc(m, 0);
1001 	imgp->image_header = (char *)sf_buf_kva(imgp->firstpage);
1002 
1003 	return (0);
1004 }
1005 
1006 void
1007 exec_unmap_first_page(struct image_params *imgp)
1008 {
1009 	vm_page_t m;
1010 
1011 	if (imgp->firstpage != NULL) {
1012 		m = sf_buf_page(imgp->firstpage);
1013 		sf_buf_free(imgp->firstpage);
1014 		imgp->firstpage = NULL;
1015 		vm_page_unwire(m, PQ_ACTIVE);
1016 	}
1017 }
1018 
1019 /*
1020  * Destroy old address space, and allocate a new stack.
1021  *	The new stack is only sgrowsiz large because it is grown
1022  *	automatically on a page fault.
1023  */
1024 int
1025 exec_new_vmspace(struct image_params *imgp, struct sysentvec *sv)
1026 {
1027 	int error;
1028 	struct proc *p = imgp->proc;
1029 	struct vmspace *vmspace = p->p_vmspace;
1030 	struct thread *td = curthread;
1031 	vm_object_t obj;
1032 	struct rlimit rlim_stack;
1033 	vm_offset_t sv_minuser, stack_addr;
1034 	vm_map_t map;
1035 	u_long ssiz;
1036 
1037 	imgp->vmspace_destroyed = 1;
1038 	imgp->sysent = sv;
1039 
1040 	sigfastblock_clear(td);
1041 
1042 	/* May be called with Giant held */
1043 	EVENTHANDLER_DIRECT_INVOKE(process_exec, p, imgp);
1044 
1045 	/*
1046 	 * Blow away entire process VM, if address space not shared,
1047 	 * otherwise, create a new VM space so that other threads are
1048 	 * not disrupted
1049 	 */
1050 	map = &vmspace->vm_map;
1051 	if (map_at_zero)
1052 		sv_minuser = sv->sv_minuser;
1053 	else
1054 		sv_minuser = MAX(sv->sv_minuser, PAGE_SIZE);
1055 	if (vmspace->vm_refcnt == 1 && vm_map_min(map) == sv_minuser &&
1056 	    vm_map_max(map) == sv->sv_maxuser &&
1057 	    cpu_exec_vmspace_reuse(p, map)) {
1058 		shmexit(vmspace);
1059 		pmap_remove_pages(vmspace_pmap(vmspace));
1060 		vm_map_remove(map, vm_map_min(map), vm_map_max(map));
1061 		/*
1062 		 * An exec terminates mlockall(MCL_FUTURE), ASLR state
1063 		 * must be re-evaluated.
1064 		 */
1065 		vm_map_lock(map);
1066 		vm_map_modflags(map, 0, MAP_WIREFUTURE | MAP_ASLR |
1067 		    MAP_ASLR_IGNSTART);
1068 		vm_map_unlock(map);
1069 	} else {
1070 		error = vmspace_exec(p, sv_minuser, sv->sv_maxuser);
1071 		if (error)
1072 			return (error);
1073 		vmspace = p->p_vmspace;
1074 		map = &vmspace->vm_map;
1075 	}
1076 	map->flags |= imgp->map_flags;
1077 
1078 	/* Map a shared page */
1079 	obj = sv->sv_shared_page_obj;
1080 	if (obj != NULL) {
1081 		vm_object_reference(obj);
1082 		error = vm_map_fixed(map, obj, 0,
1083 		    sv->sv_shared_page_base, sv->sv_shared_page_len,
1084 		    VM_PROT_READ | VM_PROT_EXECUTE,
1085 		    VM_PROT_READ | VM_PROT_EXECUTE,
1086 		    MAP_INHERIT_SHARE | MAP_ACC_NO_CHARGE);
1087 		if (error != KERN_SUCCESS) {
1088 			vm_object_deallocate(obj);
1089 			return (vm_mmap_to_errno(error));
1090 		}
1091 	}
1092 
1093 	/* Allocate a new stack */
1094 	if (imgp->stack_sz != 0) {
1095 		ssiz = trunc_page(imgp->stack_sz);
1096 		PROC_LOCK(p);
1097 		lim_rlimit_proc(p, RLIMIT_STACK, &rlim_stack);
1098 		PROC_UNLOCK(p);
1099 		if (ssiz > rlim_stack.rlim_max)
1100 			ssiz = rlim_stack.rlim_max;
1101 		if (ssiz > rlim_stack.rlim_cur) {
1102 			rlim_stack.rlim_cur = ssiz;
1103 			kern_setrlimit(curthread, RLIMIT_STACK, &rlim_stack);
1104 		}
1105 	} else if (sv->sv_maxssiz != NULL) {
1106 		ssiz = *sv->sv_maxssiz;
1107 	} else {
1108 		ssiz = maxssiz;
1109 	}
1110 	imgp->eff_stack_sz = lim_cur(curthread, RLIMIT_STACK);
1111 	if (ssiz < imgp->eff_stack_sz)
1112 		imgp->eff_stack_sz = ssiz;
1113 	stack_addr = sv->sv_usrstack - ssiz;
1114 	error = vm_map_stack(map, stack_addr, (vm_size_t)ssiz,
1115 	    obj != NULL && imgp->stack_prot != 0 ? imgp->stack_prot :
1116 	    sv->sv_stackprot, VM_PROT_ALL, MAP_STACK_GROWS_DOWN);
1117 	if (error != KERN_SUCCESS)
1118 		return (vm_mmap_to_errno(error));
1119 
1120 	/*
1121 	 * vm_ssize and vm_maxsaddr are somewhat antiquated concepts, but they
1122 	 * are still used to enforce the stack rlimit on the process stack.
1123 	 */
1124 	vmspace->vm_ssize = sgrowsiz >> PAGE_SHIFT;
1125 	vmspace->vm_maxsaddr = (char *)stack_addr;
1126 
1127 	return (0);
1128 }
1129 
1130 /*
1131  * Copy out argument and environment strings from the old process address
1132  * space into the temporary string buffer.
1133  */
1134 int
1135 exec_copyin_args(struct image_args *args, const char *fname,
1136     enum uio_seg segflg, char **argv, char **envv)
1137 {
1138 	u_long arg, env;
1139 	int error;
1140 
1141 	bzero(args, sizeof(*args));
1142 	if (argv == NULL)
1143 		return (EFAULT);
1144 
1145 	/*
1146 	 * Allocate demand-paged memory for the file name, argument, and
1147 	 * environment strings.
1148 	 */
1149 	error = exec_alloc_args(args);
1150 	if (error != 0)
1151 		return (error);
1152 
1153 	/*
1154 	 * Copy the file name.
1155 	 */
1156 	error = exec_args_add_fname(args, fname, segflg);
1157 	if (error != 0)
1158 		goto err_exit;
1159 
1160 	/*
1161 	 * extract arguments first
1162 	 */
1163 	for (;;) {
1164 		error = fueword(argv++, &arg);
1165 		if (error == -1) {
1166 			error = EFAULT;
1167 			goto err_exit;
1168 		}
1169 		if (arg == 0)
1170 			break;
1171 		error = exec_args_add_arg(args, (char *)(uintptr_t)arg,
1172 		    UIO_USERSPACE);
1173 		if (error != 0)
1174 			goto err_exit;
1175 	}
1176 
1177 	/*
1178 	 * extract environment strings
1179 	 */
1180 	if (envv) {
1181 		for (;;) {
1182 			error = fueword(envv++, &env);
1183 			if (error == -1) {
1184 				error = EFAULT;
1185 				goto err_exit;
1186 			}
1187 			if (env == 0)
1188 				break;
1189 			error = exec_args_add_env(args,
1190 			    (char *)(uintptr_t)env, UIO_USERSPACE);
1191 			if (error != 0)
1192 				goto err_exit;
1193 		}
1194 	}
1195 
1196 	return (0);
1197 
1198 err_exit:
1199 	exec_free_args(args);
1200 	return (error);
1201 }
1202 
1203 int
1204 exec_copyin_data_fds(struct thread *td, struct image_args *args,
1205     const void *data, size_t datalen, const int *fds, size_t fdslen)
1206 {
1207 	struct filedesc *ofdp;
1208 	const char *p;
1209 	int *kfds;
1210 	int error;
1211 
1212 	memset(args, '\0', sizeof(*args));
1213 	ofdp = td->td_proc->p_fd;
1214 	if (datalen >= ARG_MAX || fdslen > ofdp->fd_lastfile + 1)
1215 		return (E2BIG);
1216 	error = exec_alloc_args(args);
1217 	if (error != 0)
1218 		return (error);
1219 
1220 	args->begin_argv = args->buf;
1221 	args->stringspace = ARG_MAX;
1222 
1223 	if (datalen > 0) {
1224 		/*
1225 		 * Argument buffer has been provided. Copy it into the
1226 		 * kernel as a single string and add a terminating null
1227 		 * byte.
1228 		 */
1229 		error = copyin(data, args->begin_argv, datalen);
1230 		if (error != 0)
1231 			goto err_exit;
1232 		args->begin_argv[datalen] = '\0';
1233 		args->endp = args->begin_argv + datalen + 1;
1234 		args->stringspace -= datalen + 1;
1235 
1236 		/*
1237 		 * Traditional argument counting. Count the number of
1238 		 * null bytes.
1239 		 */
1240 		for (p = args->begin_argv; p < args->endp; ++p)
1241 			if (*p == '\0')
1242 				++args->argc;
1243 	} else {
1244 		/* No argument buffer provided. */
1245 		args->endp = args->begin_argv;
1246 	}
1247 
1248 	/* Create new file descriptor table. */
1249 	kfds = malloc(fdslen * sizeof(int), M_TEMP, M_WAITOK);
1250 	error = copyin(fds, kfds, fdslen * sizeof(int));
1251 	if (error != 0) {
1252 		free(kfds, M_TEMP);
1253 		goto err_exit;
1254 	}
1255 	error = fdcopy_remapped(ofdp, kfds, fdslen, &args->fdp);
1256 	free(kfds, M_TEMP);
1257 	if (error != 0)
1258 		goto err_exit;
1259 
1260 	return (0);
1261 err_exit:
1262 	exec_free_args(args);
1263 	return (error);
1264 }
1265 
1266 struct exec_args_kva {
1267 	vm_offset_t addr;
1268 	u_int gen;
1269 	SLIST_ENTRY(exec_args_kva) next;
1270 };
1271 
1272 DPCPU_DEFINE_STATIC(struct exec_args_kva *, exec_args_kva);
1273 
1274 static SLIST_HEAD(, exec_args_kva) exec_args_kva_freelist;
1275 static struct mtx exec_args_kva_mtx;
1276 static u_int exec_args_gen;
1277 
1278 static void
1279 exec_prealloc_args_kva(void *arg __unused)
1280 {
1281 	struct exec_args_kva *argkva;
1282 	u_int i;
1283 
1284 	SLIST_INIT(&exec_args_kva_freelist);
1285 	mtx_init(&exec_args_kva_mtx, "exec args kva", NULL, MTX_DEF);
1286 	for (i = 0; i < exec_map_entries; i++) {
1287 		argkva = malloc(sizeof(*argkva), M_PARGS, M_WAITOK);
1288 		argkva->addr = kmap_alloc_wait(exec_map, exec_map_entry_size);
1289 		argkva->gen = exec_args_gen;
1290 		SLIST_INSERT_HEAD(&exec_args_kva_freelist, argkva, next);
1291 	}
1292 }
1293 SYSINIT(exec_args_kva, SI_SUB_EXEC, SI_ORDER_ANY, exec_prealloc_args_kva, NULL);
1294 
1295 static vm_offset_t
1296 exec_alloc_args_kva(void **cookie)
1297 {
1298 	struct exec_args_kva *argkva;
1299 
1300 	argkva = (void *)atomic_readandclear_ptr(
1301 	    (uintptr_t *)DPCPU_PTR(exec_args_kva));
1302 	if (argkva == NULL) {
1303 		mtx_lock(&exec_args_kva_mtx);
1304 		while ((argkva = SLIST_FIRST(&exec_args_kva_freelist)) == NULL)
1305 			(void)mtx_sleep(&exec_args_kva_freelist,
1306 			    &exec_args_kva_mtx, 0, "execkva", 0);
1307 		SLIST_REMOVE_HEAD(&exec_args_kva_freelist, next);
1308 		mtx_unlock(&exec_args_kva_mtx);
1309 	}
1310 	*(struct exec_args_kva **)cookie = argkva;
1311 	return (argkva->addr);
1312 }
1313 
1314 static void
1315 exec_release_args_kva(struct exec_args_kva *argkva, u_int gen)
1316 {
1317 	vm_offset_t base;
1318 
1319 	base = argkva->addr;
1320 	if (argkva->gen != gen) {
1321 		(void)vm_map_madvise(exec_map, base, base + exec_map_entry_size,
1322 		    MADV_FREE);
1323 		argkva->gen = gen;
1324 	}
1325 	if (!atomic_cmpset_ptr((uintptr_t *)DPCPU_PTR(exec_args_kva),
1326 	    (uintptr_t)NULL, (uintptr_t)argkva)) {
1327 		mtx_lock(&exec_args_kva_mtx);
1328 		SLIST_INSERT_HEAD(&exec_args_kva_freelist, argkva, next);
1329 		wakeup_one(&exec_args_kva_freelist);
1330 		mtx_unlock(&exec_args_kva_mtx);
1331 	}
1332 }
1333 
1334 static void
1335 exec_free_args_kva(void *cookie)
1336 {
1337 
1338 	exec_release_args_kva(cookie, exec_args_gen);
1339 }
1340 
1341 static void
1342 exec_args_kva_lowmem(void *arg __unused)
1343 {
1344 	SLIST_HEAD(, exec_args_kva) head;
1345 	struct exec_args_kva *argkva;
1346 	u_int gen;
1347 	int i;
1348 
1349 	gen = atomic_fetchadd_int(&exec_args_gen, 1) + 1;
1350 
1351 	/*
1352 	 * Force an madvise of each KVA range. Any currently allocated ranges
1353 	 * will have MADV_FREE applied once they are freed.
1354 	 */
1355 	SLIST_INIT(&head);
1356 	mtx_lock(&exec_args_kva_mtx);
1357 	SLIST_SWAP(&head, &exec_args_kva_freelist, exec_args_kva);
1358 	mtx_unlock(&exec_args_kva_mtx);
1359 	while ((argkva = SLIST_FIRST(&head)) != NULL) {
1360 		SLIST_REMOVE_HEAD(&head, next);
1361 		exec_release_args_kva(argkva, gen);
1362 	}
1363 
1364 	CPU_FOREACH(i) {
1365 		argkva = (void *)atomic_readandclear_ptr(
1366 		    (uintptr_t *)DPCPU_ID_PTR(i, exec_args_kva));
1367 		if (argkva != NULL)
1368 			exec_release_args_kva(argkva, gen);
1369 	}
1370 }
1371 EVENTHANDLER_DEFINE(vm_lowmem, exec_args_kva_lowmem, NULL,
1372     EVENTHANDLER_PRI_ANY);
1373 
1374 /*
1375  * Allocate temporary demand-paged, zero-filled memory for the file name,
1376  * argument, and environment strings.
1377  */
1378 int
1379 exec_alloc_args(struct image_args *args)
1380 {
1381 
1382 	args->buf = (char *)exec_alloc_args_kva(&args->bufkva);
1383 	return (0);
1384 }
1385 
1386 void
1387 exec_free_args(struct image_args *args)
1388 {
1389 
1390 	if (args->buf != NULL) {
1391 		exec_free_args_kva(args->bufkva);
1392 		args->buf = NULL;
1393 	}
1394 	if (args->fname_buf != NULL) {
1395 		free(args->fname_buf, M_TEMP);
1396 		args->fname_buf = NULL;
1397 	}
1398 	if (args->fdp != NULL)
1399 		fdescfree_remapped(args->fdp);
1400 }
1401 
1402 /*
1403  * A set to functions to fill struct image args.
1404  *
1405  * NOTE: exec_args_add_fname() must be called (possibly with a NULL
1406  * fname) before the other functions.  All exec_args_add_arg() calls must
1407  * be made before any exec_args_add_env() calls.  exec_args_adjust_args()
1408  * may be called any time after exec_args_add_fname().
1409  *
1410  * exec_args_add_fname() - install path to be executed
1411  * exec_args_add_arg() - append an argument string
1412  * exec_args_add_env() - append an env string
1413  * exec_args_adjust_args() - adjust location of the argument list to
1414  *                           allow new arguments to be prepended
1415  */
1416 int
1417 exec_args_add_fname(struct image_args *args, const char *fname,
1418     enum uio_seg segflg)
1419 {
1420 	int error;
1421 	size_t length;
1422 
1423 	KASSERT(args->fname == NULL, ("fname already appended"));
1424 	KASSERT(args->endp == NULL, ("already appending to args"));
1425 
1426 	if (fname != NULL) {
1427 		args->fname = args->buf;
1428 		error = segflg == UIO_SYSSPACE ?
1429 		    copystr(fname, args->fname, PATH_MAX, &length) :
1430 		    copyinstr(fname, args->fname, PATH_MAX, &length);
1431 		if (error != 0)
1432 			return (error == ENAMETOOLONG ? E2BIG : error);
1433 	} else
1434 		length = 0;
1435 
1436 	/* Set up for _arg_*()/_env_*() */
1437 	args->endp = args->buf + length;
1438 	/* begin_argv must be set and kept updated */
1439 	args->begin_argv = args->endp;
1440 	KASSERT(exec_map_entry_size - length >= ARG_MAX,
1441 	    ("too little space remaining for arguments %zu < %zu",
1442 	    exec_map_entry_size - length, (size_t)ARG_MAX));
1443 	args->stringspace = ARG_MAX;
1444 
1445 	return (0);
1446 }
1447 
1448 static int
1449 exec_args_add_str(struct image_args *args, const char *str,
1450     enum uio_seg segflg, int *countp)
1451 {
1452 	int error;
1453 	size_t length;
1454 
1455 	KASSERT(args->endp != NULL, ("endp not initialized"));
1456 	KASSERT(args->begin_argv != NULL, ("begin_argp not initialized"));
1457 
1458 	error = (segflg == UIO_SYSSPACE) ?
1459 	    copystr(str, args->endp, args->stringspace, &length) :
1460 	    copyinstr(str, args->endp, args->stringspace, &length);
1461 	if (error != 0)
1462 		return (error == ENAMETOOLONG ? E2BIG : error);
1463 	args->stringspace -= length;
1464 	args->endp += length;
1465 	(*countp)++;
1466 
1467 	return (0);
1468 }
1469 
1470 int
1471 exec_args_add_arg(struct image_args *args, const char *argp,
1472     enum uio_seg segflg)
1473 {
1474 
1475 	KASSERT(args->envc == 0, ("appending args after env"));
1476 
1477 	return (exec_args_add_str(args, argp, segflg, &args->argc));
1478 }
1479 
1480 int
1481 exec_args_add_env(struct image_args *args, const char *envp,
1482     enum uio_seg segflg)
1483 {
1484 
1485 	if (args->envc == 0)
1486 		args->begin_envv = args->endp;
1487 
1488 	return (exec_args_add_str(args, envp, segflg, &args->envc));
1489 }
1490 
1491 int
1492 exec_args_adjust_args(struct image_args *args, size_t consume, ssize_t extend)
1493 {
1494 	ssize_t offset;
1495 
1496 	KASSERT(args->endp != NULL, ("endp not initialized"));
1497 	KASSERT(args->begin_argv != NULL, ("begin_argp not initialized"));
1498 
1499 	offset = extend - consume;
1500 	if (args->stringspace < offset)
1501 		return (E2BIG);
1502 	memmove(args->begin_argv + extend, args->begin_argv + consume,
1503 	    args->endp - args->begin_argv + consume);
1504 	if (args->envc > 0)
1505 		args->begin_envv += offset;
1506 	args->endp += offset;
1507 	args->stringspace -= offset;
1508 	return (0);
1509 }
1510 
1511 char *
1512 exec_args_get_begin_envv(struct image_args *args)
1513 {
1514 
1515 	KASSERT(args->endp != NULL, ("endp not initialized"));
1516 
1517 	if (args->envc > 0)
1518 		return (args->begin_envv);
1519 	return (args->endp);
1520 }
1521 
1522 /*
1523  * Copy strings out to the new process address space, constructing new arg
1524  * and env vector tables. Return a pointer to the base so that it can be used
1525  * as the initial stack pointer.
1526  */
1527 int
1528 exec_copyout_strings(struct image_params *imgp, uintptr_t *stack_base)
1529 {
1530 	int argc, envc;
1531 	char **vectp;
1532 	char *stringp;
1533 	uintptr_t destp, ustringp;
1534 	struct ps_strings *arginfo;
1535 	struct proc *p;
1536 	size_t execpath_len;
1537 	int error, szsigcode, szps;
1538 	char canary[sizeof(long) * 8];
1539 
1540 	szps = sizeof(pagesizes[0]) * MAXPAGESIZES;
1541 	/*
1542 	 * Calculate string base and vector table pointers.
1543 	 * Also deal with signal trampoline code for this exec type.
1544 	 */
1545 	if (imgp->execpath != NULL && imgp->auxargs != NULL)
1546 		execpath_len = strlen(imgp->execpath) + 1;
1547 	else
1548 		execpath_len = 0;
1549 	p = imgp->proc;
1550 	szsigcode = 0;
1551 	arginfo = (struct ps_strings *)p->p_sysent->sv_psstrings;
1552 	if (p->p_sysent->sv_sigcode_base == 0) {
1553 		if (p->p_sysent->sv_szsigcode != NULL)
1554 			szsigcode = *(p->p_sysent->sv_szsigcode);
1555 	}
1556 	destp =	(uintptr_t)arginfo;
1557 
1558 	/*
1559 	 * install sigcode
1560 	 */
1561 	if (szsigcode != 0) {
1562 		destp -= szsigcode;
1563 		destp = rounddown2(destp, sizeof(void *));
1564 		error = copyout(p->p_sysent->sv_sigcode, (void *)destp,
1565 		    szsigcode);
1566 		if (error != 0)
1567 			return (error);
1568 	}
1569 
1570 	/*
1571 	 * Copy the image path for the rtld.
1572 	 */
1573 	if (execpath_len != 0) {
1574 		destp -= execpath_len;
1575 		destp = rounddown2(destp, sizeof(void *));
1576 		imgp->execpathp = destp;
1577 		error = copyout(imgp->execpath, (void *)destp, execpath_len);
1578 		if (error != 0)
1579 			return (error);
1580 	}
1581 
1582 	/*
1583 	 * Prepare the canary for SSP.
1584 	 */
1585 	arc4rand(canary, sizeof(canary), 0);
1586 	destp -= sizeof(canary);
1587 	imgp->canary = destp;
1588 	error = copyout(canary, (void *)destp, sizeof(canary));
1589 	if (error != 0)
1590 		return (error);
1591 	imgp->canarylen = sizeof(canary);
1592 
1593 	/*
1594 	 * Prepare the pagesizes array.
1595 	 */
1596 	destp -= szps;
1597 	destp = rounddown2(destp, sizeof(void *));
1598 	imgp->pagesizes = destp;
1599 	error = copyout(pagesizes, (void *)destp, szps);
1600 	if (error != 0)
1601 		return (error);
1602 	imgp->pagesizeslen = szps;
1603 
1604 	/*
1605 	 * Allocate room for the argument and environment strings.
1606 	 */
1607 	destp -= ARG_MAX - imgp->args->stringspace;
1608 	destp = rounddown2(destp, sizeof(void *));
1609 	ustringp = destp;
1610 
1611 	if (imgp->sysent->sv_stackgap != NULL)
1612 		imgp->sysent->sv_stackgap(imgp, &destp);
1613 
1614 	if (imgp->auxargs) {
1615 		/*
1616 		 * Allocate room on the stack for the ELF auxargs
1617 		 * array.  It has up to AT_COUNT entries.
1618 		 */
1619 		destp -= AT_COUNT * sizeof(Elf_Auxinfo);
1620 		destp = rounddown2(destp, sizeof(void *));
1621 	}
1622 
1623 	vectp = (char **)destp;
1624 
1625 	/*
1626 	 * Allocate room for the argv[] and env vectors including the
1627 	 * terminating NULL pointers.
1628 	 */
1629 	vectp -= imgp->args->argc + 1 + imgp->args->envc + 1;
1630 
1631 	/*
1632 	 * vectp also becomes our initial stack base
1633 	 */
1634 	*stack_base = (uintptr_t)vectp;
1635 
1636 	stringp = imgp->args->begin_argv;
1637 	argc = imgp->args->argc;
1638 	envc = imgp->args->envc;
1639 
1640 	/*
1641 	 * Copy out strings - arguments and environment.
1642 	 */
1643 	error = copyout(stringp, (void *)ustringp,
1644 	    ARG_MAX - imgp->args->stringspace);
1645 	if (error != 0)
1646 		return (error);
1647 
1648 	/*
1649 	 * Fill in "ps_strings" struct for ps, w, etc.
1650 	 */
1651 	if (suword(&arginfo->ps_argvstr, (long)(intptr_t)vectp) != 0 ||
1652 	    suword32(&arginfo->ps_nargvstr, argc) != 0)
1653 		return (EFAULT);
1654 
1655 	/*
1656 	 * Fill in argument portion of vector table.
1657 	 */
1658 	for (; argc > 0; --argc) {
1659 		if (suword(vectp++, ustringp) != 0)
1660 			return (EFAULT);
1661 		while (*stringp++ != 0)
1662 			ustringp++;
1663 		ustringp++;
1664 	}
1665 
1666 	/* a null vector table pointer separates the argp's from the envp's */
1667 	if (suword(vectp++, 0) != 0)
1668 		return (EFAULT);
1669 
1670 	if (suword(&arginfo->ps_envstr, (long)(intptr_t)vectp) != 0 ||
1671 	    suword32(&arginfo->ps_nenvstr, envc) != 0)
1672 		return (EFAULT);
1673 
1674 	/*
1675 	 * Fill in environment portion of vector table.
1676 	 */
1677 	for (; envc > 0; --envc) {
1678 		if (suword(vectp++, ustringp) != 0)
1679 			return (EFAULT);
1680 		while (*stringp++ != 0)
1681 			ustringp++;
1682 		ustringp++;
1683 	}
1684 
1685 	/* end of vector table is a null pointer */
1686 	if (suword(vectp, 0) != 0)
1687 		return (EFAULT);
1688 
1689 	if (imgp->auxargs) {
1690 		vectp++;
1691 		error = imgp->sysent->sv_copyout_auxargs(imgp,
1692 		    (uintptr_t)vectp);
1693 		if (error != 0)
1694 			return (error);
1695 	}
1696 
1697 	return (0);
1698 }
1699 
1700 /*
1701  * Check permissions of file to execute.
1702  *	Called with imgp->vp locked.
1703  *	Return 0 for success or error code on failure.
1704  */
1705 int
1706 exec_check_permissions(struct image_params *imgp)
1707 {
1708 	struct vnode *vp = imgp->vp;
1709 	struct vattr *attr = imgp->attr;
1710 	struct thread *td;
1711 	int error;
1712 
1713 	td = curthread;
1714 
1715 	/* Get file attributes */
1716 	error = VOP_GETATTR(vp, attr, td->td_ucred);
1717 	if (error)
1718 		return (error);
1719 
1720 #ifdef MAC
1721 	error = mac_vnode_check_exec(td->td_ucred, imgp->vp, imgp);
1722 	if (error)
1723 		return (error);
1724 #endif
1725 
1726 	/*
1727 	 * 1) Check if file execution is disabled for the filesystem that
1728 	 *    this file resides on.
1729 	 * 2) Ensure that at least one execute bit is on. Otherwise, a
1730 	 *    privileged user will always succeed, and we don't want this
1731 	 *    to happen unless the file really is executable.
1732 	 * 3) Ensure that the file is a regular file.
1733 	 */
1734 	if ((vp->v_mount->mnt_flag & MNT_NOEXEC) ||
1735 	    (attr->va_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) == 0 ||
1736 	    (attr->va_type != VREG))
1737 		return (EACCES);
1738 
1739 	/*
1740 	 * Zero length files can't be exec'd
1741 	 */
1742 	if (attr->va_size == 0)
1743 		return (ENOEXEC);
1744 
1745 	/*
1746 	 *  Check for execute permission to file based on current credentials.
1747 	 */
1748 	error = VOP_ACCESS(vp, VEXEC, td->td_ucred, td);
1749 	if (error)
1750 		return (error);
1751 
1752 	/*
1753 	 * Check number of open-for-writes on the file and deny execution
1754 	 * if there are any.
1755 	 *
1756 	 * Add a text reference now so no one can write to the
1757 	 * executable while we're activating it.
1758 	 *
1759 	 * Remember if this was set before and unset it in case this is not
1760 	 * actually an executable image.
1761 	 */
1762 	error = VOP_SET_TEXT(vp);
1763 	if (error != 0)
1764 		return (error);
1765 	imgp->textset = true;
1766 
1767 	/*
1768 	 * Call filesystem specific open routine (which does nothing in the
1769 	 * general case).
1770 	 */
1771 	error = VOP_OPEN(vp, FREAD, td->td_ucred, td, NULL);
1772 	if (error == 0)
1773 		imgp->opened = 1;
1774 	return (error);
1775 }
1776 
1777 /*
1778  * Exec handler registration
1779  */
1780 int
1781 exec_register(const struct execsw *execsw_arg)
1782 {
1783 	const struct execsw **es, **xs, **newexecsw;
1784 	u_int count = 2;	/* New slot and trailing NULL */
1785 
1786 	if (execsw)
1787 		for (es = execsw; *es; es++)
1788 			count++;
1789 	newexecsw = malloc(count * sizeof(*es), M_TEMP, M_WAITOK);
1790 	xs = newexecsw;
1791 	if (execsw)
1792 		for (es = execsw; *es; es++)
1793 			*xs++ = *es;
1794 	*xs++ = execsw_arg;
1795 	*xs = NULL;
1796 	if (execsw)
1797 		free(execsw, M_TEMP);
1798 	execsw = newexecsw;
1799 	return (0);
1800 }
1801 
1802 int
1803 exec_unregister(const struct execsw *execsw_arg)
1804 {
1805 	const struct execsw **es, **xs, **newexecsw;
1806 	int count = 1;
1807 
1808 	if (execsw == NULL)
1809 		panic("unregister with no handlers left?\n");
1810 
1811 	for (es = execsw; *es; es++) {
1812 		if (*es == execsw_arg)
1813 			break;
1814 	}
1815 	if (*es == NULL)
1816 		return (ENOENT);
1817 	for (es = execsw; *es; es++)
1818 		if (*es != execsw_arg)
1819 			count++;
1820 	newexecsw = malloc(count * sizeof(*es), M_TEMP, M_WAITOK);
1821 	xs = newexecsw;
1822 	for (es = execsw; *es; es++)
1823 		if (*es != execsw_arg)
1824 			*xs++ = *es;
1825 	*xs = NULL;
1826 	if (execsw)
1827 		free(execsw, M_TEMP);
1828 	execsw = newexecsw;
1829 	return (0);
1830 }
1831