xref: /freebsd/sys/kern/kern_fork.c (revision 89f4fca26524640a836b48761f0f131097bb43fe)
1 /*
2  * Copyright (c) 1982, 1986, 1989, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * (c) UNIX System Laboratories, Inc.
5  * All or some portions of this file are derived from material licensed
6  * to the University of California by American Telephone and Telegraph
7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8  * the permission of UNIX System Laboratories, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. All advertising materials mentioning features or use of this software
19  *    must display the following acknowledgement:
20  *	This product includes software developed by the University of
21  *	California, Berkeley and its contributors.
22  * 4. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  *	@(#)kern_fork.c	8.6 (Berkeley) 4/8/94
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include "opt_ktrace.h"
45 #include "opt_mac.h"
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/sysproto.h>
50 #include <sys/eventhandler.h>
51 #include <sys/filedesc.h>
52 #include <sys/kernel.h>
53 #include <sys/sysctl.h>
54 #include <sys/lock.h>
55 #include <sys/malloc.h>
56 #include <sys/mutex.h>
57 #include <sys/proc.h>
58 #include <sys/pioctl.h>
59 #include <sys/resourcevar.h>
60 #include <sys/sched.h>
61 #include <sys/syscall.h>
62 #include <sys/vnode.h>
63 #include <sys/acct.h>
64 #include <sys/mac.h>
65 #include <sys/ktr.h>
66 #include <sys/ktrace.h>
67 #include <sys/kthread.h>
68 #include <sys/unistd.h>
69 #include <sys/jail.h>
70 #include <sys/sx.h>
71 
72 #include <vm/vm.h>
73 #include <vm/pmap.h>
74 #include <vm/vm_map.h>
75 #include <vm/vm_extern.h>
76 #include <vm/uma.h>
77 
78 #include <sys/vmmeter.h>
79 #include <sys/user.h>
80 #include <machine/critical.h>
81 
82 #ifndef _SYS_SYSPROTO_H_
83 struct fork_args {
84 	int     dummy;
85 };
86 #endif
87 
88 static int forksleep; /* Place for fork1() to sleep on. */
89 
90 /*
91  * MPSAFE
92  */
93 /* ARGSUSED */
94 int
95 fork(td, uap)
96 	struct thread *td;
97 	struct fork_args *uap;
98 {
99 	int error;
100 	struct proc *p2;
101 
102 	error = fork1(td, RFFDG | RFPROC, 0, &p2);
103 	if (error == 0) {
104 		td->td_retval[0] = p2->p_pid;
105 		td->td_retval[1] = 0;
106 	}
107 	return error;
108 }
109 
110 /*
111  * MPSAFE
112  */
113 /* ARGSUSED */
114 int
115 vfork(td, uap)
116 	struct thread *td;
117 	struct vfork_args *uap;
118 {
119 	int error;
120 	struct proc *p2;
121 
122 	error = fork1(td, RFFDG | RFPROC | RFPPWAIT | RFMEM, 0, &p2);
123 	if (error == 0) {
124 		td->td_retval[0] = p2->p_pid;
125 		td->td_retval[1] = 0;
126 	}
127 	return error;
128 }
129 
130 /*
131  * MPSAFE
132  */
133 int
134 rfork(td, uap)
135 	struct thread *td;
136 	struct rfork_args *uap;
137 {
138 	int error;
139 	struct proc *p2;
140 
141 	/* Don't allow kernel only flags. */
142 	if ((uap->flags & RFKERNELONLY) != 0)
143 		return (EINVAL);
144 	error = fork1(td, uap->flags, 0, &p2);
145 	if (error == 0) {
146 		td->td_retval[0] = p2 ? p2->p_pid : 0;
147 		td->td_retval[1] = 0;
148 	}
149 	return error;
150 }
151 
152 
153 int	nprocs = 1;				/* process 0 */
154 int	lastpid = 0;
155 SYSCTL_INT(_kern, OID_AUTO, lastpid, CTLFLAG_RD, &lastpid, 0,
156     "Last used PID");
157 
158 /*
159  * Random component to lastpid generation.  We mix in a random factor to make
160  * it a little harder to predict.  We sanity check the modulus value to avoid
161  * doing it in critical paths.  Don't let it be too small or we pointlessly
162  * waste randomness entropy, and don't let it be impossibly large.  Using a
163  * modulus that is too big causes a LOT more process table scans and slows
164  * down fork processing as the pidchecked caching is defeated.
165  */
166 static int randompid = 0;
167 
168 static int
169 sysctl_kern_randompid(SYSCTL_HANDLER_ARGS)
170 {
171 	int error, pid;
172 
173 	sysctl_wire_old_buffer(req, sizeof(int));
174 	sx_xlock(&allproc_lock);
175 	pid = randompid;
176 	error = sysctl_handle_int(oidp, &pid, 0, req);
177 	if (error == 0 && req->newptr != NULL) {
178 		if (pid < 0 || pid > PID_MAX - 100)	/* out of range */
179 			pid = PID_MAX - 100;
180 		else if (pid < 2)			/* NOP */
181 			pid = 0;
182 		else if (pid < 100)			/* Make it reasonable */
183 			pid = 100;
184 		randompid = pid;
185 	}
186 	sx_xunlock(&allproc_lock);
187 	return (error);
188 }
189 
190 SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW,
191     0, 0, sysctl_kern_randompid, "I", "Random PID modulus");
192 
193 int
194 fork1(td, flags, pages, procp)
195 	struct thread *td;			/* parent proc */
196 	int flags;
197 	int pages;
198 	struct proc **procp;			/* child proc */
199 {
200 	struct proc *p2, *pptr;
201 	uid_t uid;
202 	struct proc *newproc;
203 	int trypid;
204 	int ok;
205 	static int pidchecked = 0;
206 	struct filedesc *fd;
207 	struct filedesc_to_leader *fdtol;
208 	struct proc *p1 = td->td_proc;
209 	struct thread *td2;
210 	struct kse *ke2;
211 	struct ksegrp *kg2;
212 	struct sigacts *newsigacts;
213 	int error;
214 
215 	/* Can't copy and clear */
216 	if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG))
217 		return (EINVAL);
218 
219 	mtx_lock(&Giant);
220 	/*
221 	 * Here we don't create a new process, but we divorce
222 	 * certain parts of a process from itself.
223 	 */
224 	if ((flags & RFPROC) == 0) {
225 		vm_forkproc(td, NULL, NULL, flags);
226 
227 		/*
228 		 * Close all file descriptors.
229 		 */
230 		if (flags & RFCFDG) {
231 			struct filedesc *fdtmp;
232 			fdtmp = fdinit(td->td_proc->p_fd);
233 			fdfree(td);
234 			p1->p_fd = fdtmp;
235 		}
236 
237 		/*
238 		 * Unshare file descriptors (from parent.)
239 		 */
240 		if (flags & RFFDG) {
241 			FILEDESC_LOCK(p1->p_fd);
242 			if (p1->p_fd->fd_refcnt > 1) {
243 				struct filedesc *newfd;
244 
245 				newfd = fdcopy(td->td_proc->p_fd);
246 				FILEDESC_UNLOCK(p1->p_fd);
247 				fdfree(td);
248 				p1->p_fd = newfd;
249 			} else
250 				FILEDESC_UNLOCK(p1->p_fd);
251 		}
252 		mtx_unlock(&Giant);
253 		*procp = NULL;
254 		return (0);
255 	}
256 
257 	/*
258 	 * Note 1:1 allows for forking with one thread coming out on the
259 	 * other side with the expectation that the process is about to
260 	 * exec.
261 	 */
262 	if (p1->p_flag & P_THREADED) {
263 		/*
264 		 * Idle the other threads for a second.
265 		 * Since the user space is copied, it must remain stable.
266 		 * In addition, all threads (from the user perspective)
267 		 * need to either be suspended or in the kernel,
268 		 * where they will try restart in the parent and will
269 		 * be aborted in the child.
270 		 */
271 		PROC_LOCK(p1);
272 		if (thread_single(SINGLE_NO_EXIT)) {
273 			/* Abort.. someone else is single threading before us */
274 			PROC_UNLOCK(p1);
275 			mtx_unlock(&Giant);
276 			return (ERESTART);
277 		}
278 		PROC_UNLOCK(p1);
279 		/*
280 		 * All other activity in this process
281 		 * is now suspended at the user boundary,
282 		 * (or other safe places if we think of any).
283 		 */
284 	}
285 
286 	/* Allocate new proc. */
287 	newproc = uma_zalloc(proc_zone, M_WAITOK);
288 #ifdef MAC
289 	mac_init_proc(newproc);
290 #endif
291 
292 	/*
293 	 * Although process entries are dynamically created, we still keep
294 	 * a global limit on the maximum number we will create.  Don't allow
295 	 * a nonprivileged user to use the last ten processes; don't let root
296 	 * exceed the limit. The variable nprocs is the current number of
297 	 * processes, maxproc is the limit.
298 	 */
299 	sx_xlock(&allproc_lock);
300 	uid = td->td_ucred->cr_ruid;
301 	if ((nprocs >= maxproc - 10 && uid != 0) || nprocs >= maxproc) {
302 		error = EAGAIN;
303 		goto fail;
304 	}
305 
306 	/*
307 	 * Increment the count of procs running with this uid. Don't allow
308 	 * a nonprivileged user to exceed their current limit.
309 	 */
310 	PROC_LOCK(p1);
311 	ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1,
312 		(uid != 0) ? p1->p_rlimit[RLIMIT_NPROC].rlim_cur : 0);
313 	PROC_UNLOCK(p1);
314 	if (!ok) {
315 		error = EAGAIN;
316 		goto fail;
317 	}
318 
319 	/*
320 	 * Increment the nprocs resource before blocking can occur.  There
321 	 * are hard-limits as to the number of processes that can run.
322 	 */
323 	nprocs++;
324 
325 	/*
326 	 * Find an unused process ID.  We remember a range of unused IDs
327 	 * ready to use (from lastpid+1 through pidchecked-1).
328 	 *
329 	 * If RFHIGHPID is set (used during system boot), do not allocate
330 	 * low-numbered pids.
331 	 */
332 	trypid = lastpid + 1;
333 	if (flags & RFHIGHPID) {
334 		if (trypid < 10) {
335 			trypid = 10;
336 		}
337 	} else {
338 		if (randompid)
339 			trypid += arc4random() % randompid;
340 	}
341 retry:
342 	/*
343 	 * If the process ID prototype has wrapped around,
344 	 * restart somewhat above 0, as the low-numbered procs
345 	 * tend to include daemons that don't exit.
346 	 */
347 	if (trypid >= PID_MAX) {
348 		trypid = trypid % PID_MAX;
349 		if (trypid < 100)
350 			trypid += 100;
351 		pidchecked = 0;
352 	}
353 	if (trypid >= pidchecked) {
354 		int doingzomb = 0;
355 
356 		pidchecked = PID_MAX;
357 		/*
358 		 * Scan the active and zombie procs to check whether this pid
359 		 * is in use.  Remember the lowest pid that's greater
360 		 * than trypid, so we can avoid checking for a while.
361 		 */
362 		p2 = LIST_FIRST(&allproc);
363 again:
364 		for (; p2 != NULL; p2 = LIST_NEXT(p2, p_list)) {
365 			PROC_LOCK(p2);
366 			while (p2->p_pid == trypid ||
367 			    p2->p_pgrp->pg_id == trypid ||
368 			    p2->p_session->s_sid == trypid) {
369 				trypid++;
370 				if (trypid >= pidchecked) {
371 					PROC_UNLOCK(p2);
372 					goto retry;
373 				}
374 			}
375 			if (p2->p_pid > trypid && pidchecked > p2->p_pid)
376 				pidchecked = p2->p_pid;
377 			if (p2->p_pgrp->pg_id > trypid &&
378 			    pidchecked > p2->p_pgrp->pg_id)
379 				pidchecked = p2->p_pgrp->pg_id;
380 			if (p2->p_session->s_sid > trypid &&
381 			    pidchecked > p2->p_session->s_sid)
382 				pidchecked = p2->p_session->s_sid;
383 			PROC_UNLOCK(p2);
384 		}
385 		if (!doingzomb) {
386 			doingzomb = 1;
387 			p2 = LIST_FIRST(&zombproc);
388 			goto again;
389 		}
390 	}
391 
392 	/*
393 	 * RFHIGHPID does not mess with the lastpid counter during boot.
394 	 */
395 	if (flags & RFHIGHPID)
396 		pidchecked = 0;
397 	else
398 		lastpid = trypid;
399 
400 	p2 = newproc;
401 	p2->p_state = PRS_NEW;		/* protect against others */
402 	p2->p_pid = trypid;
403 	LIST_INSERT_HEAD(&allproc, p2, p_list);
404 	LIST_INSERT_HEAD(PIDHASH(p2->p_pid), p2, p_hash);
405 	sx_xunlock(&allproc_lock);
406 
407 	/*
408 	 * Malloc things while we don't hold any locks.
409 	 */
410 	if (flags & RFSIGSHARE)
411 		newsigacts = NULL;
412 	else
413 		newsigacts = sigacts_alloc();
414 
415 	/*
416 	 * Copy filedesc.
417 	 */
418 	if (flags & RFCFDG) {
419 		fd = fdinit(td->td_proc->p_fd);
420 		fdtol = NULL;
421 	} else if (flags & RFFDG) {
422 		FILEDESC_LOCK(p1->p_fd);
423 		fd = fdcopy(td->td_proc->p_fd);
424 		FILEDESC_UNLOCK(p1->p_fd);
425 		fdtol = NULL;
426 	} else {
427 		fd = fdshare(p1->p_fd);
428 		if (p1->p_fdtol == NULL)
429 			p1->p_fdtol =
430 				filedesc_to_leader_alloc(NULL,
431 							 NULL,
432 							 p1->p_leader);
433 		if ((flags & RFTHREAD) != 0) {
434 			/*
435 			 * Shared file descriptor table and
436 			 * shared process leaders.
437 			 */
438 			fdtol = p1->p_fdtol;
439 			FILEDESC_LOCK(p1->p_fd);
440 			fdtol->fdl_refcount++;
441 			FILEDESC_UNLOCK(p1->p_fd);
442 		} else {
443 			/*
444 			 * Shared file descriptor table, and
445 			 * different process leaders
446 			 */
447 			fdtol = filedesc_to_leader_alloc(p1->p_fdtol,
448 							 p1->p_fd,
449 							 p2);
450 		}
451 	}
452 	/*
453 	 * Make a proc table entry for the new process.
454 	 * Start by zeroing the section of proc that is zero-initialized,
455 	 * then copy the section that is copied directly from the parent.
456 	 */
457 	td2 = FIRST_THREAD_IN_PROC(p2);
458 	kg2 = FIRST_KSEGRP_IN_PROC(p2);
459 	ke2 = FIRST_KSE_IN_KSEGRP(kg2);
460 
461 	/* Allocate and switch to an alternate kstack if specified */
462 	if (pages != 0)
463 		vm_thread_new_altkstack(td2, pages);
464 
465 	PROC_LOCK(p2);
466 	PROC_LOCK(p1);
467 
468 #define RANGEOF(type, start, end) (offsetof(type, end) - offsetof(type, start))
469 
470 	bzero(&p2->p_startzero,
471 	    (unsigned) RANGEOF(struct proc, p_startzero, p_endzero));
472 	bzero(&ke2->ke_startzero,
473 	    (unsigned) RANGEOF(struct kse, ke_startzero, ke_endzero));
474 	bzero(&td2->td_startzero,
475 	    (unsigned) RANGEOF(struct thread, td_startzero, td_endzero));
476 	bzero(&kg2->kg_startzero,
477 	    (unsigned) RANGEOF(struct ksegrp, kg_startzero, kg_endzero));
478 
479 	bcopy(&p1->p_startcopy, &p2->p_startcopy,
480 	    (unsigned) RANGEOF(struct proc, p_startcopy, p_endcopy));
481 	bcopy(&td->td_startcopy, &td2->td_startcopy,
482 	    (unsigned) RANGEOF(struct thread, td_startcopy, td_endcopy));
483 	bcopy(&td->td_ksegrp->kg_startcopy, &kg2->kg_startcopy,
484 	    (unsigned) RANGEOF(struct ksegrp, kg_startcopy, kg_endcopy));
485 #undef RANGEOF
486 
487 	/* Set up the thread as an active thread (as if runnable). */
488 	ke2->ke_state = KES_THREAD;
489 	ke2->ke_thread = td2;
490 	td2->td_kse = ke2;
491 
492 	/*
493 	 * Duplicate sub-structures as needed.
494 	 * Increase reference counts on shared objects.
495 	 * The p_stats substruct is set in vm_forkproc.
496 	 */
497 	p2->p_flag = 0;
498 	if (p1->p_flag & P_PROFIL)
499 		startprofclock(p2);
500 	mtx_lock_spin(&sched_lock);
501 	p2->p_sflag = PS_INMEM;
502 	/*
503 	 * Allow the scheduler to adjust the priority of the child and
504 	 * parent while we hold the sched_lock.
505 	 */
506 	sched_fork(p1, p2);
507 
508 	mtx_unlock_spin(&sched_lock);
509 	p2->p_ucred = crhold(td->td_ucred);
510 	td2->td_ucred = crhold(p2->p_ucred);	/* XXXKSE */
511 
512 	pargs_hold(p2->p_args);
513 
514 	if (flags & RFSIGSHARE) {
515 		p2->p_sigacts = sigacts_hold(p1->p_sigacts);
516 	} else {
517 		sigacts_copy(newsigacts, p1->p_sigacts);
518 		p2->p_sigacts = newsigacts;
519 	}
520 	if (flags & RFLINUXTHPN)
521 	        p2->p_sigparent = SIGUSR1;
522 	else
523 	        p2->p_sigparent = SIGCHLD;
524 
525 	/* Bump references to the text vnode (for procfs) */
526 	p2->p_textvp = p1->p_textvp;
527 	if (p2->p_textvp)
528 		VREF(p2->p_textvp);
529 	p2->p_fd = fd;
530 	p2->p_fdtol = fdtol;
531 	PROC_UNLOCK(p1);
532 	PROC_UNLOCK(p2);
533 
534 	/*
535 	 * p_limit is copy-on-write, bump refcnt,
536 	 */
537 	p2->p_limit = p1->p_limit;
538 	p2->p_limit->p_refcnt++;
539 
540 	/*
541 	 * Setup linkage for kernel based threading
542 	 */
543 	if((flags & RFTHREAD) != 0) {
544 		mtx_lock(&ppeers_lock);
545 		p2->p_peers = p1->p_peers;
546 		p1->p_peers = p2;
547 		p2->p_leader = p1->p_leader;
548 		mtx_unlock(&ppeers_lock);
549 		PROC_LOCK(p1->p_leader);
550 		if ((p1->p_leader->p_flag & P_WEXIT) != 0) {
551 			PROC_UNLOCK(p1->p_leader);
552 			/*
553 			 * The task leader is exiting, so process p1 is
554 			 * going to be killed shortly.  Since p1 obviously
555 			 * isn't dead yet, we know that the leader is either
556 			 * sending SIGKILL's to all the processes in this
557 			 * task or is sleeping waiting for all the peers to
558 			 * exit.  We let p1 complete the fork, but we need
559 			 * to go ahead and kill the new process p2 since
560 			 * the task leader may not get a chance to send
561 			 * SIGKILL to it.  We leave it on the list so that
562 			 * the task leader will wait for this new process
563 			 * to commit suicide.
564 			 */
565 			PROC_LOCK(p2);
566 			psignal(p2, SIGKILL);
567 			PROC_UNLOCK(p2);
568 		} else
569 			PROC_UNLOCK(p1->p_leader);
570 	} else {
571 		p2->p_peers = NULL;
572 		p2->p_leader = p2;
573 	}
574 
575 	sx_xlock(&proctree_lock);
576 	PGRP_LOCK(p1->p_pgrp);
577 	PROC_LOCK(p2);
578 	PROC_LOCK(p1);
579 
580 	/*
581 	 * Preserve some more flags in subprocess.  P_PROFIL has already
582 	 * been preserved.
583 	 */
584 	p2->p_flag |= p1->p_flag & (P_SUGID | P_ALTSTACK);
585 	SESS_LOCK(p1->p_session);
586 	if (p1->p_session->s_ttyvp != NULL && p1->p_flag & P_CONTROLT)
587 		p2->p_flag |= P_CONTROLT;
588 	SESS_UNLOCK(p1->p_session);
589 	if (flags & RFPPWAIT)
590 		p2->p_flag |= P_PPWAIT;
591 
592 	LIST_INSERT_AFTER(p1, p2, p_pglist);
593 	PGRP_UNLOCK(p1->p_pgrp);
594 	LIST_INIT(&p2->p_children);
595 
596 	callout_init(&p2->p_itcallout, 1);
597 
598 #ifdef KTRACE
599 	/*
600 	 * Copy traceflag and tracefile if enabled.
601 	 */
602 	mtx_lock(&ktrace_mtx);
603 	KASSERT(p2->p_tracevp == NULL, ("new process has a ktrace vnode"));
604 	if (p1->p_traceflag & KTRFAC_INHERIT) {
605 		p2->p_traceflag = p1->p_traceflag;
606 		if ((p2->p_tracevp = p1->p_tracevp) != NULL) {
607 			VREF(p2->p_tracevp);
608 			KASSERT(p1->p_tracecred != NULL,
609 			    ("ktrace vnode with no cred"));
610 			p2->p_tracecred = crhold(p1->p_tracecred);
611 		}
612 	}
613 	mtx_unlock(&ktrace_mtx);
614 #endif
615 
616 	/*
617 	 * If PF_FORK is set, the child process inherits the
618 	 * procfs ioctl flags from its parent.
619 	 */
620 	if (p1->p_pfsflags & PF_FORK) {
621 		p2->p_stops = p1->p_stops;
622 		p2->p_pfsflags = p1->p_pfsflags;
623 	}
624 
625 	/*
626 	 * This begins the section where we must prevent the parent
627 	 * from being swapped.
628 	 */
629 	_PHOLD(p1);
630 	PROC_UNLOCK(p1);
631 
632 	/*
633 	 * Attach the new process to its parent.
634 	 *
635 	 * If RFNOWAIT is set, the newly created process becomes a child
636 	 * of init.  This effectively disassociates the child from the
637 	 * parent.
638 	 */
639 	if (flags & RFNOWAIT)
640 		pptr = initproc;
641 	else
642 		pptr = p1;
643 	p2->p_pptr = pptr;
644 	LIST_INSERT_HEAD(&pptr->p_children, p2, p_sibling);
645 	sx_xunlock(&proctree_lock);
646 
647 	/* Inform accounting that we have forked. */
648 	p2->p_acflag = AFORK;
649 	PROC_UNLOCK(p2);
650 
651 	/*
652 	 * Finish creating the child process.  It will return via a different
653 	 * execution path later.  (ie: directly into user mode)
654 	 */
655 	vm_forkproc(td, p2, td2, flags);
656 
657 	if (flags == (RFFDG | RFPROC)) {
658 		cnt.v_forks++;
659 		cnt.v_forkpages += p2->p_vmspace->vm_dsize +
660 		    p2->p_vmspace->vm_ssize;
661 	} else if (flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) {
662 		cnt.v_vforks++;
663 		cnt.v_vforkpages += p2->p_vmspace->vm_dsize +
664 		    p2->p_vmspace->vm_ssize;
665 	} else if (p1 == &proc0) {
666 		cnt.v_kthreads++;
667 		cnt.v_kthreadpages += p2->p_vmspace->vm_dsize +
668 		    p2->p_vmspace->vm_ssize;
669 	} else {
670 		cnt.v_rforks++;
671 		cnt.v_rforkpages += p2->p_vmspace->vm_dsize +
672 		    p2->p_vmspace->vm_ssize;
673 	}
674 
675 	/*
676 	 * Both processes are set up, now check if any loadable modules want
677 	 * to adjust anything.
678 	 *   What if they have an error? XXX
679 	 */
680 	EVENTHANDLER_INVOKE(process_fork, p1, p2, flags);
681 
682 	/*
683 	 * If RFSTOPPED not requested, make child runnable and add to
684 	 * run queue.
685 	 */
686 	microuptime(&p2->p_stats->p_start);
687 	if ((flags & RFSTOPPED) == 0) {
688 		mtx_lock_spin(&sched_lock);
689 		p2->p_state = PRS_NORMAL;
690 		TD_SET_CAN_RUN(td2);
691 		setrunqueue(td2);
692 		mtx_unlock_spin(&sched_lock);
693 	}
694 
695 	/*
696 	 * Now can be swapped.
697 	 */
698 	PROC_LOCK(p1);
699 	_PRELE(p1);
700 
701 	/*
702 	 * tell any interested parties about the new process
703 	 */
704 	KNOTE(&p1->p_klist, NOTE_FORK | p2->p_pid);
705 	PROC_UNLOCK(p1);
706 
707 	/*
708 	 * Preserve synchronization semantics of vfork.  If waiting for
709 	 * child to exec or exit, set P_PPWAIT on child, and sleep on our
710 	 * proc (in case of exit).
711 	 */
712 	PROC_LOCK(p2);
713 	while (p2->p_flag & P_PPWAIT)
714 		msleep(p1, &p2->p_mtx, PWAIT, "ppwait", 0);
715 	PROC_UNLOCK(p2);
716 
717 	/*
718 	 * If other threads are waiting, let them continue now
719 	 */
720 	if (p1->p_flag & P_THREADED) {
721 		PROC_LOCK(p1);
722 		thread_single_end();
723 		PROC_UNLOCK(p1);
724 	}
725 
726 	/*
727 	 * Return child proc pointer to parent.
728 	 */
729 	mtx_unlock(&Giant);
730 	*procp = p2;
731 	return (0);
732 fail:
733 	sx_xunlock(&allproc_lock);
734 	uma_zfree(proc_zone, newproc);
735 	if (p1->p_flag & P_THREADED) {
736 		PROC_LOCK(p1);
737 		thread_single_end();
738 		PROC_UNLOCK(p1);
739 	}
740 	tsleep(&forksleep, PUSER, "fork", hz / 2);
741 	mtx_unlock(&Giant);
742 	return (error);
743 }
744 
745 /*
746  * Handle the return of a child process from fork1().  This function
747  * is called from the MD fork_trampoline() entry point.
748  */
749 void
750 fork_exit(callout, arg, frame)
751 	void (*callout)(void *, struct trapframe *);
752 	void *arg;
753 	struct trapframe *frame;
754 {
755 	struct thread *td;
756 	struct proc *p;
757 
758 	if ((td = PCPU_GET(deadthread))) {
759 		PCPU_SET(deadthread, NULL);
760 		thread_stash(td);
761 	}
762 	td = curthread;
763 	p = td->td_proc;
764 	td->td_oncpu = PCPU_GET(cpuid);
765 	p->p_state = PRS_NORMAL;
766 	/*
767 	 * Finish setting up thread glue.  We need to initialize
768 	 * the thread into a td_critnest=1 state.  Some platforms
769 	 * may have already partially or fully initialized td_critnest
770 	 * and/or td_md.md_savecrit (when applciable).
771 	 *
772 	 * see <arch>/<arch>/critical.c
773 	 */
774 	sched_lock.mtx_lock = (uintptr_t)td;
775 	sched_lock.mtx_recurse = 0;
776 	cpu_critical_fork_exit();
777 	CTR3(KTR_PROC, "fork_exit: new thread %p (pid %d, %s)", td, p->p_pid,
778 	    p->p_comm);
779 	if (PCPU_GET(switchtime.sec) == 0)
780 		binuptime(PCPU_PTR(switchtime));
781 	PCPU_SET(switchticks, ticks);
782 	mtx_unlock_spin(&sched_lock);
783 
784 	/*
785 	 * cpu_set_fork_handler intercepts this function call to
786          * have this call a non-return function to stay in kernel mode.
787          * initproc has its own fork handler, but it does return.
788          */
789 	KASSERT(callout != NULL, ("NULL callout in fork_exit"));
790 	callout(arg, frame);
791 
792 	/*
793 	 * Check if a kernel thread misbehaved and returned from its main
794 	 * function.
795 	 */
796 	PROC_LOCK(p);
797 	if (p->p_flag & P_KTHREAD) {
798 		PROC_UNLOCK(p);
799 		mtx_lock(&Giant);
800 		printf("Kernel thread \"%s\" (pid %d) exited prematurely.\n",
801 		    p->p_comm, p->p_pid);
802 		kthread_exit(0);
803 	}
804 	PROC_UNLOCK(p);
805 #ifdef DIAGNOSTIC
806 	cred_free_thread(td);
807 #endif
808 	mtx_assert(&Giant, MA_NOTOWNED);
809 }
810 
811 /*
812  * Simplified back end of syscall(), used when returning from fork()
813  * directly into user mode.  Giant is not held on entry, and must not
814  * be held on return.  This function is passed in to fork_exit() as the
815  * first parameter and is called when returning to a new userland process.
816  */
817 void
818 fork_return(td, frame)
819 	struct thread *td;
820 	struct trapframe *frame;
821 {
822 
823 	userret(td, frame, 0);
824 #ifdef KTRACE
825 	if (KTRPOINT(td, KTR_SYSRET))
826 		ktrsysret(SYS_fork, 0, 0);
827 #endif
828 	mtx_assert(&Giant, MA_NOTOWNED);
829 }
830