xref: /freebsd/sys/kern/kern_fork.c (revision 9764c9d36e5a00939d7d0ea9aa35579657077c63)
1 /*
2  * Copyright (c) 1982, 1986, 1989, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  * (c) UNIX System Laboratories, Inc.
5  * All or some portions of this file are derived from material licensed
6  * to the University of California by American Telephone and Telegraph
7  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8  * the permission of UNIX System Laboratories, Inc.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. All advertising materials mentioning features or use of this software
19  *    must display the following acknowledgement:
20  *	This product includes software developed by the University of
21  *	California, Berkeley and its contributors.
22  * 4. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  *	@(#)kern_fork.c	8.6 (Berkeley) 4/8/94
39  * $FreeBSD$
40  */
41 
42 #include "opt_ktrace.h"
43 
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/sysproto.h>
47 #include <sys/filedesc.h>
48 #include <sys/kernel.h>
49 #include <sys/sysctl.h>
50 #include <sys/malloc.h>
51 #include <sys/mutex.h>
52 #include <sys/proc.h>
53 #include <sys/resourcevar.h>
54 #include <sys/syscall.h>
55 #include <sys/vnode.h>
56 #include <sys/acct.h>
57 #include <sys/ktr.h>
58 #include <sys/ktrace.h>
59 #include <sys/kthread.h>
60 #include <sys/unistd.h>
61 #include <sys/jail.h>
62 
63 #include <vm/vm.h>
64 #include <sys/lock.h>
65 #include <vm/pmap.h>
66 #include <vm/vm_map.h>
67 #include <vm/vm_extern.h>
68 #include <vm/vm_zone.h>
69 
70 #include <sys/vmmeter.h>
71 #include <sys/user.h>
72 
73 static MALLOC_DEFINE(M_ATFORK, "atfork", "atfork callback");
74 
75 static int	fast_vfork = 1;
76 SYSCTL_INT(_kern, OID_AUTO, fast_vfork, CTLFLAG_RW, &fast_vfork, 0,
77     "flag to indicate whether we have a fast vfork()");
78 
79 /*
80  * These are the stuctures used to create a callout list for things to do
81  * when forking a process
82  */
83 struct forklist {
84 	forklist_fn function;
85 	TAILQ_ENTRY(forklist) next;
86 };
87 
88 TAILQ_HEAD(forklist_head, forklist);
89 static struct forklist_head fork_list = TAILQ_HEAD_INITIALIZER(fork_list);
90 
91 #ifndef _SYS_SYSPROTO_H_
92 struct fork_args {
93 	int     dummy;
94 };
95 #endif
96 
97 /* ARGSUSED */
98 int
99 fork(p, uap)
100 	struct proc *p;
101 	struct fork_args *uap;
102 {
103 	int error;
104 	struct proc *p2;
105 
106 	error = fork1(p, RFFDG | RFPROC, &p2);
107 	if (error == 0) {
108 		p->p_retval[0] = p2->p_pid;
109 		p->p_retval[1] = 0;
110 	}
111 	return error;
112 }
113 
114 /* ARGSUSED */
115 int
116 vfork(p, uap)
117 	struct proc *p;
118 	struct vfork_args *uap;
119 {
120 	int error;
121 	struct proc *p2;
122 
123 	error = fork1(p, RFFDG | RFPROC | RFPPWAIT | RFMEM, &p2);
124 	if (error == 0) {
125 		p->p_retval[0] = p2->p_pid;
126 		p->p_retval[1] = 0;
127 	}
128 	return error;
129 }
130 
131 int
132 rfork(p, uap)
133 	struct proc *p;
134 	struct rfork_args *uap;
135 {
136 	int error;
137 	struct proc *p2;
138 
139 	/* mask kernel only flags out of the user flags */
140 	error = fork1(p, uap->flags & ~RFKERNELONLY, &p2);
141 	if (error == 0) {
142 		p->p_retval[0] = p2 ? p2->p_pid : 0;
143 		p->p_retval[1] = 0;
144 	}
145 	return error;
146 }
147 
148 
149 int	nprocs = 1;				/* process 0 */
150 static int nextpid = 0;
151 SYSCTL_INT(_kern, OID_AUTO, lastpid, CTLFLAG_RD, &nextpid, 0,
152     "Last used PID");
153 
154 /*
155  * Random component to nextpid generation.  We mix in a random factor to make
156  * it a little harder to predict.  We sanity check the modulus value to avoid
157  * doing it in critical paths.  Don't let it be too small or we pointlessly
158  * waste randomness entropy, and don't let it be impossibly large.  Using a
159  * modulus that is too big causes a LOT more process table scans and slows
160  * down fork processing as the pidchecked caching is defeated.
161  */
162 static int randompid = 0;
163 
164 static int
165 sysctl_kern_randompid(SYSCTL_HANDLER_ARGS)
166 {
167 	int error, pid;
168 
169 	pid = randompid;
170 	error = sysctl_handle_int(oidp, &pid, 0, req);
171 	if (error || !req->newptr)
172 		return (error);
173 	if (pid < 0 || pid > PID_MAX - 100)	/* out of range */
174 		pid = PID_MAX - 100;
175 	else if (pid < 2)			/* NOP */
176 		pid = 0;
177 	else if (pid < 100)			/* Make it reasonable */
178 		pid = 100;
179 	randompid = pid;
180 	return (error);
181 }
182 
183 SYSCTL_PROC(_kern, OID_AUTO, randompid, CTLTYPE_INT|CTLFLAG_RW,
184     0, 0, sysctl_kern_randompid, "I", "Random PID modulus");
185 
186 int
187 fork1(p1, flags, procp)
188 	struct proc *p1;			/* parent proc */
189 	int flags;
190 	struct proc **procp;			/* child proc */
191 {
192 	struct proc *p2, *pptr;
193 	uid_t uid;
194 	struct proc *newproc;
195 	int trypid;
196 	int ok;
197 	static int pidchecked = 0;
198 	struct forklist *ep;
199 
200 	/* Can't copy and clear */
201 	if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG))
202 		return (EINVAL);
203 
204 	/*
205 	 * Here we don't create a new process, but we divorce
206 	 * certain parts of a process from itself.
207 	 */
208 	if ((flags & RFPROC) == 0) {
209 
210 		vm_fork(p1, 0, flags);
211 
212 		/*
213 		 * Close all file descriptors.
214 		 */
215 		if (flags & RFCFDG) {
216 			struct filedesc *fdtmp;
217 			fdtmp = fdinit(p1);
218 			fdfree(p1);
219 			p1->p_fd = fdtmp;
220 		}
221 
222 		/*
223 		 * Unshare file descriptors (from parent.)
224 		 */
225 		if (flags & RFFDG) {
226 			if (p1->p_fd->fd_refcnt > 1) {
227 				struct filedesc *newfd;
228 				newfd = fdcopy(p1);
229 				fdfree(p1);
230 				p1->p_fd = newfd;
231 			}
232 		}
233 		*procp = NULL;
234 		return (0);
235 	}
236 
237 	/*
238 	 * Although process entries are dynamically created, we still keep
239 	 * a global limit on the maximum number we will create.  Don't allow
240 	 * a nonprivileged user to use the last process; don't let root
241 	 * exceed the limit. The variable nprocs is the current number of
242 	 * processes, maxproc is the limit.
243 	 */
244 	uid = p1->p_cred->p_ruid;
245 	if ((nprocs >= maxproc - 1 && uid != 0) || nprocs >= maxproc) {
246 		tablefull("proc");
247 		return (EAGAIN);
248 	}
249 	/*
250 	 * Increment the nprocs resource before blocking can occur.  There
251 	 * are hard-limits as to the number of processes that can run.
252 	 */
253 	nprocs++;
254 
255 	/*
256 	 * Increment the count of procs running with this uid. Don't allow
257 	 * a nonprivileged user to exceed their current limit.
258 	 */
259 	ok = chgproccnt(p1->p_cred->p_uidinfo, 1,
260 		(uid != 0) ? p1->p_rlimit[RLIMIT_NPROC].rlim_cur : 0);
261 	if (!ok) {
262 		/*
263 		 * Back out the process count
264 		 */
265 		nprocs--;
266 		return (EAGAIN);
267 	}
268 
269 	/* Allocate new proc. */
270 	newproc = zalloc(proc_zone);
271 
272 	/*
273 	 * Setup linkage for kernel based threading
274 	 */
275 	if((flags & RFTHREAD) != 0) {
276 		newproc->p_peers = p1->p_peers;
277 		p1->p_peers = newproc;
278 		newproc->p_leader = p1->p_leader;
279 	} else {
280 		newproc->p_peers = NULL;
281 		newproc->p_leader = newproc;
282 	}
283 
284 	newproc->p_vmspace = NULL;
285 
286 	/*
287 	 * Find an unused process ID.  We remember a range of unused IDs
288 	 * ready to use (from nextpid+1 through pidchecked-1).
289 	 *
290 	 * If RFHIGHPID is set (used during system boot), do not allocate
291 	 * low-numbered pids.
292 	 */
293 	ALLPROC_LOCK(AP_EXCLUSIVE);
294 	trypid = nextpid + 1;
295 	if (flags & RFHIGHPID) {
296 		if (trypid < 10) {
297 			trypid = 10;
298 		}
299 	} else {
300 		if (randompid)
301 			trypid += arc4random() % randompid;
302 	}
303 retry:
304 	/*
305 	 * If the process ID prototype has wrapped around,
306 	 * restart somewhat above 0, as the low-numbered procs
307 	 * tend to include daemons that don't exit.
308 	 */
309 	if (trypid >= PID_MAX) {
310 		trypid = trypid % PID_MAX;
311 		if (trypid < 100)
312 			trypid += 100;
313 		pidchecked = 0;
314 	}
315 	if (trypid >= pidchecked) {
316 		int doingzomb = 0;
317 
318 		pidchecked = PID_MAX;
319 		/*
320 		 * Scan the active and zombie procs to check whether this pid
321 		 * is in use.  Remember the lowest pid that's greater
322 		 * than trypid, so we can avoid checking for a while.
323 		 */
324 		p2 = LIST_FIRST(&allproc);
325 again:
326 		for (; p2 != NULL; p2 = LIST_NEXT(p2, p_list)) {
327 			while (p2->p_pid == trypid ||
328 			    p2->p_pgrp->pg_id == trypid ||
329 			    p2->p_session->s_sid == trypid) {
330 				trypid++;
331 				if (trypid >= pidchecked)
332 					goto retry;
333 			}
334 			if (p2->p_pid > trypid && pidchecked > p2->p_pid)
335 				pidchecked = p2->p_pid;
336 			if (p2->p_pgrp->pg_id > trypid &&
337 			    pidchecked > p2->p_pgrp->pg_id)
338 				pidchecked = p2->p_pgrp->pg_id;
339 			if (p2->p_session->s_sid > trypid &&
340 			    pidchecked > p2->p_session->s_sid)
341 				pidchecked = p2->p_session->s_sid;
342 		}
343 		if (!doingzomb) {
344 			doingzomb = 1;
345 			p2 = LIST_FIRST(&zombproc);
346 			goto again;
347 		}
348 	}
349 
350 	/*
351 	 * RFHIGHPID does not mess with the nextpid counter during boot.
352 	 */
353 	if (flags & RFHIGHPID)
354 		pidchecked = 0;
355 	else
356 		nextpid = trypid;
357 
358 	p2 = newproc;
359 	p2->p_intr_nesting_level = 0;
360 	p2->p_stat = SIDL;			/* protect against others */
361 	p2->p_pid = trypid;
362 	LIST_INSERT_HEAD(&allproc, p2, p_list);
363 	LIST_INSERT_HEAD(PIDHASH(p2->p_pid), p2, p_hash);
364 	ALLPROC_LOCK(AP_RELEASE);
365 
366 	/*
367 	 * Make a proc table entry for the new process.
368 	 * Start by zeroing the section of proc that is zero-initialized,
369 	 * then copy the section that is copied directly from the parent.
370 	 */
371 	bzero(&p2->p_startzero,
372 	    (unsigned) ((caddr_t)&p2->p_endzero - (caddr_t)&p2->p_startzero));
373 	bcopy(&p1->p_startcopy, &p2->p_startcopy,
374 	    (unsigned) ((caddr_t)&p2->p_endcopy - (caddr_t)&p2->p_startcopy));
375 
376 	mtx_init(&p2->p_mtx, "process lock", MTX_DEF);
377 	p2->p_aioinfo = NULL;
378 
379 	/*
380 	 * Duplicate sub-structures as needed.
381 	 * Increase reference counts on shared objects.
382 	 * The p_stats and p_sigacts substructs are set in vm_fork.
383 	 */
384 	p2->p_flag = 0;
385 	mtx_lock_spin(&sched_lock);
386 	p2->p_sflag = PS_INMEM;
387 	if (p1->p_sflag & PS_PROFIL)
388 		startprofclock(p2);
389 	mtx_unlock_spin(&sched_lock);
390 	MALLOC(p2->p_cred, struct pcred *, sizeof(struct pcred),
391 	    M_SUBPROC, M_WAITOK);
392 	bcopy(p1->p_cred, p2->p_cred, sizeof(*p2->p_cred));
393 	p2->p_cred->p_refcnt = 1;
394 	crhold(p1->p_ucred);
395 	uihold(p1->p_cred->p_uidinfo);
396 
397 	if (p2->p_args)
398 		p2->p_args->ar_ref++;
399 
400 	if (flags & RFSIGSHARE) {
401 		p2->p_procsig = p1->p_procsig;
402 		p2->p_procsig->ps_refcnt++;
403 		if (p1->p_sigacts == &p1->p_addr->u_sigacts) {
404 			struct sigacts *newsigacts;
405 			int s;
406 
407 			/* Create the shared sigacts structure */
408 			MALLOC(newsigacts, struct sigacts *,
409 			    sizeof(struct sigacts), M_SUBPROC, M_WAITOK);
410 			s = splhigh();
411 			/*
412 			 * Set p_sigacts to the new shared structure.
413 			 * Note that this is updating p1->p_sigacts at the
414 			 * same time, since p_sigacts is just a pointer to
415 			 * the shared p_procsig->ps_sigacts.
416 			 */
417 			p2->p_sigacts  = newsigacts;
418 			bcopy(&p1->p_addr->u_sigacts, p2->p_sigacts,
419 			    sizeof(*p2->p_sigacts));
420 			*p2->p_sigacts = p1->p_addr->u_sigacts;
421 			splx(s);
422 		}
423 	} else {
424 		MALLOC(p2->p_procsig, struct procsig *, sizeof(struct procsig),
425 		    M_SUBPROC, M_WAITOK);
426 		bcopy(p1->p_procsig, p2->p_procsig, sizeof(*p2->p_procsig));
427 		p2->p_procsig->ps_refcnt = 1;
428 		p2->p_sigacts = NULL;	/* finished in vm_fork() */
429 	}
430 	if (flags & RFLINUXTHPN)
431 	        p2->p_sigparent = SIGUSR1;
432 	else
433 	        p2->p_sigparent = SIGCHLD;
434 
435 	/* bump references to the text vnode (for procfs) */
436 	p2->p_textvp = p1->p_textvp;
437 	if (p2->p_textvp)
438 		VREF(p2->p_textvp);
439 
440 	if (flags & RFCFDG)
441 		p2->p_fd = fdinit(p1);
442 	else if (flags & RFFDG)
443 		p2->p_fd = fdcopy(p1);
444 	else
445 		p2->p_fd = fdshare(p1);
446 
447 	/*
448 	 * If p_limit is still copy-on-write, bump refcnt,
449 	 * otherwise get a copy that won't be modified.
450 	 * (If PL_SHAREMOD is clear, the structure is shared
451 	 * copy-on-write.)
452 	 */
453 	if (p1->p_limit->p_lflags & PL_SHAREMOD)
454 		p2->p_limit = limcopy(p1->p_limit);
455 	else {
456 		p2->p_limit = p1->p_limit;
457 		p2->p_limit->p_refcnt++;
458 	}
459 
460 	/*
461 	 * Preserve some more flags in subprocess.  P_PROFIL has already
462 	 * been preserved.
463 	 */
464 	p2->p_flag |= p1->p_flag & P_SUGID;
465 	if (p1->p_session->s_ttyvp != NULL && p1->p_flag & P_CONTROLT)
466 		p2->p_flag |= P_CONTROLT;
467 	if (flags & RFPPWAIT)
468 		p2->p_flag |= P_PPWAIT;
469 
470 	LIST_INSERT_AFTER(p1, p2, p_pglist);
471 
472 	/*
473 	 * Attach the new process to its parent.
474 	 *
475 	 * If RFNOWAIT is set, the newly created process becomes a child
476 	 * of init.  This effectively disassociates the child from the
477 	 * parent.
478 	 */
479 	if (flags & RFNOWAIT)
480 		pptr = initproc;
481 	else
482 		pptr = p1;
483 	PROCTREE_LOCK(PT_EXCLUSIVE);
484 	p2->p_pptr = pptr;
485 	LIST_INSERT_HEAD(&pptr->p_children, p2, p_sibling);
486 	PROCTREE_LOCK(PT_RELEASE);
487 	LIST_INIT(&p2->p_children);
488 	LIST_INIT(&p2->p_heldmtx);
489 	LIST_INIT(&p2->p_contested);
490 
491 	callout_init(&p2->p_itcallout, 0);
492 	callout_init(&p2->p_slpcallout, 1);
493 
494 #ifdef KTRACE
495 	/*
496 	 * Copy traceflag and tracefile if enabled.
497 	 * If not inherited, these were zeroed above.
498 	 */
499 	if (p1->p_traceflag&KTRFAC_INHERIT) {
500 		p2->p_traceflag = p1->p_traceflag;
501 		if ((p2->p_tracep = p1->p_tracep) != NULL)
502 			VREF(p2->p_tracep);
503 	}
504 #endif
505 
506 	/*
507 	 * set priority of child to be that of parent
508 	 */
509 	p2->p_estcpu = p1->p_estcpu;
510 
511 	/*
512 	 * This begins the section where we must prevent the parent
513 	 * from being swapped.
514 	 */
515 	PHOLD(p1);
516 
517 	/*
518 	 * Finish creating the child process.  It will return via a different
519 	 * execution path later.  (ie: directly into user mode)
520 	 */
521 	vm_fork(p1, p2, flags);
522 
523 	if (flags == (RFFDG | RFPROC)) {
524 		cnt.v_forks++;
525 		cnt.v_forkpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize;
526 	} else if (flags == (RFFDG | RFPROC | RFPPWAIT | RFMEM)) {
527 		cnt.v_vforks++;
528 		cnt.v_vforkpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize;
529 	} else if (p1 == &proc0) {
530 		cnt.v_kthreads++;
531 		cnt.v_kthreadpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize;
532 	} else {
533 		cnt.v_rforks++;
534 		cnt.v_rforkpages += p2->p_vmspace->vm_dsize + p2->p_vmspace->vm_ssize;
535 	}
536 
537 	/*
538 	 * Both processes are set up, now check if any loadable modules want
539 	 * to adjust anything.
540 	 *   What if they have an error? XXX
541 	 */
542 	TAILQ_FOREACH(ep, &fork_list, next) {
543 		(*ep->function)(p1, p2, flags);
544 	}
545 
546 	/*
547 	 * If RFSTOPPED not requested, make child runnable and add to
548 	 * run queue.
549 	 */
550 	microtime(&(p2->p_stats->p_start));
551 	p2->p_acflag = AFORK;
552 	if ((flags & RFSTOPPED) == 0) {
553 		splhigh();
554 		mtx_lock_spin(&sched_lock);
555 		p2->p_stat = SRUN;
556 		setrunqueue(p2);
557 		mtx_unlock_spin(&sched_lock);
558 		spl0();
559 	}
560 
561 	/*
562 	 * Now can be swapped.
563 	 */
564 	PRELE(p1);
565 
566 	/*
567 	 * tell any interested parties about the new process
568 	 */
569 	KNOTE(&p1->p_klist, NOTE_FORK | p2->p_pid);
570 
571 	/*
572 	 * Preserve synchronization semantics of vfork.  If waiting for
573 	 * child to exec or exit, set P_PPWAIT on child, and sleep on our
574 	 * proc (in case of exit).
575 	 */
576 	while (p2->p_flag & P_PPWAIT)
577 		tsleep(p1, PWAIT, "ppwait", 0);
578 
579 	/*
580 	 * Return child proc pointer to parent.
581 	 */
582 	*procp = p2;
583 	return (0);
584 }
585 
586 /*
587  * The next two functionms are general routines to handle adding/deleting
588  * items on the fork callout list.
589  *
590  * at_fork():
591  * Take the arguments given and put them onto the fork callout list,
592  * However first make sure that it's not already there.
593  * Returns 0 on success or a standard error number.
594  */
595 
596 int
597 at_fork(function)
598 	forklist_fn function;
599 {
600 	struct forklist *ep;
601 
602 #ifdef INVARIANTS
603 	/* let the programmer know if he's been stupid */
604 	if (rm_at_fork(function))
605 		printf("WARNING: fork callout entry (%p) already present\n",
606 		    function);
607 #endif
608 	ep = malloc(sizeof(*ep), M_ATFORK, M_NOWAIT);
609 	if (ep == NULL)
610 		return (ENOMEM);
611 	ep->function = function;
612 	TAILQ_INSERT_TAIL(&fork_list, ep, next);
613 	return (0);
614 }
615 
616 /*
617  * Scan the exit callout list for the given item and remove it..
618  * Returns the number of items removed (0 or 1)
619  */
620 
621 int
622 rm_at_fork(function)
623 	forklist_fn function;
624 {
625 	struct forklist *ep;
626 
627 	TAILQ_FOREACH(ep, &fork_list, next) {
628 		if (ep->function == function) {
629 			TAILQ_REMOVE(&fork_list, ep, next);
630 			free(ep, M_ATFORK);
631 			return(1);
632 		}
633 	}
634 	return (0);
635 }
636 
637 /*
638  * Handle the return of a child process from fork1().  This function
639  * is called from the MD fork_trampoline() entry point.
640  */
641 void
642 fork_exit(callout, arg, frame)
643 	void (*callout)(void *, struct trapframe *);
644 	void *arg;
645 	struct trapframe *frame;
646 {
647 	struct proc *p;
648 
649 	/*
650 	 * Setup the sched_lock state so that we can release it.
651 	 */
652 	sched_lock.mtx_lock = (uintptr_t)curproc;
653 	sched_lock.mtx_recurse = 0;
654 	mtx_unlock_spin(&sched_lock);
655 	/*
656 	 * XXX: We really shouldn't have to do this.
657 	 */
658 	enable_intr();
659 
660 #ifdef SMP
661 	if (PCPU_GET(switchtime.tv_sec) == 0)
662 		microuptime(PCPU_PTR(switchtime));
663 	PCPU_SET(switchticks, ticks);
664 #endif
665 
666 	/*
667 	 * cpu_set_fork_handler intercepts this function call to
668          * have this call a non-return function to stay in kernel mode.
669          * initproc has its own fork handler, but it does return.
670          */
671 	KASSERT(callout != NULL, ("NULL callout in fork_exit"));
672 	callout(arg, frame);
673 
674 	/*
675 	 * Check if a kernel thread misbehaved and returned from its main
676 	 * function.
677 	 */
678 	p = CURPROC;
679 	if (p->p_flag & P_KTHREAD) {
680 		mtx_lock(&Giant);
681 		printf("Kernel thread \"%s\" (pid %d) exited prematurely.\n",
682 		    p->p_comm, p->p_pid);
683 		kthread_exit(0);
684 	}
685 	mtx_assert(&Giant, MA_NOTOWNED);
686 }
687 
688 /*
689  * Simplified back end of syscall(), used when returning from fork()
690  * directly into user mode.  Giant is not held on entry, and must not
691  * be held on return.  This function is passed in to fork_exit() as the
692  * first parameter and is called when returning to a new userland process.
693  */
694 void
695 fork_return(p, frame)
696 	struct proc *p;
697 	struct trapframe *frame;
698 {
699 
700 	userret(p, frame, 0);
701 #ifdef KTRACE
702 	if (KTRPOINT(p, KTR_SYSRET)) {
703 		if (!mtx_owned(&Giant))
704 			mtx_lock(&Giant);
705 		ktrsysret(p->p_tracep, SYS_fork, 0, 0);
706 	}
707 #endif
708 	if (mtx_owned(&Giant))
709 		mtx_unlock(&Giant);
710 	mtx_assert(&Giant, MA_NOTOWNED);
711 }
712