xref: /titanic_51/usr/src/uts/common/os/fork.c (revision 1c3a0e9dd9f5bb14378d99eee9ed261ce24fc4d3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
28 /*	  All Rights Reserved  	*/
29 
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/sysmacros.h>
33 #include <sys/signal.h>
34 #include <sys/cred.h>
35 #include <sys/policy.h>
36 #include <sys/user.h>
37 #include <sys/systm.h>
38 #include <sys/cpuvar.h>
39 #include <sys/vfs.h>
40 #include <sys/vnode.h>
41 #include <sys/file.h>
42 #include <sys/errno.h>
43 #include <sys/time.h>
44 #include <sys/proc.h>
45 #include <sys/cmn_err.h>
46 #include <sys/acct.h>
47 #include <sys/tuneable.h>
48 #include <sys/class.h>
49 #include <sys/kmem.h>
50 #include <sys/session.h>
51 #include <sys/ucontext.h>
52 #include <sys/stack.h>
53 #include <sys/procfs.h>
54 #include <sys/prsystm.h>
55 #include <sys/vmsystm.h>
56 #include <sys/vtrace.h>
57 #include <sys/debug.h>
58 #include <sys/shm_impl.h>
59 #include <sys/door_data.h>
60 #include <vm/as.h>
61 #include <vm/rm.h>
62 #include <c2/audit.h>
63 #include <sys/var.h>
64 #include <sys/schedctl.h>
65 #include <sys/utrap.h>
66 #include <sys/task.h>
67 #include <sys/resource.h>
68 #include <sys/cyclic.h>
69 #include <sys/lgrp.h>
70 #include <sys/rctl.h>
71 #include <sys/contract_impl.h>
72 #include <sys/contract/process_impl.h>
73 #include <sys/list.h>
74 #include <sys/dtrace.h>
75 #include <sys/pool.h>
76 #include <sys/zone.h>
77 #include <sys/sdt.h>
78 #include <sys/class.h>
79 #include <sys/corectl.h>
80 #include <sys/brand.h>
81 #include <sys/fork.h>
82 
83 static int64_t cfork(int, int, int);
84 static int getproc(proc_t **, int);
85 static void fork_fail(proc_t *);
86 static void forklwp_fail(proc_t *);
87 
88 int fork_fail_pending;
89 
90 extern struct kmem_cache *process_cache;
91 
92 /*
93  * forkall system call.
94  */
95 int64_t
96 forkall(void)
97 {
98 	return (cfork(0, 0, 0));
99 }
100 
101 /*
102  * The parent is stopped until the child invokes relvm().
103  */
104 int64_t
105 vfork(void)
106 {
107 	curthread->t_post_sys = 1;	/* so vfwait() will be called */
108 	return (cfork(1, 1, 0));
109 }
110 
111 /*
112  * fork system call, aka fork1.
113  */
114 int64_t
115 fork1(void)
116 {
117 	return (cfork(0, 1, 0));
118 }
119 
120 /*
121  * The forkall(), vfork(), and fork1() system calls are no longer
122  * invoked by libc.  They are retained only for the benefit of
123  * old statically-linked applications.  They should be eliminated
124  * when we no longer care about such old and broken applications.
125  */
126 
127 /*
128  * forksys system call - forkx, forkallx, vforkx.
129  * This is the interface now invoked by libc.
130  */
131 int64_t
132 forksys(int subcode, int flags)
133 {
134 	switch (subcode) {
135 	case 0:
136 		return (cfork(0, 1, flags));	/* forkx(flags) */
137 	case 1:
138 		return (cfork(0, 0, flags));	/* forkallx(flags) */
139 	case 2:
140 		curthread->t_post_sys = 1;	/* so vfwait() will be called */
141 		return (cfork(1, 1, flags));	/* vforkx(flags) */
142 	default:
143 		return ((int64_t)set_errno(EINVAL));
144 	}
145 }
146 
147 /* ARGSUSED */
148 static int64_t
149 cfork(int isvfork, int isfork1, int flags)
150 {
151 	proc_t *p = ttoproc(curthread);
152 	struct as *as;
153 	proc_t *cp, **orphpp;
154 	klwp_t *clone;
155 	kthread_t *t;
156 	task_t *tk;
157 	rval_t	r;
158 	int error;
159 	int i;
160 	rctl_set_t *dup_set;
161 	rctl_alloc_gp_t *dup_gp;
162 	rctl_entity_p_t e;
163 	lwpdir_t *ldp;
164 	lwpent_t *lep;
165 	lwpent_t *clep;
166 
167 	/*
168 	 * Allow only these two flags.
169 	 */
170 	if ((flags & ~(FORK_NOSIGCHLD | FORK_WAITPID)) != 0) {
171 		error = EINVAL;
172 		goto forkerr;
173 	}
174 
175 	/*
176 	 * fork is not supported for the /proc agent lwp.
177 	 */
178 	if (curthread == p->p_agenttp) {
179 		error = ENOTSUP;
180 		goto forkerr;
181 	}
182 
183 	if ((error = secpolicy_basic_fork(CRED())) != 0)
184 		goto forkerr;
185 
186 	/*
187 	 * If the calling lwp is doing a fork1() then the
188 	 * other lwps in this process are not duplicated and
189 	 * don't need to be held where their kernel stacks can be
190 	 * cloned.  If doing forkall(), the process is held with
191 	 * SHOLDFORK, so that the lwps are at a point where their
192 	 * stacks can be copied which is on entry or exit from
193 	 * the kernel.
194 	 */
195 	if (!holdlwps(isfork1 ? SHOLDFORK1 : SHOLDFORK)) {
196 		aston(curthread);
197 		error = EINTR;
198 		goto forkerr;
199 	}
200 
201 #if defined(__sparc)
202 	/*
203 	 * Ensure that the user stack is fully constructed
204 	 * before creating the child process structure.
205 	 */
206 	(void) flush_user_windows_to_stack(NULL);
207 #endif
208 
209 	mutex_enter(&p->p_lock);
210 	/*
211 	 * If this is vfork(), cancel any suspend request we might
212 	 * have gotten from some other thread via lwp_suspend().
213 	 * Otherwise we could end up with a deadlock on return
214 	 * from the vfork() in both the parent and the child.
215 	 */
216 	if (isvfork)
217 		curthread->t_proc_flag &= ~TP_HOLDLWP;
218 	/*
219 	 * Prevent our resource set associations from being changed during fork.
220 	 */
221 	pool_barrier_enter();
222 	mutex_exit(&p->p_lock);
223 
224 	/*
225 	 * Create a child proc struct. Place a VN_HOLD on appropriate vnodes.
226 	 */
227 	if (getproc(&cp, 0) < 0) {
228 		mutex_enter(&p->p_lock);
229 		pool_barrier_exit();
230 		continuelwps(p);
231 		mutex_exit(&p->p_lock);
232 		error = EAGAIN;
233 		goto forkerr;
234 	}
235 
236 	TRACE_2(TR_FAC_PROC, TR_PROC_FORK, "proc_fork:cp %p p %p", cp, p);
237 
238 	/*
239 	 * Assign an address space to child
240 	 */
241 	if (isvfork) {
242 		/*
243 		 * Clear any watched areas and remember the
244 		 * watched pages for restoring in vfwait().
245 		 */
246 		as = p->p_as;
247 		if (avl_numnodes(&as->a_wpage) != 0) {
248 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
249 			as_clearwatch(as);
250 			p->p_wpage = as->a_wpage;
251 			avl_create(&as->a_wpage, wp_compare,
252 			    sizeof (struct watched_page),
253 			    offsetof(struct watched_page, wp_link));
254 			AS_LOCK_EXIT(as, &as->a_lock);
255 		}
256 		cp->p_as = as;
257 		cp->p_flag |= SVFORK;
258 	} else {
259 		/*
260 		 * We need to hold P_PR_LOCK until the address space has
261 		 * been duplicated and we've had a chance to remove from the
262 		 * child any DTrace probes that were in the parent. Holding
263 		 * P_PR_LOCK prevents any new probes from being added and any
264 		 * extant probes from being removed.
265 		 */
266 		mutex_enter(&p->p_lock);
267 		sprlock_proc(p);
268 		p->p_flag |= SFORKING;
269 		mutex_exit(&p->p_lock);
270 
271 		error = as_dup(p->p_as, cp);
272 		if (error != 0) {
273 			mutex_enter(&p->p_lock);
274 			sprunlock(p);
275 			fork_fail(cp);
276 			mutex_enter(&pidlock);
277 			orphpp = &p->p_orphan;
278 			while (*orphpp != cp)
279 				orphpp = &(*orphpp)->p_nextorph;
280 			*orphpp = cp->p_nextorph;
281 			if (p->p_child == cp)
282 				p->p_child = cp->p_sibling;
283 			if (cp->p_sibling)
284 				cp->p_sibling->p_psibling = cp->p_psibling;
285 			if (cp->p_psibling)
286 				cp->p_psibling->p_sibling = cp->p_sibling;
287 			mutex_enter(&cp->p_lock);
288 			tk = cp->p_task;
289 			task_detach(cp);
290 			ASSERT(cp->p_pool->pool_ref > 0);
291 			atomic_add_32(&cp->p_pool->pool_ref, -1);
292 			mutex_exit(&cp->p_lock);
293 			pid_exit(cp);
294 			mutex_exit(&pidlock);
295 			task_rele(tk);
296 
297 			mutex_enter(&p->p_lock);
298 			p->p_flag &= ~SFORKING;
299 			pool_barrier_exit();
300 			continuelwps(p);
301 			mutex_exit(&p->p_lock);
302 			/*
303 			 * Preserve ENOMEM error condition but
304 			 * map all others to EAGAIN.
305 			 */
306 			error = (error == ENOMEM) ? ENOMEM : EAGAIN;
307 			goto forkerr;
308 		}
309 
310 		/*
311 		 * Remove all DTrace tracepoints from the child process. We
312 		 * need to do this _before_ duplicating USDT providers since
313 		 * any associated probes may be immediately enabled.
314 		 */
315 		if (p->p_dtrace_count > 0)
316 			dtrace_fasttrap_fork(p, cp);
317 
318 		mutex_enter(&p->p_lock);
319 		sprunlock(p);
320 
321 		/* Duplicate parent's shared memory */
322 		if (p->p_segacct)
323 			shmfork(p, cp);
324 
325 		/*
326 		 * Duplicate any helper actions and providers. The SFORKING
327 		 * we set above informs the code to enable USDT probes that
328 		 * sprlock() may fail because the child is being forked.
329 		 */
330 		if (p->p_dtrace_helpers != NULL) {
331 			ASSERT(dtrace_helpers_fork != NULL);
332 			(*dtrace_helpers_fork)(p, cp);
333 		}
334 
335 		mutex_enter(&p->p_lock);
336 		p->p_flag &= ~SFORKING;
337 		mutex_exit(&p->p_lock);
338 	}
339 
340 	/*
341 	 * Duplicate parent's resource controls.
342 	 */
343 	dup_set = rctl_set_create();
344 	for (;;) {
345 		dup_gp = rctl_set_dup_prealloc(p->p_rctls);
346 		mutex_enter(&p->p_rctls->rcs_lock);
347 		if (rctl_set_dup_ready(p->p_rctls, dup_gp))
348 			break;
349 		mutex_exit(&p->p_rctls->rcs_lock);
350 		rctl_prealloc_destroy(dup_gp);
351 	}
352 	e.rcep_p.proc = cp;
353 	e.rcep_t = RCENTITY_PROCESS;
354 	cp->p_rctls = rctl_set_dup(p->p_rctls, p, cp, &e, dup_set, dup_gp,
355 	    RCD_DUP | RCD_CALLBACK);
356 	mutex_exit(&p->p_rctls->rcs_lock);
357 
358 	rctl_prealloc_destroy(dup_gp);
359 
360 	/*
361 	 * Allocate the child's lwp directory and lwpid hash table.
362 	 */
363 	if (isfork1)
364 		cp->p_lwpdir_sz = 2;
365 	else
366 		cp->p_lwpdir_sz = p->p_lwpdir_sz;
367 	cp->p_lwpdir = cp->p_lwpfree = ldp =
368 	    kmem_zalloc(cp->p_lwpdir_sz * sizeof (lwpdir_t), KM_SLEEP);
369 	for (i = 1; i < cp->p_lwpdir_sz; i++, ldp++)
370 		ldp->ld_next = ldp + 1;
371 	cp->p_tidhash_sz = (cp->p_lwpdir_sz + 2) / 2;
372 	cp->p_tidhash =
373 	    kmem_zalloc(cp->p_tidhash_sz * sizeof (tidhash_t), KM_SLEEP);
374 
375 	/*
376 	 * Duplicate parent's lwps.
377 	 * Mutual exclusion is not needed because the process is
378 	 * in the hold state and only the current lwp is running.
379 	 */
380 	klgrpset_clear(cp->p_lgrpset);
381 	if (isfork1) {
382 		clone = forklwp(ttolwp(curthread), cp, curthread->t_tid);
383 		if (clone == NULL)
384 			goto forklwperr;
385 		/*
386 		 * Inherit only the lwp_wait()able flag,
387 		 * Daemon threads should not call fork1(), but oh well...
388 		 */
389 		lwptot(clone)->t_proc_flag |=
390 		    (curthread->t_proc_flag & TP_TWAIT);
391 	} else {
392 		/* this is forkall(), no one can be in lwp_wait() */
393 		ASSERT(p->p_lwpwait == 0 && p->p_lwpdwait == 0);
394 		/* for each entry in the parent's lwp directory... */
395 		for (i = 0, ldp = p->p_lwpdir; i < p->p_lwpdir_sz; i++, ldp++) {
396 			klwp_t *clwp;
397 			kthread_t *ct;
398 
399 			if ((lep = ldp->ld_entry) == NULL)
400 				continue;
401 
402 			if ((t = lep->le_thread) != NULL) {
403 				clwp = forklwp(ttolwp(t), cp, t->t_tid);
404 				if (clwp == NULL)
405 					goto forklwperr;
406 				ct = lwptot(clwp);
407 				/*
408 				 * Inherit lwp_wait()able and daemon flags.
409 				 */
410 				ct->t_proc_flag |=
411 				    (t->t_proc_flag & (TP_TWAIT|TP_DAEMON));
412 				/*
413 				 * Keep track of the clone of curthread to
414 				 * post return values through lwp_setrval().
415 				 * Mark other threads for special treatment
416 				 * by lwp_rtt() / post_syscall().
417 				 */
418 				if (t == curthread)
419 					clone = clwp;
420 				else
421 					ct->t_flag |= T_FORKALL;
422 			} else {
423 				/*
424 				 * Replicate zombie lwps in the child.
425 				 */
426 				clep = kmem_zalloc(sizeof (*clep), KM_SLEEP);
427 				clep->le_lwpid = lep->le_lwpid;
428 				clep->le_start = lep->le_start;
429 				lwp_hash_in(cp, clep,
430 				    cp->p_tidhash, cp->p_tidhash_sz, 0);
431 			}
432 		}
433 	}
434 
435 	/*
436 	 * Put new process in the parent's process contract, or put it
437 	 * in a new one if there is an active process template.  Send a
438 	 * fork event (if requested) to whatever contract the child is
439 	 * a member of.  Fails if the parent has been SIGKILLed.
440 	 */
441 	if (contract_process_fork(NULL, cp, p, B_TRUE) == NULL)
442 		goto forklwperr;
443 
444 	/*
445 	 * No fork failures occur beyond this point.
446 	 */
447 
448 	cp->p_lwpid = p->p_lwpid;
449 	if (!isfork1) {
450 		cp->p_lwpdaemon = p->p_lwpdaemon;
451 		cp->p_zombcnt = p->p_zombcnt;
452 		/*
453 		 * If the parent's lwp ids have wrapped around, so have the
454 		 * child's.
455 		 */
456 		cp->p_flag |= p->p_flag & SLWPWRAP;
457 	}
458 
459 	mutex_enter(&p->p_lock);
460 	corectl_path_hold(cp->p_corefile = p->p_corefile);
461 	corectl_content_hold(cp->p_content = p->p_content);
462 	mutex_exit(&p->p_lock);
463 
464 	/*
465 	 * Duplicate process context ops, if any.
466 	 */
467 	if (p->p_pctx)
468 		forkpctx(p, cp);
469 
470 #ifdef __sparc
471 	utrap_dup(p, cp);
472 #endif
473 	/*
474 	 * If the child process has been marked to stop on exit
475 	 * from this fork, arrange for all other lwps to stop in
476 	 * sympathy with the active lwp.
477 	 */
478 	if (PTOU(cp)->u_systrap &&
479 	    prismember(&PTOU(cp)->u_exitmask, curthread->t_sysnum)) {
480 		mutex_enter(&cp->p_lock);
481 		t = cp->p_tlist;
482 		do {
483 			t->t_proc_flag |= TP_PRSTOP;
484 			aston(t);	/* so TP_PRSTOP will be seen */
485 		} while ((t = t->t_forw) != cp->p_tlist);
486 		mutex_exit(&cp->p_lock);
487 	}
488 	/*
489 	 * If the parent process has been marked to stop on exit
490 	 * from this fork, and its asynchronous-stop flag has not
491 	 * been set, arrange for all other lwps to stop before
492 	 * they return back to user level.
493 	 */
494 	if (!(p->p_proc_flag & P_PR_ASYNC) && PTOU(p)->u_systrap &&
495 	    prismember(&PTOU(p)->u_exitmask, curthread->t_sysnum)) {
496 		mutex_enter(&p->p_lock);
497 		t = p->p_tlist;
498 		do {
499 			t->t_proc_flag |= TP_PRSTOP;
500 			aston(t);	/* so TP_PRSTOP will be seen */
501 		} while ((t = t->t_forw) != p->p_tlist);
502 		mutex_exit(&p->p_lock);
503 	}
504 
505 	if (PROC_IS_BRANDED(p))
506 		BROP(p)->b_lwp_setrval(clone, p->p_pid, 1);
507 	else
508 		lwp_setrval(clone, p->p_pid, 1);
509 
510 	/* set return values for parent */
511 	r.r_val1 = (int)cp->p_pid;
512 	r.r_val2 = 0;
513 
514 	/*
515 	 * pool_barrier_exit() can now be called because the child process has:
516 	 * - all identifying features cloned or set (p_pid, p_task, p_pool)
517 	 * - all resource sets associated (p_tlist->*->t_cpupart, p_as->a_mset)
518 	 * - any other fields set which are used in resource set binding.
519 	 */
520 	mutex_enter(&p->p_lock);
521 	pool_barrier_exit();
522 	mutex_exit(&p->p_lock);
523 
524 	mutex_enter(&pidlock);
525 	mutex_enter(&cp->p_lock);
526 
527 	/*
528 	 * Set flags telling the child what (not) to do on exit.
529 	 */
530 	if (flags & FORK_NOSIGCHLD)
531 		cp->p_pidflag |= CLDNOSIGCHLD;
532 	if (flags & FORK_WAITPID)
533 		cp->p_pidflag |= CLDWAITPID;
534 
535 	/*
536 	 * Now that there are lwps and threads attached, add the new
537 	 * process to the process group.
538 	 */
539 	pgjoin(cp, p->p_pgidp);
540 	cp->p_stat = SRUN;
541 	/*
542 	 * We are now done with all the lwps in the child process.
543 	 */
544 	t = cp->p_tlist;
545 	do {
546 		/*
547 		 * Set the lwp_suspend()ed lwps running.
548 		 * They will suspend properly at syscall exit.
549 		 */
550 		if (t->t_proc_flag & TP_HOLDLWP)
551 			lwp_create_done(t);
552 		else {
553 			/* set TS_CREATE to allow continuelwps() to work */
554 			thread_lock(t);
555 			ASSERT(t->t_state == TS_STOPPED &&
556 			    !(t->t_schedflag & (TS_CREATE|TS_CSTART)));
557 			t->t_schedflag |= TS_CREATE;
558 			thread_unlock(t);
559 		}
560 	} while ((t = t->t_forw) != cp->p_tlist);
561 	mutex_exit(&cp->p_lock);
562 
563 	if (isvfork) {
564 		CPU_STATS_ADDQ(CPU, sys, sysvfork, 1);
565 		mutex_enter(&p->p_lock);
566 		p->p_flag |= SVFWAIT;
567 		curthread->t_flag |= T_VFPARENT;
568 		DTRACE_PROC1(create, proc_t *, cp);
569 		cv_broadcast(&pr_pid_cv[p->p_slot]);	/* inform /proc */
570 		mutex_exit(&p->p_lock);
571 		/*
572 		 * Grab child's p_lock before dropping pidlock to ensure
573 		 * the process will not disappear before we set it running.
574 		 */
575 		mutex_enter(&cp->p_lock);
576 		mutex_exit(&pidlock);
577 		sigdefault(cp);
578 		continuelwps(cp);
579 		mutex_exit(&cp->p_lock);
580 	} else {
581 		CPU_STATS_ADDQ(CPU, sys, sysfork, 1);
582 		DTRACE_PROC1(create, proc_t *, cp);
583 		/*
584 		 * It is CL_FORKRET's job to drop pidlock.
585 		 * If we do it here, the process could be set running
586 		 * and disappear before CL_FORKRET() is called.
587 		 */
588 		CL_FORKRET(curthread, cp->p_tlist);
589 		schedctl_set_cidpri(curthread);
590 		ASSERT(MUTEX_NOT_HELD(&pidlock));
591 	}
592 
593 	return (r.r_vals);
594 
595 forklwperr:
596 	if (isvfork) {
597 		if (avl_numnodes(&p->p_wpage) != 0) {
598 			/* restore watchpoints to parent */
599 			as = p->p_as;
600 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
601 			as->a_wpage = p->p_wpage;
602 			avl_create(&p->p_wpage, wp_compare,
603 			    sizeof (struct watched_page),
604 			    offsetof(struct watched_page, wp_link));
605 			as_setwatch(as);
606 			AS_LOCK_EXIT(as, &as->a_lock);
607 		}
608 	} else {
609 		if (cp->p_segacct)
610 			shmexit(cp);
611 		as = cp->p_as;
612 		cp->p_as = &kas;
613 		as_free(as);
614 	}
615 
616 	if (cp->p_lwpdir) {
617 		for (i = 0, ldp = cp->p_lwpdir; i < cp->p_lwpdir_sz; i++, ldp++)
618 			if ((lep = ldp->ld_entry) != NULL)
619 				kmem_free(lep, sizeof (*lep));
620 		kmem_free(cp->p_lwpdir,
621 		    cp->p_lwpdir_sz * sizeof (*cp->p_lwpdir));
622 	}
623 	cp->p_lwpdir = NULL;
624 	cp->p_lwpfree = NULL;
625 	cp->p_lwpdir_sz = 0;
626 
627 	if (cp->p_tidhash)
628 		kmem_free(cp->p_tidhash,
629 		    cp->p_tidhash_sz * sizeof (*cp->p_tidhash));
630 	cp->p_tidhash = NULL;
631 	cp->p_tidhash_sz = 0;
632 
633 	forklwp_fail(cp);
634 	fork_fail(cp);
635 	rctl_set_free(cp->p_rctls);
636 	mutex_enter(&pidlock);
637 
638 	/*
639 	 * Detach failed child from task.
640 	 */
641 	mutex_enter(&cp->p_lock);
642 	tk = cp->p_task;
643 	task_detach(cp);
644 	ASSERT(cp->p_pool->pool_ref > 0);
645 	atomic_add_32(&cp->p_pool->pool_ref, -1);
646 	mutex_exit(&cp->p_lock);
647 
648 	orphpp = &p->p_orphan;
649 	while (*orphpp != cp)
650 		orphpp = &(*orphpp)->p_nextorph;
651 	*orphpp = cp->p_nextorph;
652 	if (p->p_child == cp)
653 		p->p_child = cp->p_sibling;
654 	if (cp->p_sibling)
655 		cp->p_sibling->p_psibling = cp->p_psibling;
656 	if (cp->p_psibling)
657 		cp->p_psibling->p_sibling = cp->p_sibling;
658 	pid_exit(cp);
659 	mutex_exit(&pidlock);
660 
661 	task_rele(tk);
662 
663 	mutex_enter(&p->p_lock);
664 	pool_barrier_exit();
665 	continuelwps(p);
666 	mutex_exit(&p->p_lock);
667 	error = EAGAIN;
668 forkerr:
669 	return ((int64_t)set_errno(error));
670 }
671 
672 /*
673  * Free allocated resources from getproc() if a fork failed.
674  */
675 static void
676 fork_fail(proc_t *cp)
677 {
678 	uf_info_t *fip = P_FINFO(cp);
679 
680 	fcnt_add(fip, -1);
681 	sigdelq(cp, NULL, 0);
682 
683 	mutex_enter(&pidlock);
684 	upcount_dec(crgetruid(cp->p_cred), crgetzoneid(cp->p_cred));
685 	mutex_exit(&pidlock);
686 
687 	/*
688 	 * single threaded, so no locking needed here
689 	 */
690 	crfree(cp->p_cred);
691 
692 	kmem_free(fip->fi_list, fip->fi_nfiles * sizeof (uf_entry_t));
693 
694 	VN_RELE(PTOU(curproc)->u_cdir);
695 	if (PTOU(curproc)->u_rdir)
696 		VN_RELE(PTOU(curproc)->u_rdir);
697 	if (cp->p_exec)
698 		VN_RELE(cp->p_exec);
699 	if (cp->p_execdir)
700 		VN_RELE(cp->p_execdir);
701 	if (PTOU(curproc)->u_cwd)
702 		refstr_rele(PTOU(curproc)->u_cwd);
703 }
704 
705 /*
706  * Clean up the lwps already created for this child process.
707  * The fork failed while duplicating all the lwps of the parent
708  * and those lwps already created must be freed.
709  * This process is invisible to the rest of the system,
710  * so we don't need to hold p->p_lock to protect the list.
711  */
712 static void
713 forklwp_fail(proc_t *p)
714 {
715 	kthread_t *t;
716 	task_t *tk;
717 
718 	while ((t = p->p_tlist) != NULL) {
719 		/*
720 		 * First remove the lwp from the process's p_tlist.
721 		 */
722 		if (t != t->t_forw)
723 			p->p_tlist = t->t_forw;
724 		else
725 			p->p_tlist = NULL;
726 		p->p_lwpcnt--;
727 		t->t_forw->t_back = t->t_back;
728 		t->t_back->t_forw = t->t_forw;
729 
730 		tk = p->p_task;
731 		mutex_enter(&p->p_zone->zone_nlwps_lock);
732 		tk->tk_nlwps--;
733 		tk->tk_proj->kpj_nlwps--;
734 		p->p_zone->zone_nlwps--;
735 		mutex_exit(&p->p_zone->zone_nlwps_lock);
736 
737 		ASSERT(t->t_schedctl == NULL);
738 
739 		if (t->t_door != NULL) {
740 			kmem_free(t->t_door, sizeof (door_data_t));
741 			t->t_door = NULL;
742 		}
743 		lwp_ctmpl_clear(ttolwp(t));
744 
745 		/*
746 		 * Remove the thread from the all threads list.
747 		 * We need to hold pidlock for this.
748 		 */
749 		mutex_enter(&pidlock);
750 		t->t_next->t_prev = t->t_prev;
751 		t->t_prev->t_next = t->t_next;
752 		CL_EXIT(t);	/* tell the scheduler that we're exiting */
753 		cv_broadcast(&t->t_joincv);	/* tell anyone in thread_join */
754 		mutex_exit(&pidlock);
755 
756 		/*
757 		 * Let the lgroup load averages know that this thread isn't
758 		 * going to show up (i.e. un-do what was done on behalf of
759 		 * this thread by the earlier lgrp_move_thread()).
760 		 */
761 		kpreempt_disable();
762 		lgrp_move_thread(t, NULL, 1);
763 		kpreempt_enable();
764 
765 		/*
766 		 * The thread was created TS_STOPPED.
767 		 * We change it to TS_FREE to avoid an
768 		 * ASSERT() panic in thread_free().
769 		 */
770 		t->t_state = TS_FREE;
771 		thread_rele(t);
772 		thread_free(t);
773 	}
774 }
775 
776 extern struct as kas;
777 
778 /*
779  * fork a kernel process.
780  */
781 int
782 newproc(void (*pc)(), caddr_t arg, id_t cid, int pri, struct contract **ct)
783 {
784 	proc_t *p;
785 	struct user *up;
786 	klwp_t *lwp;
787 	cont_process_t *ctp = NULL;
788 	rctl_entity_p_t e;
789 
790 	ASSERT(!(cid == syscid && ct != NULL));
791 	if (cid == syscid) {
792 		rctl_alloc_gp_t *init_gp;
793 		rctl_set_t *init_set;
794 
795 		if (getproc(&p, 1) < 0)
796 			return (EAGAIN);
797 
798 		/*
799 		 * Release the hold on the p_exec and p_execdir, these
800 		 * were acquired in getproc()
801 		 */
802 		if (p->p_execdir != NULL)
803 			VN_RELE(p->p_execdir);
804 		if (p->p_exec != NULL)
805 			VN_RELE(p->p_exec);
806 		p->p_flag |= SNOWAIT;
807 		p->p_exec = NULL;
808 		p->p_execdir = NULL;
809 
810 		init_set = rctl_set_create();
811 		init_gp = rctl_set_init_prealloc(RCENTITY_PROCESS);
812 
813 		/*
814 		 * kernel processes do not inherit /proc tracing flags.
815 		 */
816 		sigemptyset(&p->p_sigmask);
817 		premptyset(&p->p_fltmask);
818 		up = PTOU(p);
819 		up->u_systrap = 0;
820 		premptyset(&(up->u_entrymask));
821 		premptyset(&(up->u_exitmask));
822 		mutex_enter(&p->p_lock);
823 		e.rcep_p.proc = p;
824 		e.rcep_t = RCENTITY_PROCESS;
825 		p->p_rctls = rctl_set_init(RCENTITY_PROCESS, p, &e, init_set,
826 		    init_gp);
827 		mutex_exit(&p->p_lock);
828 
829 		rctl_prealloc_destroy(init_gp);
830 	} else  {
831 		rctl_alloc_gp_t *init_gp, *default_gp;
832 		rctl_set_t *init_set;
833 		task_t *tk, *tk_old;
834 
835 		if (getproc(&p, 0) < 0)
836 			return (EAGAIN);
837 		/*
838 		 * init creates a new task, distinct from the task
839 		 * containing kernel "processes".
840 		 */
841 		tk = task_create(0, p->p_zone);
842 		mutex_enter(&tk->tk_zone->zone_nlwps_lock);
843 		tk->tk_proj->kpj_ntasks++;
844 		mutex_exit(&tk->tk_zone->zone_nlwps_lock);
845 
846 		default_gp = rctl_rlimit_set_prealloc(RLIM_NLIMITS);
847 		init_gp = rctl_set_init_prealloc(RCENTITY_PROCESS);
848 		init_set = rctl_set_create();
849 
850 		mutex_enter(&pidlock);
851 		mutex_enter(&p->p_lock);
852 		tk_old = p->p_task;	/* switch to new task */
853 
854 		task_detach(p);
855 		task_begin(tk, p);
856 		mutex_exit(&pidlock);
857 
858 		e.rcep_p.proc = p;
859 		e.rcep_t = RCENTITY_PROCESS;
860 		p->p_rctls = rctl_set_init(RCENTITY_PROCESS, p, &e, init_set,
861 		    init_gp);
862 		rctlproc_default_init(p, default_gp);
863 		mutex_exit(&p->p_lock);
864 
865 		task_rele(tk_old);
866 		rctl_prealloc_destroy(default_gp);
867 		rctl_prealloc_destroy(init_gp);
868 	}
869 
870 	p->p_as = &kas;
871 
872 	if ((lwp = lwp_create(pc, arg, 0, p, TS_STOPPED, pri,
873 	    &curthread->t_hold, cid, 1)) == NULL) {
874 		task_t *tk;
875 		fork_fail(p);
876 		mutex_enter(&pidlock);
877 		mutex_enter(&p->p_lock);
878 		tk = p->p_task;
879 		task_detach(p);
880 		ASSERT(p->p_pool->pool_ref > 0);
881 		atomic_add_32(&p->p_pool->pool_ref, -1);
882 		mutex_exit(&p->p_lock);
883 		pid_exit(p);
884 		mutex_exit(&pidlock);
885 		task_rele(tk);
886 
887 		return (EAGAIN);
888 	}
889 
890 	if (cid != syscid) {
891 		ctp = contract_process_fork(sys_process_tmpl, p, curproc,
892 		    B_FALSE);
893 		ASSERT(ctp != NULL);
894 		if (ct != NULL)
895 			*ct = &ctp->conp_contract;
896 	}
897 
898 	p->p_lwpid = 1;
899 	mutex_enter(&pidlock);
900 	pgjoin(p, curproc->p_pgidp);
901 	p->p_stat = SRUN;
902 	mutex_enter(&p->p_lock);
903 	lwptot(lwp)->t_proc_flag &= ~TP_HOLDLWP;
904 	lwp_create_done(lwptot(lwp));
905 	mutex_exit(&p->p_lock);
906 	mutex_exit(&pidlock);
907 	return (0);
908 }
909 
910 /*
911  * create a child proc struct.
912  */
913 static int
914 getproc(proc_t **cpp, int kernel)
915 {
916 	proc_t		*pp, *cp;
917 	pid_t		newpid;
918 	struct user	*uarea;
919 	extern uint_t	nproc;
920 	struct cred	*cr;
921 	uid_t		ruid;
922 	zoneid_t	zoneid;
923 
924 	if (!page_mem_avail(tune.t_minarmem))
925 		return (-1);
926 	if (zone_status_get(curproc->p_zone) >= ZONE_IS_SHUTTING_DOWN)
927 		return (-1);	/* no point in starting new processes */
928 
929 	pp = curproc;
930 	cp = kmem_cache_alloc(process_cache, KM_SLEEP);
931 	bzero(cp, sizeof (proc_t));
932 
933 	/*
934 	 * Make proc entry for child process
935 	 */
936 	mutex_init(&cp->p_splock, NULL, MUTEX_DEFAULT, NULL);
937 	mutex_init(&cp->p_crlock, NULL, MUTEX_DEFAULT, NULL);
938 	mutex_init(&cp->p_pflock, NULL, MUTEX_DEFAULT, NULL);
939 #if defined(__x86)
940 	mutex_init(&cp->p_ldtlock, NULL, MUTEX_DEFAULT, NULL);
941 #endif
942 	mutex_init(&cp->p_maplock, NULL, MUTEX_DEFAULT, NULL);
943 	cp->p_stat = SIDL;
944 	cp->p_mstart = gethrtime();
945 	/*
946 	 * p_zone must be set before we call pid_allocate since the process
947 	 * will be visible after that and code such as prfind_zone will
948 	 * look at the p_zone field.
949 	 */
950 	cp->p_zone = pp->p_zone;
951 	cp->p_t1_lgrpid = LGRP_NONE;
952 	cp->p_tr_lgrpid = LGRP_NONE;
953 
954 	if ((newpid = pid_allocate(cp, PID_ALLOC_PROC)) == -1) {
955 		if (nproc == v.v_proc) {
956 			CPU_STATS_ADDQ(CPU, sys, procovf, 1);
957 			cmn_err(CE_WARN, "out of processes");
958 		}
959 		goto bad;
960 	}
961 
962 	/*
963 	 * If not privileged make sure that this user hasn't exceeded
964 	 * v.v_maxup processes, and that users collectively haven't
965 	 * exceeded v.v_maxupttl processes.
966 	 */
967 	mutex_enter(&pidlock);
968 	ASSERT(nproc < v.v_proc);	/* otherwise how'd we get our pid? */
969 	cr = CRED();
970 	ruid = crgetruid(cr);
971 	zoneid = crgetzoneid(cr);
972 	if (nproc >= v.v_maxup && 	/* short-circuit; usually false */
973 	    (nproc >= v.v_maxupttl ||
974 	    upcount_get(ruid, zoneid) >= v.v_maxup) &&
975 	    secpolicy_newproc(cr) != 0) {
976 		mutex_exit(&pidlock);
977 		zcmn_err(zoneid, CE_NOTE,
978 		    "out of per-user processes for uid %d", ruid);
979 		goto bad;
980 	}
981 
982 	/*
983 	 * Everything is cool, put the new proc on the active process list.
984 	 * It is already on the pid list and in /proc.
985 	 * Increment the per uid process count (upcount).
986 	 */
987 	nproc++;
988 	upcount_inc(ruid, zoneid);
989 
990 	cp->p_next = practive;
991 	practive->p_prev = cp;
992 	practive = cp;
993 
994 	cp->p_ignore = pp->p_ignore;
995 	cp->p_siginfo = pp->p_siginfo;
996 	cp->p_flag = pp->p_flag & (SJCTL|SNOWAIT|SNOCD);
997 	cp->p_sessp = pp->p_sessp;
998 	sess_hold(pp);
999 	cp->p_exec = pp->p_exec;
1000 	cp->p_execdir = pp->p_execdir;
1001 	cp->p_brand = pp->p_brand;
1002 	if (PROC_IS_BRANDED(pp))
1003 		BROP(pp)->b_copy_procdata(cp, pp);
1004 
1005 	cp->p_bssbase = pp->p_bssbase;
1006 	cp->p_brkbase = pp->p_brkbase;
1007 	cp->p_brksize = pp->p_brksize;
1008 	cp->p_brkpageszc = pp->p_brkpageszc;
1009 	cp->p_stksize = pp->p_stksize;
1010 	cp->p_stkpageszc = pp->p_stkpageszc;
1011 	cp->p_stkprot = pp->p_stkprot;
1012 	cp->p_datprot = pp->p_datprot;
1013 	cp->p_usrstack = pp->p_usrstack;
1014 	cp->p_model = pp->p_model;
1015 	cp->p_ppid = pp->p_pid;
1016 	cp->p_ancpid = pp->p_pid;
1017 	cp->p_portcnt = pp->p_portcnt;
1018 
1019 	/*
1020 	 * Initialize watchpoint structures
1021 	 */
1022 	avl_create(&cp->p_warea, wa_compare, sizeof (struct watched_area),
1023 	    offsetof(struct watched_area, wa_link));
1024 
1025 	/*
1026 	 * Initialize immediate resource control values.
1027 	 */
1028 	cp->p_stk_ctl = pp->p_stk_ctl;
1029 	cp->p_fsz_ctl = pp->p_fsz_ctl;
1030 	cp->p_vmem_ctl = pp->p_vmem_ctl;
1031 	cp->p_fno_ctl = pp->p_fno_ctl;
1032 
1033 	/*
1034 	 * Link up to parent-child-sibling chain.  No need to lock
1035 	 * in general since only a call to freeproc() (done by the
1036 	 * same parent as newproc()) diddles with the child chain.
1037 	 */
1038 	cp->p_sibling = pp->p_child;
1039 	if (pp->p_child)
1040 		pp->p_child->p_psibling = cp;
1041 
1042 	cp->p_parent = pp;
1043 	pp->p_child = cp;
1044 
1045 	cp->p_child_ns = NULL;
1046 	cp->p_sibling_ns = NULL;
1047 
1048 	cp->p_nextorph = pp->p_orphan;
1049 	cp->p_nextofkin = pp;
1050 	pp->p_orphan = cp;
1051 
1052 	/*
1053 	 * Inherit profiling state; do not inherit REALPROF profiling state.
1054 	 */
1055 	cp->p_prof = pp->p_prof;
1056 	cp->p_rprof_cyclic = CYCLIC_NONE;
1057 
1058 	/*
1059 	 * Inherit pool pointer from the parent.  Kernel processes are
1060 	 * always bound to the default pool.
1061 	 */
1062 	mutex_enter(&pp->p_lock);
1063 	if (kernel) {
1064 		cp->p_pool = pool_default;
1065 		cp->p_flag |= SSYS;
1066 	} else {
1067 		cp->p_pool = pp->p_pool;
1068 	}
1069 	atomic_add_32(&cp->p_pool->pool_ref, 1);
1070 	mutex_exit(&pp->p_lock);
1071 
1072 	/*
1073 	 * Add the child process to the current task.  Kernel processes
1074 	 * are always attached to task0.
1075 	 */
1076 	mutex_enter(&cp->p_lock);
1077 	if (kernel)
1078 		task_attach(task0p, cp);
1079 	else
1080 		task_attach(pp->p_task, cp);
1081 	mutex_exit(&cp->p_lock);
1082 	mutex_exit(&pidlock);
1083 
1084 	avl_create(&cp->p_ct_held, contract_compar, sizeof (contract_t),
1085 	    offsetof(contract_t, ct_ctlist));
1086 
1087 	/*
1088 	 * Duplicate any audit information kept in the process table
1089 	 */
1090 	if (audit_active)	/* copy audit data to cp */
1091 		audit_newproc(cp);
1092 
1093 	crhold(cp->p_cred = cr);
1094 
1095 	/*
1096 	 * Bump up the counts on the file structures pointed at by the
1097 	 * parent's file table since the child will point at them too.
1098 	 */
1099 	fcnt_add(P_FINFO(pp), 1);
1100 
1101 	VN_HOLD(PTOU(pp)->u_cdir);
1102 	if (PTOU(pp)->u_rdir)
1103 		VN_HOLD(PTOU(pp)->u_rdir);
1104 	if (PTOU(pp)->u_cwd)
1105 		refstr_hold(PTOU(pp)->u_cwd);
1106 
1107 	/*
1108 	 * copy the parent's uarea.
1109 	 */
1110 	uarea = PTOU(cp);
1111 	bcopy(PTOU(pp), uarea, sizeof (*uarea));
1112 	flist_fork(P_FINFO(pp), P_FINFO(cp));
1113 
1114 	gethrestime(&uarea->u_start);
1115 	uarea->u_ticks = lbolt;
1116 	uarea->u_mem = rm_asrss(pp->p_as);
1117 	uarea->u_acflag = AFORK;
1118 
1119 	/*
1120 	 * If inherit-on-fork, copy /proc tracing flags to child.
1121 	 */
1122 	if ((pp->p_proc_flag & P_PR_FORK) != 0) {
1123 		cp->p_proc_flag |= pp->p_proc_flag & (P_PR_TRACE|P_PR_FORK);
1124 		cp->p_sigmask = pp->p_sigmask;
1125 		cp->p_fltmask = pp->p_fltmask;
1126 	} else {
1127 		sigemptyset(&cp->p_sigmask);
1128 		premptyset(&cp->p_fltmask);
1129 		uarea->u_systrap = 0;
1130 		premptyset(&uarea->u_entrymask);
1131 		premptyset(&uarea->u_exitmask);
1132 	}
1133 	/*
1134 	 * If microstate accounting is being inherited, mark child
1135 	 */
1136 	if ((pp->p_flag & SMSFORK) != 0)
1137 		cp->p_flag |= pp->p_flag & (SMSFORK|SMSACCT);
1138 
1139 	/*
1140 	 * Inherit fixalignment flag from the parent
1141 	 */
1142 	cp->p_fixalignment = pp->p_fixalignment;
1143 
1144 	if (cp->p_exec)
1145 		VN_HOLD(cp->p_exec);
1146 	if (cp->p_execdir)
1147 		VN_HOLD(cp->p_execdir);
1148 	*cpp = cp;
1149 	return (0);
1150 
1151 bad:
1152 	ASSERT(MUTEX_NOT_HELD(&pidlock));
1153 
1154 	mutex_destroy(&cp->p_crlock);
1155 	mutex_destroy(&cp->p_pflock);
1156 #if defined(__x86)
1157 	mutex_destroy(&cp->p_ldtlock);
1158 #endif
1159 	if (newpid != -1) {
1160 		proc_entry_free(cp->p_pidp);
1161 		(void) pid_rele(cp->p_pidp);
1162 	}
1163 	kmem_cache_free(process_cache, cp);
1164 
1165 	/*
1166 	 * We most likely got into this situation because some process is
1167 	 * forking out of control.  As punishment, put it to sleep for a
1168 	 * bit so it can't eat the machine alive.  Sleep interval is chosen
1169 	 * to allow no more than one fork failure per cpu per clock tick
1170 	 * on average (yes, I just made this up).  This has two desirable
1171 	 * properties: (1) it sets a constant limit on the fork failure
1172 	 * rate, and (2) the busier the system is, the harsher the penalty
1173 	 * for abusing it becomes.
1174 	 */
1175 	INCR_COUNT(&fork_fail_pending, &pidlock);
1176 	delay(fork_fail_pending / ncpus + 1);
1177 	DECR_COUNT(&fork_fail_pending, &pidlock);
1178 
1179 	return (-1); /* out of memory or proc slots */
1180 }
1181 
1182 /*
1183  * Release virtual memory.
1184  * In the case of vfork(), the child was given exclusive access to its
1185  * parent's address space.  The parent is waiting in vfwait() for the
1186  * child to release its exclusive claim via relvm().
1187  */
1188 void
1189 relvm()
1190 {
1191 	proc_t *p = curproc;
1192 
1193 	ASSERT((unsigned)p->p_lwpcnt <= 1);
1194 
1195 	prrelvm();	/* inform /proc */
1196 
1197 	if (p->p_flag & SVFORK) {
1198 		proc_t *pp = p->p_parent;
1199 		/*
1200 		 * The child process is either exec'ing or exit'ing.
1201 		 * The child is now separated from the parent's address
1202 		 * space.  The parent process is made dispatchable.
1203 		 *
1204 		 * This is a delicate locking maneuver, involving
1205 		 * both the parent's p_lock and the child's p_lock.
1206 		 * As soon as the SVFORK flag is turned off, the
1207 		 * parent is free to run, but it must not run until
1208 		 * we wake it up using its p_cv because it might
1209 		 * exit and we would be referencing invalid memory.
1210 		 * Therefore, we hold the parent with its p_lock
1211 		 * while protecting our p_flags with our own p_lock.
1212 		 */
1213 try_again:
1214 		mutex_enter(&p->p_lock);	/* grab child's lock first */
1215 		prbarrier(p);		/* make sure /proc is blocked out */
1216 		mutex_enter(&pp->p_lock);
1217 
1218 		/*
1219 		 * Check if parent is locked by /proc.
1220 		 */
1221 		if (pp->p_proc_flag & P_PR_LOCK) {
1222 			/*
1223 			 * Delay until /proc is done with the parent.
1224 			 * We must drop our (the child's) p->p_lock, wait
1225 			 * via prbarrier() on the parent, then start over.
1226 			 */
1227 			mutex_exit(&p->p_lock);
1228 			prbarrier(pp);
1229 			mutex_exit(&pp->p_lock);
1230 			goto try_again;
1231 		}
1232 		p->p_flag &= ~SVFORK;
1233 		kpreempt_disable();
1234 		p->p_as = &kas;
1235 
1236 		/*
1237 		 * notify hat of change in thread's address space
1238 		 */
1239 		hat_thread_exit(curthread);
1240 		kpreempt_enable();
1241 
1242 		/*
1243 		 * child sizes are copied back to parent because
1244 		 * child may have grown.
1245 		 */
1246 		pp->p_brkbase = p->p_brkbase;
1247 		pp->p_brksize = p->p_brksize;
1248 		pp->p_stksize = p->p_stksize;
1249 		/*
1250 		 * The parent is no longer waiting for the vfork()d child.
1251 		 * Restore the parent's watched pages, if any.  This is
1252 		 * safe because we know the parent is not locked by /proc
1253 		 */
1254 		pp->p_flag &= ~SVFWAIT;
1255 		if (avl_numnodes(&pp->p_wpage) != 0) {
1256 			pp->p_as->a_wpage = pp->p_wpage;
1257 			avl_create(&pp->p_wpage, wp_compare,
1258 			    sizeof (struct watched_page),
1259 			    offsetof(struct watched_page, wp_link));
1260 		}
1261 		cv_signal(&pp->p_cv);
1262 		mutex_exit(&pp->p_lock);
1263 		mutex_exit(&p->p_lock);
1264 	} else {
1265 		if (p->p_as != &kas) {
1266 			struct as *as;
1267 
1268 			if (p->p_segacct)
1269 				shmexit(p);
1270 
1271 			/*
1272 			 * We grab p_lock for the benefit of /proc
1273 			 */
1274 			kpreempt_disable();
1275 			mutex_enter(&p->p_lock);
1276 			prbarrier(p);	/* make sure /proc is blocked out */
1277 			as = p->p_as;
1278 			p->p_as = &kas;
1279 			mutex_exit(&p->p_lock);
1280 
1281 			/*
1282 			 * notify hat of change in thread's address space
1283 			 */
1284 			hat_thread_exit(curthread);
1285 			kpreempt_enable();
1286 
1287 			as_free(as);
1288 			p->p_tr_lgrpid = LGRP_NONE;
1289 		}
1290 	}
1291 }
1292 
1293 /*
1294  * Wait for child to exec or exit.
1295  * Called by parent of vfork'ed process.
1296  * See important comments in relvm(), above.
1297  */
1298 void
1299 vfwait(pid_t pid)
1300 {
1301 	int signalled = 0;
1302 	proc_t *pp = ttoproc(curthread);
1303 	proc_t *cp;
1304 
1305 	/*
1306 	 * Wait for child to exec or exit.
1307 	 */
1308 	for (;;) {
1309 		mutex_enter(&pidlock);
1310 		cp = prfind(pid);
1311 		if (cp == NULL || cp->p_parent != pp) {
1312 			/*
1313 			 * Child has exit()ed.
1314 			 */
1315 			mutex_exit(&pidlock);
1316 			break;
1317 		}
1318 		/*
1319 		 * Grab the child's p_lock before releasing pidlock.
1320 		 * Otherwise, the child could exit and we would be
1321 		 * referencing invalid memory.
1322 		 */
1323 		mutex_enter(&cp->p_lock);
1324 		mutex_exit(&pidlock);
1325 		if (!(cp->p_flag & SVFORK)) {
1326 			/*
1327 			 * Child has exec()ed or is exit()ing.
1328 			 */
1329 			mutex_exit(&cp->p_lock);
1330 			break;
1331 		}
1332 		mutex_enter(&pp->p_lock);
1333 		mutex_exit(&cp->p_lock);
1334 		/*
1335 		 * We might be waked up spuriously from the cv_wait().
1336 		 * We have to do the whole operation over again to be
1337 		 * sure the child's SVFORK flag really is turned off.
1338 		 * We cannot make reference to the child because it can
1339 		 * exit before we return and we would be referencing
1340 		 * invalid memory.
1341 		 *
1342 		 * Because this is potentially a very long-term wait,
1343 		 * we call cv_wait_sig() (for its jobcontrol and /proc
1344 		 * side-effects) unless there is a current signal, in
1345 		 * which case we use cv_wait() because we cannot return
1346 		 * from this function until the child has released the
1347 		 * address space.  Calling cv_wait_sig() with a current
1348 		 * signal would lead to an indefinite loop here because
1349 		 * cv_wait_sig() returns immediately in this case.
1350 		 */
1351 		if (signalled)
1352 			cv_wait(&pp->p_cv, &pp->p_lock);
1353 		else
1354 			signalled = !cv_wait_sig(&pp->p_cv, &pp->p_lock);
1355 		mutex_exit(&pp->p_lock);
1356 	}
1357 
1358 	/* restore watchpoints to parent */
1359 	if (pr_watch_active(pp)) {
1360 		struct as *as = pp->p_as;
1361 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1362 		as_setwatch(as);
1363 		AS_LOCK_EXIT(as, &as->a_lock);
1364 	}
1365 
1366 	mutex_enter(&pp->p_lock);
1367 	prbarrier(pp);	/* barrier against /proc locking */
1368 	continuelwps(pp);
1369 	mutex_exit(&pp->p_lock);
1370 }
1371