xref: /illumos-gate/usr/src/uts/common/os/fork.c (revision 8fd04b8338ed5093ec2d1e668fa620b7de44c177)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
28 /*	  All Rights Reserved  	*/
29 
30 #include <sys/types.h>
31 #include <sys/param.h>
32 #include <sys/sysmacros.h>
33 #include <sys/signal.h>
34 #include <sys/cred.h>
35 #include <sys/policy.h>
36 #include <sys/user.h>
37 #include <sys/systm.h>
38 #include <sys/cpuvar.h>
39 #include <sys/vfs.h>
40 #include <sys/vnode.h>
41 #include <sys/file.h>
42 #include <sys/errno.h>
43 #include <sys/time.h>
44 #include <sys/proc.h>
45 #include <sys/cmn_err.h>
46 #include <sys/acct.h>
47 #include <sys/tuneable.h>
48 #include <sys/class.h>
49 #include <sys/kmem.h>
50 #include <sys/session.h>
51 #include <sys/ucontext.h>
52 #include <sys/stack.h>
53 #include <sys/procfs.h>
54 #include <sys/prsystm.h>
55 #include <sys/vmsystm.h>
56 #include <sys/vtrace.h>
57 #include <sys/debug.h>
58 #include <sys/shm_impl.h>
59 #include <sys/door_data.h>
60 #include <vm/as.h>
61 #include <vm/rm.h>
62 #include <c2/audit.h>
63 #include <sys/var.h>
64 #include <sys/schedctl.h>
65 #include <sys/utrap.h>
66 #include <sys/task.h>
67 #include <sys/resource.h>
68 #include <sys/cyclic.h>
69 #include <sys/lgrp.h>
70 #include <sys/rctl.h>
71 #include <sys/contract_impl.h>
72 #include <sys/contract/process_impl.h>
73 #include <sys/list.h>
74 #include <sys/dtrace.h>
75 #include <sys/pool.h>
76 #include <sys/zone.h>
77 #include <sys/sdt.h>
78 #include <sys/class.h>
79 #include <sys/corectl.h>
80 #include <sys/brand.h>
81 #include <sys/fork.h>
82 
83 static int64_t cfork(int, int, int);
84 static int getproc(proc_t **, pid_t, uint_t);
85 #define	GETPROC_USER	0x0
86 #define	GETPROC_KERNEL	0x1
87 
88 static void fork_fail(proc_t *);
89 static void forklwp_fail(proc_t *);
90 
91 int fork_fail_pending;
92 
93 extern struct kmem_cache *process_cache;
94 
95 /*
96  * The vfork() system call trap is no longer invoked by libc.
97  * It is retained only for the benefit of applications running
98  * within a solaris10 branded zone.  It should be eliminated
99  * when we no longer support solaris10 branded zones.
100  */
101 int64_t
102 vfork(void)
103 {
104 	curthread->t_post_sys = 1;	/* so vfwait() will be called */
105 	return (cfork(1, 1, 0));
106 }
107 
108 /*
109  * forksys system call - forkx, forkallx, vforkx.  This is the
110  * interface invoked by libc for fork1(), forkall(), and vfork()
111  */
112 int64_t
113 forksys(int subcode, int flags)
114 {
115 	switch (subcode) {
116 	case 0:
117 		return (cfork(0, 1, flags));	/* forkx(flags) */
118 	case 1:
119 		return (cfork(0, 0, flags));	/* forkallx(flags) */
120 	case 2:
121 		curthread->t_post_sys = 1;	/* so vfwait() will be called */
122 		return (cfork(1, 1, flags));	/* vforkx(flags) */
123 	default:
124 		return ((int64_t)set_errno(EINVAL));
125 	}
126 }
127 
128 /* ARGSUSED */
129 static int64_t
130 cfork(int isvfork, int isfork1, int flags)
131 {
132 	proc_t *p = ttoproc(curthread);
133 	struct as *as;
134 	proc_t *cp, **orphpp;
135 	klwp_t *clone;
136 	kthread_t *t;
137 	task_t *tk;
138 	rval_t	r;
139 	int error;
140 	int i;
141 	rctl_set_t *dup_set;
142 	rctl_alloc_gp_t *dup_gp;
143 	rctl_entity_p_t e;
144 	lwpdir_t *ldp;
145 	lwpent_t *lep;
146 	lwpent_t *clep;
147 
148 	/*
149 	 * Allow only these two flags.
150 	 */
151 	if ((flags & ~(FORK_NOSIGCHLD | FORK_WAITPID)) != 0) {
152 		error = EINVAL;
153 		goto forkerr;
154 	}
155 
156 	/*
157 	 * fork is not supported for the /proc agent lwp.
158 	 */
159 	if (curthread == p->p_agenttp) {
160 		error = ENOTSUP;
161 		goto forkerr;
162 	}
163 
164 	if ((error = secpolicy_basic_fork(CRED())) != 0)
165 		goto forkerr;
166 
167 	/*
168 	 * If the calling lwp is doing a fork1() then the
169 	 * other lwps in this process are not duplicated and
170 	 * don't need to be held where their kernel stacks can be
171 	 * cloned.  If doing forkall(), the process is held with
172 	 * SHOLDFORK, so that the lwps are at a point where their
173 	 * stacks can be copied which is on entry or exit from
174 	 * the kernel.
175 	 */
176 	if (!holdlwps(isfork1 ? SHOLDFORK1 : SHOLDFORK)) {
177 		aston(curthread);
178 		error = EINTR;
179 		goto forkerr;
180 	}
181 
182 #if defined(__sparc)
183 	/*
184 	 * Ensure that the user stack is fully constructed
185 	 * before creating the child process structure.
186 	 */
187 	(void) flush_user_windows_to_stack(NULL);
188 #endif
189 
190 	mutex_enter(&p->p_lock);
191 	/*
192 	 * If this is vfork(), cancel any suspend request we might
193 	 * have gotten from some other thread via lwp_suspend().
194 	 * Otherwise we could end up with a deadlock on return
195 	 * from the vfork() in both the parent and the child.
196 	 */
197 	if (isvfork)
198 		curthread->t_proc_flag &= ~TP_HOLDLWP;
199 	/*
200 	 * Prevent our resource set associations from being changed during fork.
201 	 */
202 	pool_barrier_enter();
203 	mutex_exit(&p->p_lock);
204 
205 	/*
206 	 * Create a child proc struct. Place a VN_HOLD on appropriate vnodes.
207 	 */
208 	if (getproc(&cp, 0, GETPROC_USER) < 0) {
209 		mutex_enter(&p->p_lock);
210 		pool_barrier_exit();
211 		continuelwps(p);
212 		mutex_exit(&p->p_lock);
213 		error = EAGAIN;
214 		goto forkerr;
215 	}
216 
217 	TRACE_2(TR_FAC_PROC, TR_PROC_FORK, "proc_fork:cp %p p %p", cp, p);
218 
219 	/*
220 	 * Assign an address space to child
221 	 */
222 	if (isvfork) {
223 		/*
224 		 * Clear any watched areas and remember the
225 		 * watched pages for restoring in vfwait().
226 		 */
227 		as = p->p_as;
228 		if (avl_numnodes(&as->a_wpage) != 0) {
229 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
230 			as_clearwatch(as);
231 			p->p_wpage = as->a_wpage;
232 			avl_create(&as->a_wpage, wp_compare,
233 			    sizeof (struct watched_page),
234 			    offsetof(struct watched_page, wp_link));
235 			AS_LOCK_EXIT(as, &as->a_lock);
236 		}
237 		cp->p_as = as;
238 		cp->p_flag |= SVFORK;
239 
240 		/*
241 		 * Use the parent's shm segment list information for
242 		 * the child as it uses its address space till it execs.
243 		 */
244 		cp->p_segacct = p->p_segacct;
245 	} else {
246 		/*
247 		 * We need to hold P_PR_LOCK until the address space has
248 		 * been duplicated and we've had a chance to remove from the
249 		 * child any DTrace probes that were in the parent. Holding
250 		 * P_PR_LOCK prevents any new probes from being added and any
251 		 * extant probes from being removed.
252 		 */
253 		mutex_enter(&p->p_lock);
254 		sprlock_proc(p);
255 		p->p_flag |= SFORKING;
256 		mutex_exit(&p->p_lock);
257 
258 		error = as_dup(p->p_as, cp);
259 		if (error != 0) {
260 			mutex_enter(&p->p_lock);
261 			sprunlock(p);
262 			fork_fail(cp);
263 			mutex_enter(&pidlock);
264 			orphpp = &p->p_orphan;
265 			while (*orphpp != cp)
266 				orphpp = &(*orphpp)->p_nextorph;
267 			*orphpp = cp->p_nextorph;
268 			if (p->p_child == cp)
269 				p->p_child = cp->p_sibling;
270 			if (cp->p_sibling)
271 				cp->p_sibling->p_psibling = cp->p_psibling;
272 			if (cp->p_psibling)
273 				cp->p_psibling->p_sibling = cp->p_sibling;
274 			mutex_enter(&cp->p_lock);
275 			tk = cp->p_task;
276 			task_detach(cp);
277 			ASSERT(cp->p_pool->pool_ref > 0);
278 			atomic_add_32(&cp->p_pool->pool_ref, -1);
279 			mutex_exit(&cp->p_lock);
280 			pid_exit(cp);
281 			mutex_exit(&pidlock);
282 			task_rele(tk);
283 
284 			mutex_enter(&p->p_lock);
285 			p->p_flag &= ~SFORKING;
286 			pool_barrier_exit();
287 			continuelwps(p);
288 			mutex_exit(&p->p_lock);
289 			/*
290 			 * Preserve ENOMEM error condition but
291 			 * map all others to EAGAIN.
292 			 */
293 			error = (error == ENOMEM) ? ENOMEM : EAGAIN;
294 			goto forkerr;
295 		}
296 
297 		/*
298 		 * Remove all DTrace tracepoints from the child process. We
299 		 * need to do this _before_ duplicating USDT providers since
300 		 * any associated probes may be immediately enabled.
301 		 */
302 		if (p->p_dtrace_count > 0)
303 			dtrace_fasttrap_fork(p, cp);
304 
305 		mutex_enter(&p->p_lock);
306 		sprunlock(p);
307 
308 		/* Duplicate parent's shared memory */
309 		if (p->p_segacct)
310 			shmfork(p, cp);
311 
312 		/*
313 		 * Duplicate any helper actions and providers. The SFORKING
314 		 * we set above informs the code to enable USDT probes that
315 		 * sprlock() may fail because the child is being forked.
316 		 */
317 		if (p->p_dtrace_helpers != NULL) {
318 			ASSERT(dtrace_helpers_fork != NULL);
319 			(*dtrace_helpers_fork)(p, cp);
320 		}
321 
322 		mutex_enter(&p->p_lock);
323 		p->p_flag &= ~SFORKING;
324 		mutex_exit(&p->p_lock);
325 	}
326 
327 	/*
328 	 * Duplicate parent's resource controls.
329 	 */
330 	dup_set = rctl_set_create();
331 	for (;;) {
332 		dup_gp = rctl_set_dup_prealloc(p->p_rctls);
333 		mutex_enter(&p->p_rctls->rcs_lock);
334 		if (rctl_set_dup_ready(p->p_rctls, dup_gp))
335 			break;
336 		mutex_exit(&p->p_rctls->rcs_lock);
337 		rctl_prealloc_destroy(dup_gp);
338 	}
339 	e.rcep_p.proc = cp;
340 	e.rcep_t = RCENTITY_PROCESS;
341 	cp->p_rctls = rctl_set_dup(p->p_rctls, p, cp, &e, dup_set, dup_gp,
342 	    RCD_DUP | RCD_CALLBACK);
343 	mutex_exit(&p->p_rctls->rcs_lock);
344 
345 	rctl_prealloc_destroy(dup_gp);
346 
347 	/*
348 	 * Allocate the child's lwp directory and lwpid hash table.
349 	 */
350 	if (isfork1)
351 		cp->p_lwpdir_sz = 2;
352 	else
353 		cp->p_lwpdir_sz = p->p_lwpdir_sz;
354 	cp->p_lwpdir = cp->p_lwpfree = ldp =
355 	    kmem_zalloc(cp->p_lwpdir_sz * sizeof (lwpdir_t), KM_SLEEP);
356 	for (i = 1; i < cp->p_lwpdir_sz; i++, ldp++)
357 		ldp->ld_next = ldp + 1;
358 	cp->p_tidhash_sz = (cp->p_lwpdir_sz + 2) / 2;
359 	cp->p_tidhash =
360 	    kmem_zalloc(cp->p_tidhash_sz * sizeof (tidhash_t), KM_SLEEP);
361 
362 	/*
363 	 * Duplicate parent's lwps.
364 	 * Mutual exclusion is not needed because the process is
365 	 * in the hold state and only the current lwp is running.
366 	 */
367 	klgrpset_clear(cp->p_lgrpset);
368 	if (isfork1) {
369 		clone = forklwp(ttolwp(curthread), cp, curthread->t_tid);
370 		if (clone == NULL)
371 			goto forklwperr;
372 		/*
373 		 * Inherit only the lwp_wait()able flag,
374 		 * Daemon threads should not call fork1(), but oh well...
375 		 */
376 		lwptot(clone)->t_proc_flag |=
377 		    (curthread->t_proc_flag & TP_TWAIT);
378 	} else {
379 		/* this is forkall(), no one can be in lwp_wait() */
380 		ASSERT(p->p_lwpwait == 0 && p->p_lwpdwait == 0);
381 		/* for each entry in the parent's lwp directory... */
382 		for (i = 0, ldp = p->p_lwpdir; i < p->p_lwpdir_sz; i++, ldp++) {
383 			klwp_t *clwp;
384 			kthread_t *ct;
385 
386 			if ((lep = ldp->ld_entry) == NULL)
387 				continue;
388 
389 			if ((t = lep->le_thread) != NULL) {
390 				clwp = forklwp(ttolwp(t), cp, t->t_tid);
391 				if (clwp == NULL)
392 					goto forklwperr;
393 				ct = lwptot(clwp);
394 				/*
395 				 * Inherit lwp_wait()able and daemon flags.
396 				 */
397 				ct->t_proc_flag |=
398 				    (t->t_proc_flag & (TP_TWAIT|TP_DAEMON));
399 				/*
400 				 * Keep track of the clone of curthread to
401 				 * post return values through lwp_setrval().
402 				 * Mark other threads for special treatment
403 				 * by lwp_rtt() / post_syscall().
404 				 */
405 				if (t == curthread)
406 					clone = clwp;
407 				else
408 					ct->t_flag |= T_FORKALL;
409 			} else {
410 				/*
411 				 * Replicate zombie lwps in the child.
412 				 */
413 				clep = kmem_zalloc(sizeof (*clep), KM_SLEEP);
414 				clep->le_lwpid = lep->le_lwpid;
415 				clep->le_start = lep->le_start;
416 				lwp_hash_in(cp, clep,
417 				    cp->p_tidhash, cp->p_tidhash_sz, 0);
418 			}
419 		}
420 	}
421 
422 	/*
423 	 * Put new process in the parent's process contract, or put it
424 	 * in a new one if there is an active process template.  Send a
425 	 * fork event (if requested) to whatever contract the child is
426 	 * a member of.  Fails if the parent has been SIGKILLed.
427 	 */
428 	if (contract_process_fork(NULL, cp, p, B_TRUE) == NULL)
429 		goto forklwperr;
430 
431 	/*
432 	 * No fork failures occur beyond this point.
433 	 */
434 
435 	cp->p_lwpid = p->p_lwpid;
436 	if (!isfork1) {
437 		cp->p_lwpdaemon = p->p_lwpdaemon;
438 		cp->p_zombcnt = p->p_zombcnt;
439 		/*
440 		 * If the parent's lwp ids have wrapped around, so have the
441 		 * child's.
442 		 */
443 		cp->p_flag |= p->p_flag & SLWPWRAP;
444 	}
445 
446 	mutex_enter(&p->p_lock);
447 	corectl_path_hold(cp->p_corefile = p->p_corefile);
448 	corectl_content_hold(cp->p_content = p->p_content);
449 	mutex_exit(&p->p_lock);
450 
451 	/*
452 	 * Duplicate process context ops, if any.
453 	 */
454 	if (p->p_pctx)
455 		forkpctx(p, cp);
456 
457 #ifdef __sparc
458 	utrap_dup(p, cp);
459 #endif
460 	/*
461 	 * If the child process has been marked to stop on exit
462 	 * from this fork, arrange for all other lwps to stop in
463 	 * sympathy with the active lwp.
464 	 */
465 	if (PTOU(cp)->u_systrap &&
466 	    prismember(&PTOU(cp)->u_exitmask, curthread->t_sysnum)) {
467 		mutex_enter(&cp->p_lock);
468 		t = cp->p_tlist;
469 		do {
470 			t->t_proc_flag |= TP_PRSTOP;
471 			aston(t);	/* so TP_PRSTOP will be seen */
472 		} while ((t = t->t_forw) != cp->p_tlist);
473 		mutex_exit(&cp->p_lock);
474 	}
475 	/*
476 	 * If the parent process has been marked to stop on exit
477 	 * from this fork, and its asynchronous-stop flag has not
478 	 * been set, arrange for all other lwps to stop before
479 	 * they return back to user level.
480 	 */
481 	if (!(p->p_proc_flag & P_PR_ASYNC) && PTOU(p)->u_systrap &&
482 	    prismember(&PTOU(p)->u_exitmask, curthread->t_sysnum)) {
483 		mutex_enter(&p->p_lock);
484 		t = p->p_tlist;
485 		do {
486 			t->t_proc_flag |= TP_PRSTOP;
487 			aston(t);	/* so TP_PRSTOP will be seen */
488 		} while ((t = t->t_forw) != p->p_tlist);
489 		mutex_exit(&p->p_lock);
490 	}
491 
492 	if (PROC_IS_BRANDED(p))
493 		BROP(p)->b_lwp_setrval(clone, p->p_pid, 1);
494 	else
495 		lwp_setrval(clone, p->p_pid, 1);
496 
497 	/* set return values for parent */
498 	r.r_val1 = (int)cp->p_pid;
499 	r.r_val2 = 0;
500 
501 	/*
502 	 * pool_barrier_exit() can now be called because the child process has:
503 	 * - all identifying features cloned or set (p_pid, p_task, p_pool)
504 	 * - all resource sets associated (p_tlist->*->t_cpupart, p_as->a_mset)
505 	 * - any other fields set which are used in resource set binding.
506 	 */
507 	mutex_enter(&p->p_lock);
508 	pool_barrier_exit();
509 	mutex_exit(&p->p_lock);
510 
511 	mutex_enter(&pidlock);
512 	mutex_enter(&cp->p_lock);
513 
514 	/*
515 	 * Set flags telling the child what (not) to do on exit.
516 	 */
517 	if (flags & FORK_NOSIGCHLD)
518 		cp->p_pidflag |= CLDNOSIGCHLD;
519 	if (flags & FORK_WAITPID)
520 		cp->p_pidflag |= CLDWAITPID;
521 
522 	/*
523 	 * Now that there are lwps and threads attached, add the new
524 	 * process to the process group.
525 	 */
526 	pgjoin(cp, p->p_pgidp);
527 	cp->p_stat = SRUN;
528 	/*
529 	 * We are now done with all the lwps in the child process.
530 	 */
531 	t = cp->p_tlist;
532 	do {
533 		/*
534 		 * Set the lwp_suspend()ed lwps running.
535 		 * They will suspend properly at syscall exit.
536 		 */
537 		if (t->t_proc_flag & TP_HOLDLWP)
538 			lwp_create_done(t);
539 		else {
540 			/* set TS_CREATE to allow continuelwps() to work */
541 			thread_lock(t);
542 			ASSERT(t->t_state == TS_STOPPED &&
543 			    !(t->t_schedflag & (TS_CREATE|TS_CSTART)));
544 			t->t_schedflag |= TS_CREATE;
545 			thread_unlock(t);
546 		}
547 	} while ((t = t->t_forw) != cp->p_tlist);
548 	mutex_exit(&cp->p_lock);
549 
550 	if (isvfork) {
551 		CPU_STATS_ADDQ(CPU, sys, sysvfork, 1);
552 		mutex_enter(&p->p_lock);
553 		p->p_flag |= SVFWAIT;
554 		curthread->t_flag |= T_VFPARENT;
555 		DTRACE_PROC1(create, proc_t *, cp);
556 		cv_broadcast(&pr_pid_cv[p->p_slot]);	/* inform /proc */
557 		mutex_exit(&p->p_lock);
558 		/*
559 		 * Grab child's p_lock before dropping pidlock to ensure
560 		 * the process will not disappear before we set it running.
561 		 */
562 		mutex_enter(&cp->p_lock);
563 		mutex_exit(&pidlock);
564 		sigdefault(cp);
565 		continuelwps(cp);
566 		mutex_exit(&cp->p_lock);
567 	} else {
568 		CPU_STATS_ADDQ(CPU, sys, sysfork, 1);
569 		DTRACE_PROC1(create, proc_t *, cp);
570 		/*
571 		 * It is CL_FORKRET's job to drop pidlock.
572 		 * If we do it here, the process could be set running
573 		 * and disappear before CL_FORKRET() is called.
574 		 */
575 		CL_FORKRET(curthread, cp->p_tlist);
576 		schedctl_set_cidpri(curthread);
577 		ASSERT(MUTEX_NOT_HELD(&pidlock));
578 	}
579 
580 	return (r.r_vals);
581 
582 forklwperr:
583 	if (isvfork) {
584 		if (avl_numnodes(&p->p_wpage) != 0) {
585 			/* restore watchpoints to parent */
586 			as = p->p_as;
587 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
588 			as->a_wpage = p->p_wpage;
589 			avl_create(&p->p_wpage, wp_compare,
590 			    sizeof (struct watched_page),
591 			    offsetof(struct watched_page, wp_link));
592 			as_setwatch(as);
593 			AS_LOCK_EXIT(as, &as->a_lock);
594 		}
595 	} else {
596 		if (cp->p_segacct)
597 			shmexit(cp);
598 		as = cp->p_as;
599 		cp->p_as = &kas;
600 		as_free(as);
601 	}
602 
603 	if (cp->p_lwpdir) {
604 		for (i = 0, ldp = cp->p_lwpdir; i < cp->p_lwpdir_sz; i++, ldp++)
605 			if ((lep = ldp->ld_entry) != NULL)
606 				kmem_free(lep, sizeof (*lep));
607 		kmem_free(cp->p_lwpdir,
608 		    cp->p_lwpdir_sz * sizeof (*cp->p_lwpdir));
609 	}
610 	cp->p_lwpdir = NULL;
611 	cp->p_lwpfree = NULL;
612 	cp->p_lwpdir_sz = 0;
613 
614 	if (cp->p_tidhash)
615 		kmem_free(cp->p_tidhash,
616 		    cp->p_tidhash_sz * sizeof (*cp->p_tidhash));
617 	cp->p_tidhash = NULL;
618 	cp->p_tidhash_sz = 0;
619 
620 	forklwp_fail(cp);
621 	fork_fail(cp);
622 	rctl_set_free(cp->p_rctls);
623 	mutex_enter(&pidlock);
624 
625 	/*
626 	 * Detach failed child from task.
627 	 */
628 	mutex_enter(&cp->p_lock);
629 	tk = cp->p_task;
630 	task_detach(cp);
631 	ASSERT(cp->p_pool->pool_ref > 0);
632 	atomic_add_32(&cp->p_pool->pool_ref, -1);
633 	mutex_exit(&cp->p_lock);
634 
635 	orphpp = &p->p_orphan;
636 	while (*orphpp != cp)
637 		orphpp = &(*orphpp)->p_nextorph;
638 	*orphpp = cp->p_nextorph;
639 	if (p->p_child == cp)
640 		p->p_child = cp->p_sibling;
641 	if (cp->p_sibling)
642 		cp->p_sibling->p_psibling = cp->p_psibling;
643 	if (cp->p_psibling)
644 		cp->p_psibling->p_sibling = cp->p_sibling;
645 	pid_exit(cp);
646 	mutex_exit(&pidlock);
647 
648 	task_rele(tk);
649 
650 	mutex_enter(&p->p_lock);
651 	pool_barrier_exit();
652 	continuelwps(p);
653 	mutex_exit(&p->p_lock);
654 	error = EAGAIN;
655 forkerr:
656 	return ((int64_t)set_errno(error));
657 }
658 
659 /*
660  * Free allocated resources from getproc() if a fork failed.
661  */
662 static void
663 fork_fail(proc_t *cp)
664 {
665 	uf_info_t *fip = P_FINFO(cp);
666 
667 	fcnt_add(fip, -1);
668 	sigdelq(cp, NULL, 0);
669 
670 	mutex_enter(&pidlock);
671 	upcount_dec(crgetruid(cp->p_cred), crgetzoneid(cp->p_cred));
672 	mutex_exit(&pidlock);
673 
674 	/*
675 	 * single threaded, so no locking needed here
676 	 */
677 	crfree(cp->p_cred);
678 
679 	kmem_free(fip->fi_list, fip->fi_nfiles * sizeof (uf_entry_t));
680 
681 	VN_RELE(PTOU(curproc)->u_cdir);
682 	if (PTOU(curproc)->u_rdir)
683 		VN_RELE(PTOU(curproc)->u_rdir);
684 	if (cp->p_exec)
685 		VN_RELE(cp->p_exec);
686 	if (cp->p_execdir)
687 		VN_RELE(cp->p_execdir);
688 	if (PTOU(curproc)->u_cwd)
689 		refstr_rele(PTOU(curproc)->u_cwd);
690 }
691 
692 /*
693  * Clean up the lwps already created for this child process.
694  * The fork failed while duplicating all the lwps of the parent
695  * and those lwps already created must be freed.
696  * This process is invisible to the rest of the system,
697  * so we don't need to hold p->p_lock to protect the list.
698  */
699 static void
700 forklwp_fail(proc_t *p)
701 {
702 	kthread_t *t;
703 	task_t *tk;
704 
705 	while ((t = p->p_tlist) != NULL) {
706 		/*
707 		 * First remove the lwp from the process's p_tlist.
708 		 */
709 		if (t != t->t_forw)
710 			p->p_tlist = t->t_forw;
711 		else
712 			p->p_tlist = NULL;
713 		p->p_lwpcnt--;
714 		t->t_forw->t_back = t->t_back;
715 		t->t_back->t_forw = t->t_forw;
716 
717 		tk = p->p_task;
718 		mutex_enter(&p->p_zone->zone_nlwps_lock);
719 		tk->tk_nlwps--;
720 		tk->tk_proj->kpj_nlwps--;
721 		p->p_zone->zone_nlwps--;
722 		mutex_exit(&p->p_zone->zone_nlwps_lock);
723 
724 		ASSERT(t->t_schedctl == NULL);
725 
726 		if (t->t_door != NULL) {
727 			kmem_free(t->t_door, sizeof (door_data_t));
728 			t->t_door = NULL;
729 		}
730 		lwp_ctmpl_clear(ttolwp(t));
731 
732 		/*
733 		 * Remove the thread from the all threads list.
734 		 * We need to hold pidlock for this.
735 		 */
736 		mutex_enter(&pidlock);
737 		t->t_next->t_prev = t->t_prev;
738 		t->t_prev->t_next = t->t_next;
739 		CL_EXIT(t);	/* tell the scheduler that we're exiting */
740 		cv_broadcast(&t->t_joincv);	/* tell anyone in thread_join */
741 		mutex_exit(&pidlock);
742 
743 		/*
744 		 * Let the lgroup load averages know that this thread isn't
745 		 * going to show up (i.e. un-do what was done on behalf of
746 		 * this thread by the earlier lgrp_move_thread()).
747 		 */
748 		kpreempt_disable();
749 		lgrp_move_thread(t, NULL, 1);
750 		kpreempt_enable();
751 
752 		/*
753 		 * The thread was created TS_STOPPED.
754 		 * We change it to TS_FREE to avoid an
755 		 * ASSERT() panic in thread_free().
756 		 */
757 		t->t_state = TS_FREE;
758 		thread_rele(t);
759 		thread_free(t);
760 	}
761 }
762 
763 extern struct as kas;
764 
765 /*
766  * fork a kernel process.
767  */
768 int
769 newproc(void (*pc)(), caddr_t arg, id_t cid, int pri, struct contract **ct,
770     pid_t pid)
771 {
772 	proc_t *p;
773 	struct user *up;
774 	kthread_t *t;
775 	cont_process_t *ctp = NULL;
776 	rctl_entity_p_t e;
777 
778 	ASSERT(cid != sysdccid);
779 	ASSERT(cid != syscid || ct == NULL);
780 	if (CLASS_KERNEL(cid)) {
781 		rctl_alloc_gp_t *init_gp;
782 		rctl_set_t *init_set;
783 
784 		ASSERT(pid != 1);
785 
786 		if (getproc(&p, pid, GETPROC_KERNEL) < 0)
787 			return (EAGAIN);
788 
789 		/*
790 		 * Release the hold on the p_exec and p_execdir, these
791 		 * were acquired in getproc()
792 		 */
793 		if (p->p_execdir != NULL)
794 			VN_RELE(p->p_execdir);
795 		if (p->p_exec != NULL)
796 			VN_RELE(p->p_exec);
797 		p->p_flag |= SNOWAIT;
798 		p->p_exec = NULL;
799 		p->p_execdir = NULL;
800 
801 		init_set = rctl_set_create();
802 		init_gp = rctl_set_init_prealloc(RCENTITY_PROCESS);
803 
804 		/*
805 		 * kernel processes do not inherit /proc tracing flags.
806 		 */
807 		sigemptyset(&p->p_sigmask);
808 		premptyset(&p->p_fltmask);
809 		up = PTOU(p);
810 		up->u_systrap = 0;
811 		premptyset(&(up->u_entrymask));
812 		premptyset(&(up->u_exitmask));
813 		mutex_enter(&p->p_lock);
814 		e.rcep_p.proc = p;
815 		e.rcep_t = RCENTITY_PROCESS;
816 		p->p_rctls = rctl_set_init(RCENTITY_PROCESS, p, &e, init_set,
817 		    init_gp);
818 		mutex_exit(&p->p_lock);
819 
820 		rctl_prealloc_destroy(init_gp);
821 
822 		t = lwp_kernel_create(p, pc, arg, TS_STOPPED, pri);
823 	} else {
824 		rctl_alloc_gp_t *init_gp, *default_gp;
825 		rctl_set_t *init_set;
826 		task_t *tk, *tk_old;
827 		klwp_t *lwp;
828 
829 		if (getproc(&p, pid, GETPROC_USER) < 0)
830 			return (EAGAIN);
831 		/*
832 		 * init creates a new task, distinct from the task
833 		 * containing kernel "processes".
834 		 */
835 		tk = task_create(0, p->p_zone);
836 		mutex_enter(&tk->tk_zone->zone_nlwps_lock);
837 		tk->tk_proj->kpj_ntasks++;
838 		mutex_exit(&tk->tk_zone->zone_nlwps_lock);
839 
840 		default_gp = rctl_rlimit_set_prealloc(RLIM_NLIMITS);
841 		init_gp = rctl_set_init_prealloc(RCENTITY_PROCESS);
842 		init_set = rctl_set_create();
843 
844 		mutex_enter(&pidlock);
845 		mutex_enter(&p->p_lock);
846 		tk_old = p->p_task;	/* switch to new task */
847 
848 		task_detach(p);
849 		task_begin(tk, p);
850 		mutex_exit(&pidlock);
851 
852 		e.rcep_p.proc = p;
853 		e.rcep_t = RCENTITY_PROCESS;
854 		p->p_rctls = rctl_set_init(RCENTITY_PROCESS, p, &e, init_set,
855 		    init_gp);
856 		rctlproc_default_init(p, default_gp);
857 		mutex_exit(&p->p_lock);
858 
859 		task_rele(tk_old);
860 		rctl_prealloc_destroy(default_gp);
861 		rctl_prealloc_destroy(init_gp);
862 
863 		if ((lwp = lwp_create(pc, arg, 0, p, TS_STOPPED, pri,
864 		    &curthread->t_hold, cid, 1)) == NULL) {
865 			task_t *tk;
866 			fork_fail(p);
867 			mutex_enter(&pidlock);
868 			mutex_enter(&p->p_lock);
869 			tk = p->p_task;
870 			task_detach(p);
871 			ASSERT(p->p_pool->pool_ref > 0);
872 			atomic_add_32(&p->p_pool->pool_ref, -1);
873 			mutex_exit(&p->p_lock);
874 			pid_exit(p);
875 			mutex_exit(&pidlock);
876 			task_rele(tk);
877 
878 			return (EAGAIN);
879 		}
880 		t = lwptot(lwp);
881 
882 		ctp = contract_process_fork(sys_process_tmpl, p, curproc,
883 		    B_FALSE);
884 		ASSERT(ctp != NULL);
885 		if (ct != NULL)
886 			*ct = &ctp->conp_contract;
887 	}
888 
889 	ASSERT3U(t->t_tid, ==, 1);
890 	p->p_lwpid = 1;
891 	mutex_enter(&pidlock);
892 	pgjoin(p, p->p_parent->p_pgidp);
893 	p->p_stat = SRUN;
894 	mutex_enter(&p->p_lock);
895 	t->t_proc_flag &= ~TP_HOLDLWP;
896 	lwp_create_done(t);
897 	mutex_exit(&p->p_lock);
898 	mutex_exit(&pidlock);
899 	return (0);
900 }
901 
902 /*
903  * create a child proc struct.
904  */
905 static int
906 getproc(proc_t **cpp, pid_t pid, uint_t flags)
907 {
908 	proc_t		*pp, *cp;
909 	pid_t		newpid;
910 	struct user	*uarea;
911 	extern uint_t	nproc;
912 	struct cred	*cr;
913 	uid_t		ruid;
914 	zoneid_t	zoneid;
915 
916 	if (!page_mem_avail(tune.t_minarmem))
917 		return (-1);
918 	if (zone_status_get(curproc->p_zone) >= ZONE_IS_SHUTTING_DOWN)
919 		return (-1);	/* no point in starting new processes */
920 
921 	pp = (flags & GETPROC_KERNEL) ? &p0 : curproc;
922 	cp = kmem_cache_alloc(process_cache, KM_SLEEP);
923 	bzero(cp, sizeof (proc_t));
924 
925 	/*
926 	 * Make proc entry for child process
927 	 */
928 	mutex_init(&cp->p_splock, NULL, MUTEX_DEFAULT, NULL);
929 	mutex_init(&cp->p_crlock, NULL, MUTEX_DEFAULT, NULL);
930 	mutex_init(&cp->p_pflock, NULL, MUTEX_DEFAULT, NULL);
931 #if defined(__x86)
932 	mutex_init(&cp->p_ldtlock, NULL, MUTEX_DEFAULT, NULL);
933 #endif
934 	mutex_init(&cp->p_maplock, NULL, MUTEX_DEFAULT, NULL);
935 	cp->p_stat = SIDL;
936 	cp->p_mstart = gethrtime();
937 	cp->p_as = &kas;
938 	/*
939 	 * p_zone must be set before we call pid_allocate since the process
940 	 * will be visible after that and code such as prfind_zone will
941 	 * look at the p_zone field.
942 	 */
943 	cp->p_zone = pp->p_zone;
944 	cp->p_t1_lgrpid = LGRP_NONE;
945 	cp->p_tr_lgrpid = LGRP_NONE;
946 
947 	if ((newpid = pid_allocate(cp, pid, PID_ALLOC_PROC)) == -1) {
948 		if (nproc == v.v_proc) {
949 			CPU_STATS_ADDQ(CPU, sys, procovf, 1);
950 			cmn_err(CE_WARN, "out of processes");
951 		}
952 		goto bad;
953 	}
954 
955 	mutex_enter(&pp->p_lock);
956 	cp->p_exec = pp->p_exec;
957 	cp->p_execdir = pp->p_execdir;
958 	mutex_exit(&pp->p_lock);
959 
960 	if (cp->p_exec) {
961 		VN_HOLD(cp->p_exec);
962 		/*
963 		 * Each VOP_OPEN() must be paired with a corresponding
964 		 * VOP_CLOSE(). In this case, the executable will be
965 		 * closed for the child in either proc_exit() or gexec().
966 		 */
967 		if (VOP_OPEN(&cp->p_exec, FREAD, CRED(), NULL) != 0) {
968 			VN_RELE(cp->p_exec);
969 			cp->p_exec = NULLVP;
970 			cp->p_execdir = NULLVP;
971 			goto bad;
972 		}
973 	}
974 	if (cp->p_execdir)
975 		VN_HOLD(cp->p_execdir);
976 
977 	/*
978 	 * If not privileged make sure that this user hasn't exceeded
979 	 * v.v_maxup processes, and that users collectively haven't
980 	 * exceeded v.v_maxupttl processes.
981 	 */
982 	mutex_enter(&pidlock);
983 	ASSERT(nproc < v.v_proc);	/* otherwise how'd we get our pid? */
984 	cr = CRED();
985 	ruid = crgetruid(cr);
986 	zoneid = crgetzoneid(cr);
987 	if (nproc >= v.v_maxup && 	/* short-circuit; usually false */
988 	    (nproc >= v.v_maxupttl ||
989 	    upcount_get(ruid, zoneid) >= v.v_maxup) &&
990 	    secpolicy_newproc(cr) != 0) {
991 		mutex_exit(&pidlock);
992 		zcmn_err(zoneid, CE_NOTE,
993 		    "out of per-user processes for uid %d", ruid);
994 		goto bad;
995 	}
996 
997 	/*
998 	 * Everything is cool, put the new proc on the active process list.
999 	 * It is already on the pid list and in /proc.
1000 	 * Increment the per uid process count (upcount).
1001 	 */
1002 	nproc++;
1003 	upcount_inc(ruid, zoneid);
1004 
1005 	cp->p_next = practive;
1006 	practive->p_prev = cp;
1007 	practive = cp;
1008 
1009 	cp->p_ignore = pp->p_ignore;
1010 	cp->p_siginfo = pp->p_siginfo;
1011 	cp->p_flag = pp->p_flag & (SJCTL|SNOWAIT|SNOCD);
1012 	cp->p_sessp = pp->p_sessp;
1013 	sess_hold(pp);
1014 	cp->p_brand = pp->p_brand;
1015 	if (PROC_IS_BRANDED(pp))
1016 		BROP(pp)->b_copy_procdata(cp, pp);
1017 	cp->p_bssbase = pp->p_bssbase;
1018 	cp->p_brkbase = pp->p_brkbase;
1019 	cp->p_brksize = pp->p_brksize;
1020 	cp->p_brkpageszc = pp->p_brkpageszc;
1021 	cp->p_stksize = pp->p_stksize;
1022 	cp->p_stkpageszc = pp->p_stkpageszc;
1023 	cp->p_stkprot = pp->p_stkprot;
1024 	cp->p_datprot = pp->p_datprot;
1025 	cp->p_usrstack = pp->p_usrstack;
1026 	cp->p_model = pp->p_model;
1027 	cp->p_ppid = pp->p_pid;
1028 	cp->p_ancpid = pp->p_pid;
1029 	cp->p_portcnt = pp->p_portcnt;
1030 
1031 	/*
1032 	 * Initialize watchpoint structures
1033 	 */
1034 	avl_create(&cp->p_warea, wa_compare, sizeof (struct watched_area),
1035 	    offsetof(struct watched_area, wa_link));
1036 
1037 	/*
1038 	 * Initialize immediate resource control values.
1039 	 */
1040 	cp->p_stk_ctl = pp->p_stk_ctl;
1041 	cp->p_fsz_ctl = pp->p_fsz_ctl;
1042 	cp->p_vmem_ctl = pp->p_vmem_ctl;
1043 	cp->p_fno_ctl = pp->p_fno_ctl;
1044 
1045 	/*
1046 	 * Link up to parent-child-sibling chain.  No need to lock
1047 	 * in general since only a call to freeproc() (done by the
1048 	 * same parent as newproc()) diddles with the child chain.
1049 	 */
1050 	cp->p_sibling = pp->p_child;
1051 	if (pp->p_child)
1052 		pp->p_child->p_psibling = cp;
1053 
1054 	cp->p_parent = pp;
1055 	pp->p_child = cp;
1056 
1057 	cp->p_child_ns = NULL;
1058 	cp->p_sibling_ns = NULL;
1059 
1060 	cp->p_nextorph = pp->p_orphan;
1061 	cp->p_nextofkin = pp;
1062 	pp->p_orphan = cp;
1063 
1064 	/*
1065 	 * Inherit profiling state; do not inherit REALPROF profiling state.
1066 	 */
1067 	cp->p_prof = pp->p_prof;
1068 	cp->p_rprof_cyclic = CYCLIC_NONE;
1069 
1070 	/*
1071 	 * Inherit pool pointer from the parent.  Kernel processes are
1072 	 * always bound to the default pool.
1073 	 */
1074 	mutex_enter(&pp->p_lock);
1075 	if (flags & GETPROC_KERNEL) {
1076 		cp->p_pool = pool_default;
1077 		cp->p_flag |= SSYS;
1078 	} else {
1079 		cp->p_pool = pp->p_pool;
1080 	}
1081 	atomic_add_32(&cp->p_pool->pool_ref, 1);
1082 	mutex_exit(&pp->p_lock);
1083 
1084 	/*
1085 	 * Add the child process to the current task.  Kernel processes
1086 	 * are always attached to task0.
1087 	 */
1088 	mutex_enter(&cp->p_lock);
1089 	if (flags & GETPROC_KERNEL)
1090 		task_attach(task0p, cp);
1091 	else
1092 		task_attach(pp->p_task, cp);
1093 	mutex_exit(&cp->p_lock);
1094 	mutex_exit(&pidlock);
1095 
1096 	avl_create(&cp->p_ct_held, contract_compar, sizeof (contract_t),
1097 	    offsetof(contract_t, ct_ctlist));
1098 
1099 	/*
1100 	 * Duplicate any audit information kept in the process table
1101 	 */
1102 	if (audit_active)	/* copy audit data to cp */
1103 		audit_newproc(cp);
1104 
1105 	crhold(cp->p_cred = cr);
1106 
1107 	/*
1108 	 * Bump up the counts on the file structures pointed at by the
1109 	 * parent's file table since the child will point at them too.
1110 	 */
1111 	fcnt_add(P_FINFO(pp), 1);
1112 
1113 	if (PTOU(pp)->u_cdir) {
1114 		VN_HOLD(PTOU(pp)->u_cdir);
1115 	} else {
1116 		ASSERT(pp == &p0);
1117 		/*
1118 		 * We must be at or before vfs_mountroot(); it will take care of
1119 		 * assigning our current directory.
1120 		 */
1121 	}
1122 	if (PTOU(pp)->u_rdir)
1123 		VN_HOLD(PTOU(pp)->u_rdir);
1124 	if (PTOU(pp)->u_cwd)
1125 		refstr_hold(PTOU(pp)->u_cwd);
1126 
1127 	/*
1128 	 * copy the parent's uarea.
1129 	 */
1130 	uarea = PTOU(cp);
1131 	bcopy(PTOU(pp), uarea, sizeof (*uarea));
1132 	flist_fork(P_FINFO(pp), P_FINFO(cp));
1133 
1134 	gethrestime(&uarea->u_start);
1135 	uarea->u_ticks = ddi_get_lbolt();
1136 	uarea->u_mem = rm_asrss(pp->p_as);
1137 	uarea->u_acflag = AFORK;
1138 
1139 	/*
1140 	 * If inherit-on-fork, copy /proc tracing flags to child.
1141 	 */
1142 	if ((pp->p_proc_flag & P_PR_FORK) != 0) {
1143 		cp->p_proc_flag |= pp->p_proc_flag & (P_PR_TRACE|P_PR_FORK);
1144 		cp->p_sigmask = pp->p_sigmask;
1145 		cp->p_fltmask = pp->p_fltmask;
1146 	} else {
1147 		sigemptyset(&cp->p_sigmask);
1148 		premptyset(&cp->p_fltmask);
1149 		uarea->u_systrap = 0;
1150 		premptyset(&uarea->u_entrymask);
1151 		premptyset(&uarea->u_exitmask);
1152 	}
1153 	/*
1154 	 * If microstate accounting is being inherited, mark child
1155 	 */
1156 	if ((pp->p_flag & SMSFORK) != 0)
1157 		cp->p_flag |= pp->p_flag & (SMSFORK|SMSACCT);
1158 
1159 	/*
1160 	 * Inherit fixalignment flag from the parent
1161 	 */
1162 	cp->p_fixalignment = pp->p_fixalignment;
1163 
1164 	*cpp = cp;
1165 	return (0);
1166 
1167 bad:
1168 	ASSERT(MUTEX_NOT_HELD(&pidlock));
1169 
1170 	mutex_destroy(&cp->p_crlock);
1171 	mutex_destroy(&cp->p_pflock);
1172 #if defined(__x86)
1173 	mutex_destroy(&cp->p_ldtlock);
1174 #endif
1175 	if (newpid != -1) {
1176 		proc_entry_free(cp->p_pidp);
1177 		(void) pid_rele(cp->p_pidp);
1178 	}
1179 	kmem_cache_free(process_cache, cp);
1180 
1181 	/*
1182 	 * We most likely got into this situation because some process is
1183 	 * forking out of control.  As punishment, put it to sleep for a
1184 	 * bit so it can't eat the machine alive.  Sleep interval is chosen
1185 	 * to allow no more than one fork failure per cpu per clock tick
1186 	 * on average (yes, I just made this up).  This has two desirable
1187 	 * properties: (1) it sets a constant limit on the fork failure
1188 	 * rate, and (2) the busier the system is, the harsher the penalty
1189 	 * for abusing it becomes.
1190 	 */
1191 	INCR_COUNT(&fork_fail_pending, &pidlock);
1192 	delay(fork_fail_pending / ncpus + 1);
1193 	DECR_COUNT(&fork_fail_pending, &pidlock);
1194 
1195 	return (-1); /* out of memory or proc slots */
1196 }
1197 
1198 /*
1199  * Release virtual memory.
1200  * In the case of vfork(), the child was given exclusive access to its
1201  * parent's address space.  The parent is waiting in vfwait() for the
1202  * child to release its exclusive claim via relvm().
1203  */
1204 void
1205 relvm()
1206 {
1207 	proc_t *p = curproc;
1208 
1209 	ASSERT((unsigned)p->p_lwpcnt <= 1);
1210 
1211 	prrelvm();	/* inform /proc */
1212 
1213 	if (p->p_flag & SVFORK) {
1214 		proc_t *pp = p->p_parent;
1215 		/*
1216 		 * The child process is either exec'ing or exit'ing.
1217 		 * The child is now separated from the parent's address
1218 		 * space.  The parent process is made dispatchable.
1219 		 *
1220 		 * This is a delicate locking maneuver, involving
1221 		 * both the parent's p_lock and the child's p_lock.
1222 		 * As soon as the SVFORK flag is turned off, the
1223 		 * parent is free to run, but it must not run until
1224 		 * we wake it up using its p_cv because it might
1225 		 * exit and we would be referencing invalid memory.
1226 		 * Therefore, we hold the parent with its p_lock
1227 		 * while protecting our p_flags with our own p_lock.
1228 		 */
1229 try_again:
1230 		mutex_enter(&p->p_lock);	/* grab child's lock first */
1231 		prbarrier(p);		/* make sure /proc is blocked out */
1232 		mutex_enter(&pp->p_lock);
1233 
1234 		/*
1235 		 * Check if parent is locked by /proc.
1236 		 */
1237 		if (pp->p_proc_flag & P_PR_LOCK) {
1238 			/*
1239 			 * Delay until /proc is done with the parent.
1240 			 * We must drop our (the child's) p->p_lock, wait
1241 			 * via prbarrier() on the parent, then start over.
1242 			 */
1243 			mutex_exit(&p->p_lock);
1244 			prbarrier(pp);
1245 			mutex_exit(&pp->p_lock);
1246 			goto try_again;
1247 		}
1248 		p->p_flag &= ~SVFORK;
1249 		kpreempt_disable();
1250 		p->p_as = &kas;
1251 
1252 		/*
1253 		 * notify hat of change in thread's address space
1254 		 */
1255 		hat_thread_exit(curthread);
1256 		kpreempt_enable();
1257 
1258 		/*
1259 		 * child sizes are copied back to parent because
1260 		 * child may have grown.
1261 		 */
1262 		pp->p_brkbase = p->p_brkbase;
1263 		pp->p_brksize = p->p_brksize;
1264 		pp->p_stksize = p->p_stksize;
1265 
1266 		/*
1267 		 * Copy back the shm accounting information
1268 		 * to the parent process.
1269 		 */
1270 		pp->p_segacct = p->p_segacct;
1271 		p->p_segacct = NULL;
1272 
1273 		/*
1274 		 * The parent is no longer waiting for the vfork()d child.
1275 		 * Restore the parent's watched pages, if any.  This is
1276 		 * safe because we know the parent is not locked by /proc
1277 		 */
1278 		pp->p_flag &= ~SVFWAIT;
1279 		if (avl_numnodes(&pp->p_wpage) != 0) {
1280 			pp->p_as->a_wpage = pp->p_wpage;
1281 			avl_create(&pp->p_wpage, wp_compare,
1282 			    sizeof (struct watched_page),
1283 			    offsetof(struct watched_page, wp_link));
1284 		}
1285 		cv_signal(&pp->p_cv);
1286 		mutex_exit(&pp->p_lock);
1287 		mutex_exit(&p->p_lock);
1288 	} else {
1289 		if (p->p_as != &kas) {
1290 			struct as *as;
1291 
1292 			if (p->p_segacct)
1293 				shmexit(p);
1294 
1295 			/*
1296 			 * We grab p_lock for the benefit of /proc
1297 			 */
1298 			kpreempt_disable();
1299 			mutex_enter(&p->p_lock);
1300 			prbarrier(p);	/* make sure /proc is blocked out */
1301 			as = p->p_as;
1302 			p->p_as = &kas;
1303 			mutex_exit(&p->p_lock);
1304 
1305 			/*
1306 			 * notify hat of change in thread's address space
1307 			 */
1308 			hat_thread_exit(curthread);
1309 			kpreempt_enable();
1310 
1311 			as_free(as);
1312 			p->p_tr_lgrpid = LGRP_NONE;
1313 		}
1314 	}
1315 }
1316 
1317 /*
1318  * Wait for child to exec or exit.
1319  * Called by parent of vfork'ed process.
1320  * See important comments in relvm(), above.
1321  */
1322 void
1323 vfwait(pid_t pid)
1324 {
1325 	int signalled = 0;
1326 	proc_t *pp = ttoproc(curthread);
1327 	proc_t *cp;
1328 
1329 	/*
1330 	 * Wait for child to exec or exit.
1331 	 */
1332 	for (;;) {
1333 		mutex_enter(&pidlock);
1334 		cp = prfind(pid);
1335 		if (cp == NULL || cp->p_parent != pp) {
1336 			/*
1337 			 * Child has exit()ed.
1338 			 */
1339 			mutex_exit(&pidlock);
1340 			break;
1341 		}
1342 		/*
1343 		 * Grab the child's p_lock before releasing pidlock.
1344 		 * Otherwise, the child could exit and we would be
1345 		 * referencing invalid memory.
1346 		 */
1347 		mutex_enter(&cp->p_lock);
1348 		mutex_exit(&pidlock);
1349 		if (!(cp->p_flag & SVFORK)) {
1350 			/*
1351 			 * Child has exec()ed or is exit()ing.
1352 			 */
1353 			mutex_exit(&cp->p_lock);
1354 			break;
1355 		}
1356 		mutex_enter(&pp->p_lock);
1357 		mutex_exit(&cp->p_lock);
1358 		/*
1359 		 * We might be waked up spuriously from the cv_wait().
1360 		 * We have to do the whole operation over again to be
1361 		 * sure the child's SVFORK flag really is turned off.
1362 		 * We cannot make reference to the child because it can
1363 		 * exit before we return and we would be referencing
1364 		 * invalid memory.
1365 		 *
1366 		 * Because this is potentially a very long-term wait,
1367 		 * we call cv_wait_sig() (for its jobcontrol and /proc
1368 		 * side-effects) unless there is a current signal, in
1369 		 * which case we use cv_wait() because we cannot return
1370 		 * from this function until the child has released the
1371 		 * address space.  Calling cv_wait_sig() with a current
1372 		 * signal would lead to an indefinite loop here because
1373 		 * cv_wait_sig() returns immediately in this case.
1374 		 */
1375 		if (signalled)
1376 			cv_wait(&pp->p_cv, &pp->p_lock);
1377 		else
1378 			signalled = !cv_wait_sig(&pp->p_cv, &pp->p_lock);
1379 		mutex_exit(&pp->p_lock);
1380 	}
1381 
1382 	/* restore watchpoints to parent */
1383 	if (pr_watch_active(pp)) {
1384 		struct as *as = pp->p_as;
1385 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1386 		as_setwatch(as);
1387 		AS_LOCK_EXIT(as, &as->a_lock);
1388 	}
1389 
1390 	mutex_enter(&pp->p_lock);
1391 	prbarrier(pp);	/* barrier against /proc locking */
1392 	continuelwps(pp);
1393 	mutex_exit(&pp->p_lock);
1394 }
1395