xref: /titanic_51/usr/src/uts/intel/ia32/os/syscall.c (revision 37714ae43602c675f9dc59b070bfdf9fa702872c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/param.h>
28 #include <sys/vmparam.h>
29 #include <sys/types.h>
30 #include <sys/sysmacros.h>
31 #include <sys/systm.h>
32 #include <sys/signal.h>
33 #include <sys/stack.h>
34 #include <sys/cred.h>
35 #include <sys/cmn_err.h>
36 #include <sys/user.h>
37 #include <sys/privregs.h>
38 #include <sys/psw.h>
39 #include <sys/debug.h>
40 #include <sys/errno.h>
41 #include <sys/proc.h>
42 #include <sys/modctl.h>
43 #include <sys/var.h>
44 #include <sys/inline.h>
45 #include <sys/syscall.h>
46 #include <sys/ucontext.h>
47 #include <sys/cpuvar.h>
48 #include <sys/siginfo.h>
49 #include <sys/trap.h>
50 #include <sys/vtrace.h>
51 #include <sys/sysinfo.h>
52 #include <sys/procfs.h>
53 #include <sys/prsystm.h>
54 #include <c2/audit.h>
55 #include <sys/modctl.h>
56 #include <sys/aio_impl.h>
57 #include <sys/tnf.h>
58 #include <sys/tnf_probe.h>
59 #include <sys/copyops.h>
60 #include <sys/priv.h>
61 #include <sys/msacct.h>
62 
63 int syscalltrace = 0;
64 #ifdef SYSCALLTRACE
65 static kmutex_t systrace_lock;		/* syscall tracing lock */
66 #else
67 #define	syscalltrace 0
68 #endif /* SYSCALLTRACE */
69 
70 typedef	int64_t (*llfcn_t)();	/* function returning long long */
71 
72 int pre_syscall(void);
73 void post_syscall(long rval1, long rval2);
74 static krwlock_t *lock_syscall(struct sysent *, uint_t);
75 void deferred_singlestep_trap(caddr_t);
76 
77 #ifdef _SYSCALL32_IMPL
78 #define	LWP_GETSYSENT(lwp)	\
79 	(lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? sysent : sysent32)
80 #else
81 #define	LWP_GETSYSENT(lwp)	(sysent)
82 #endif
83 
84 /*
85  * If watchpoints are active, don't make copying in of
86  * system call arguments take a read watchpoint trap.
87  */
88 static int
89 copyin_args(struct regs *rp, long *ap, uint_t nargs)
90 {
91 	greg_t *sp = 1 + (greg_t *)rp->r_sp;		/* skip ret addr */
92 
93 	ASSERT(nargs <= MAXSYSARGS);
94 
95 	return (copyin_nowatch(sp, ap, nargs * sizeof (*sp)));
96 }
97 
98 #if defined(_SYSCALL32_IMPL)
99 static int
100 copyin_args32(struct regs *rp, long *ap, uint_t nargs)
101 {
102 	greg32_t *sp = 1 + (greg32_t *)rp->r_sp;	/* skip ret addr */
103 	uint32_t a32[MAXSYSARGS];
104 	int rc;
105 
106 	ASSERT(nargs <= MAXSYSARGS);
107 
108 	if ((rc = copyin_nowatch(sp, a32, nargs * sizeof (*sp))) == 0) {
109 		uint32_t *a32p = &a32[0];
110 
111 		while (nargs--)
112 			*ap++ = (ulong_t)*a32p++;
113 	}
114 	return (rc);
115 }
116 #define	COPYIN_ARGS32	copyin_args32
117 #else
118 #define	COPYIN_ARGS32	copyin_args
119 #endif
120 
121 /*
122  * Error handler for system calls where arg copy gets fault.
123  */
124 static longlong_t
125 syscall_err()
126 {
127 	return (0);
128 }
129 
130 /*
131  * Corresponding sysent entry to allow syscall_entry caller
132  * to invoke syscall_err.
133  */
134 static struct sysent sysent_err =  {
135 	0, SE_32RVAL1, NULL, NULL, (llfcn_t)syscall_err
136 };
137 
138 /*
139  * Called from syscall() when a non-trivial 32-bit system call occurs.
140  * 	Sets up the args and returns a pointer to the handler.
141  */
142 struct sysent *
143 syscall_entry(kthread_t *t, long *argp)
144 {
145 	klwp_t *lwp = ttolwp(t);
146 	struct regs *rp = lwptoregs(lwp);
147 	unsigned int code;
148 	struct sysent *callp;
149 	struct sysent *se = LWP_GETSYSENT(lwp);
150 	int error = 0;
151 	uint_t nargs;
152 
153 	ASSERT(t == curthread && curthread->t_schedflag & TS_DONT_SWAP);
154 
155 	lwp->lwp_ru.sysc++;
156 	lwp->lwp_eosys = NORMALRETURN;	/* assume this will be normal */
157 
158 	/*
159 	 * Set lwp_ap to point to the args, even if none are needed for this
160 	 * system call.  This is for the loadable-syscall case where the
161 	 * number of args won't be known until the system call is loaded, and
162 	 * also maintains a non-NULL lwp_ap setup for get_syscall_args(). Note
163 	 * that lwp_ap MUST be set to a non-NULL value _BEFORE_ t_sysnum is
164 	 * set to non-zero; otherwise get_syscall_args(), seeing a non-zero
165 	 * t_sysnum for this thread, will charge ahead and dereference lwp_ap.
166 	 */
167 	lwp->lwp_ap = argp;		/* for get_syscall_args */
168 
169 	code = rp->r_r0;
170 	t->t_sysnum = (short)code;
171 	callp = code >= NSYSCALL ? &nosys_ent : se + code;
172 
173 	if ((t->t_pre_sys | syscalltrace) != 0) {
174 		error = pre_syscall();
175 
176 		/*
177 		 * pre_syscall() has taken care so that lwp_ap is current;
178 		 * it either points to syscall-entry-saved amd64 regs,
179 		 * or it points to lwp_arg[], which has been re-copied from
180 		 * the ia32 ustack, but either way, it's a current copy after
181 		 * /proc has possibly mucked with the syscall args.
182 		 */
183 
184 		if (error)
185 			return (&sysent_err);	/* use dummy handler */
186 	}
187 
188 	/*
189 	 * Fetch the system call arguments to the kernel stack copy used
190 	 * for syscall handling.
191 	 * Note: for loadable system calls the number of arguments required
192 	 * may not be known at this point, and will be zero if the system call
193 	 * was never loaded.  Once the system call has been loaded, the number
194 	 * of args is not allowed to be changed.
195 	 */
196 	if ((nargs = (uint_t)callp->sy_narg) != 0 &&
197 	    COPYIN_ARGS32(rp, argp, nargs)) {
198 		(void) set_errno(EFAULT);
199 		return (&sysent_err);	/* use dummy handler */
200 	}
201 
202 	return (callp);		/* return sysent entry for caller */
203 }
204 
205 void
206 syscall_exit(kthread_t *t, long rval1, long rval2)
207 {
208 	/*
209 	 * Handle signals and other post-call events if necessary.
210 	 */
211 	if ((t->t_post_sys_ast | syscalltrace) == 0) {
212 		klwp_t *lwp = ttolwp(t);
213 		struct regs *rp = lwptoregs(lwp);
214 
215 		/*
216 		 * Normal return.
217 		 * Clear error indication and set return values.
218 		 */
219 		rp->r_ps &= ~PS_C;	/* reset carry bit */
220 		rp->r_r0 = rval1;
221 		rp->r_r1 = rval2;
222 		lwp->lwp_state = LWP_USER;
223 	} else
224 		post_syscall(rval1, rval2);
225 	t->t_sysnum = 0;		/* invalidate args */
226 }
227 
228 /*
229  * Perform pre-system-call processing, including stopping for tracing,
230  * auditing, etc.
231  *
232  * This routine is called only if the t_pre_sys flag is set. Any condition
233  * requiring pre-syscall handling must set the t_pre_sys flag. If the
234  * condition is persistent, this routine will repost t_pre_sys.
235  */
236 int
237 pre_syscall()
238 {
239 	kthread_t *t = curthread;
240 	unsigned code = t->t_sysnum;
241 	klwp_t *lwp = ttolwp(t);
242 	proc_t *p = ttoproc(t);
243 	int	repost;
244 
245 	t->t_pre_sys = repost = 0;	/* clear pre-syscall processing flag */
246 
247 	ASSERT(t->t_schedflag & TS_DONT_SWAP);
248 
249 #if defined(DEBUG)
250 	/*
251 	 * On the i386 kernel, lwp_ap points at the piece of the thread
252 	 * stack that we copy the users arguments into.
253 	 *
254 	 * On the amd64 kernel, the syscall arguments in the rdi..r9
255 	 * registers should be pointed at by lwp_ap.  If the args need to
256 	 * be copied so that those registers can be changed without losing
257 	 * the ability to get the args for /proc, they can be saved by
258 	 * save_syscall_args(), and lwp_ap will be restored by post_syscall().
259 	 */
260 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
261 #if defined(_LP64)
262 		ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi);
263 	} else {
264 #endif
265 		ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase &&
266 		    (caddr_t)lwp->lwp_ap < t->t_stk);
267 	}
268 #endif	/* DEBUG */
269 
270 	/*
271 	 * Make sure the thread is holding the latest credentials for the
272 	 * process.  The credentials in the process right now apply to this
273 	 * thread for the entire system call.
274 	 */
275 	if (t->t_cred != p->p_cred) {
276 		cred_t *oldcred = t->t_cred;
277 		/*
278 		 * DTrace accesses t_cred in probe context.  t_cred must
279 		 * always be either NULL, or point to a valid, allocated cred
280 		 * structure.
281 		 */
282 		t->t_cred = crgetcred();
283 		crfree(oldcred);
284 	}
285 
286 	/*
287 	 * From the proc(4) manual page:
288 	 * When entry to a system call is being traced, the traced process
289 	 * stops after having begun the call to the system but before the
290 	 * system call arguments have been fetched from the process.
291 	 */
292 	if (PTOU(p)->u_systrap) {
293 		if (prismember(&PTOU(p)->u_entrymask, code)) {
294 			mutex_enter(&p->p_lock);
295 			/*
296 			 * Recheck stop condition, now that lock is held.
297 			 */
298 			if (PTOU(p)->u_systrap &&
299 			    prismember(&PTOU(p)->u_entrymask, code)) {
300 				stop(PR_SYSENTRY, code);
301 
302 				/*
303 				 * /proc may have modified syscall args,
304 				 * either in regs for amd64 or on ustack
305 				 * for ia32.  Either way, arrange to
306 				 * copy them again, both for the syscall
307 				 * handler and for other consumers in
308 				 * post_syscall (like audit).  Here, we
309 				 * only do amd64, and just set lwp_ap
310 				 * back to the kernel-entry stack copy;
311 				 * the syscall ml code redoes
312 				 * move-from-regs to set up for the
313 				 * syscall handler after we return.  For
314 				 * ia32, save_syscall_args() below makes
315 				 * an lwp_ap-accessible copy.
316 				 */
317 #if defined(_LP64)
318 				if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
319 					lwp->lwp_argsaved = 0;
320 					lwp->lwp_ap =
321 					    (long *)&lwptoregs(lwp)->r_rdi;
322 				}
323 #endif
324 			}
325 			mutex_exit(&p->p_lock);
326 		}
327 		repost = 1;
328 	}
329 
330 	/*
331 	 * ia32 kernel, or ia32 proc on amd64 kernel: keep args in
332 	 * lwp_arg for post-syscall processing, regardless of whether
333 	 * they might have been changed in /proc above.
334 	 */
335 #if defined(_LP64)
336 	if (lwp_getdatamodel(lwp) != DATAMODEL_NATIVE)
337 #endif
338 		(void) save_syscall_args();
339 
340 	if (lwp->lwp_sysabort) {
341 		/*
342 		 * lwp_sysabort may have been set via /proc while the process
343 		 * was stopped on PR_SYSENTRY.  If so, abort the system call.
344 		 * Override any error from the copyin() of the arguments.
345 		 */
346 		lwp->lwp_sysabort = 0;
347 		(void) set_errno(EINTR);	/* forces post_sys */
348 		t->t_pre_sys = 1;	/* repost anyway */
349 		return (1);		/* don't do system call, return EINTR */
350 	}
351 
352 	if (audit_active) {	/* begin auditing for this syscall */
353 		int error;
354 		if (error = audit_start(T_SYSCALL, code, 0, lwp)) {
355 			t->t_pre_sys = 1;	/* repost anyway */
356 			(void) set_errno(error);
357 			return (1);
358 		}
359 		repost = 1;
360 	}
361 
362 #ifndef NPROBE
363 	/* Kernel probe */
364 	if (tnf_tracing_active) {
365 		TNF_PROBE_1(syscall_start, "syscall thread", /* CSTYLED */,
366 			tnf_sysnum,	sysnum,		t->t_sysnum);
367 		t->t_post_sys = 1;	/* make sure post_syscall runs */
368 		repost = 1;
369 	}
370 #endif /* NPROBE */
371 
372 #ifdef SYSCALLTRACE
373 	if (syscalltrace) {
374 		int i;
375 		long *ap;
376 		char *cp;
377 		char *sysname;
378 		struct sysent *callp;
379 
380 		if (code >= NSYSCALL)
381 			callp = &nosys_ent;	/* nosys has no args */
382 		else
383 			callp = LWP_GETSYSENT(lwp) + code;
384 		(void) save_syscall_args();
385 		mutex_enter(&systrace_lock);
386 		printf("%d: ", p->p_pid);
387 		if (code >= NSYSCALL)
388 			printf("0x%x", code);
389 		else {
390 			sysname = mod_getsysname(code);
391 			printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" :
392 			    sysname, code, callp->sy_callc);
393 		}
394 		cp = "(";
395 		for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) {
396 			printf("%s%lx", cp, *ap);
397 			cp = ", ";
398 		}
399 		if (i)
400 			printf(")");
401 		printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread);
402 		mutex_exit(&systrace_lock);
403 	}
404 #endif /* SYSCALLTRACE */
405 
406 	/*
407 	 * If there was a continuing reason for pre-syscall processing,
408 	 * set the t_pre_sys flag for the next system call.
409 	 */
410 	if (repost)
411 		t->t_pre_sys = 1;
412 	lwp->lwp_error = 0;	/* for old drivers */
413 	lwp->lwp_badpriv = PRIV_NONE;
414 	return (0);
415 }
416 
417 
418 /*
419  * Post-syscall processing.  Perform abnormal system call completion
420  * actions such as /proc tracing, profiling, signals, preemption, etc.
421  *
422  * This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
423  * Any condition requiring pre-syscall handling must set one of these.
424  * If the condition is persistent, this routine will repost t_post_sys.
425  */
426 void
427 post_syscall(long rval1, long rval2)
428 {
429 	kthread_t *t = curthread;
430 	klwp_t *lwp = ttolwp(t);
431 	proc_t *p = ttoproc(t);
432 	struct regs *rp = lwptoregs(lwp);
433 	uint_t	error;
434 	uint_t	code = t->t_sysnum;
435 	int	repost = 0;
436 	int	proc_stop = 0;		/* non-zero if stopping */
437 	int	sigprof = 0;		/* non-zero if sending SIGPROF */
438 
439 	t->t_post_sys = 0;
440 
441 	error = lwp->lwp_errno;
442 
443 	/*
444 	 * Code can be zero if this is a new LWP returning after a forkall(),
445 	 * other than the one which matches the one in the parent which called
446 	 * forkall().  In these LWPs, skip most of post-syscall activity.
447 	 */
448 	if (code == 0)
449 		goto sig_check;
450 	/*
451 	 * If the trace flag is set, mark the lwp to take a single-step trap
452 	 * on return to user level (below). The x86 lcall interface and
453 	 * sysenter has already done this, and turned off the flag, but
454 	 * amd64 syscall interface has not.
455 	 */
456 	if (rp->r_ps & PS_T) {
457 		lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
458 		rp->r_ps &= ~PS_T;
459 		aston(curthread);
460 	}
461 	if (audit_active) {	/* put out audit record for this syscall */
462 		rval_t	rval;
463 
464 		/* XX64 -- truncation of 64-bit return values? */
465 		rval.r_val1 = (int)rval1;
466 		rval.r_val2 = (int)rval2;
467 		audit_finish(T_SYSCALL, code, error, &rval);
468 		repost = 1;
469 	}
470 
471 	if (curthread->t_pdmsg != NULL) {
472 		char *m = curthread->t_pdmsg;
473 
474 		uprintf("%s", m);
475 		kmem_free(m, strlen(m) + 1);
476 		curthread->t_pdmsg = NULL;
477 	}
478 
479 	/*
480 	 * If we're going to stop for /proc tracing, set the flag and
481 	 * save the arguments so that the return values don't smash them.
482 	 */
483 	if (PTOU(p)->u_systrap) {
484 		if (prismember(&PTOU(p)->u_exitmask, code)) {
485 			if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
486 				(void) save_syscall_args();
487 			proc_stop = 1;
488 		}
489 		repost = 1;
490 	}
491 
492 	/*
493 	 * Similarly check to see if SIGPROF might be sent.
494 	 */
495 	if (curthread->t_rprof != NULL &&
496 	    curthread->t_rprof->rp_anystate != 0) {
497 		if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
498 			(void) save_syscall_args();
499 		sigprof = 1;
500 	}
501 
502 	if (lwp->lwp_eosys == NORMALRETURN) {
503 		if (error == 0) {
504 #ifdef SYSCALLTRACE
505 			if (syscalltrace) {
506 				mutex_enter(&systrace_lock);
507 				printf(
508 				    "%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
509 				    p->p_pid, rval1, rval2, curthread);
510 				mutex_exit(&systrace_lock);
511 			}
512 #endif /* SYSCALLTRACE */
513 			rp->r_ps &= ~PS_C;
514 			rp->r_r0 = rval1;
515 			rp->r_r1 = rval2;
516 		} else {
517 			int sig;
518 #ifdef SYSCALLTRACE
519 			if (syscalltrace) {
520 				mutex_enter(&systrace_lock);
521 				printf("%d: error=%d, id 0x%p\n",
522 				    p->p_pid, error, curthread);
523 				mutex_exit(&systrace_lock);
524 			}
525 #endif /* SYSCALLTRACE */
526 			if (error == EINTR && t->t_activefd.a_stale)
527 				error = EBADF;
528 			if (error == EINTR &&
529 			    (sig = lwp->lwp_cursig) != 0 &&
530 			    sigismember(&PTOU(p)->u_sigrestart, sig) &&
531 			    PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
532 			    PTOU(p)->u_signal[sig - 1] != SIG_IGN)
533 				error = ERESTART;
534 			rp->r_r0 = error;
535 			rp->r_ps |= PS_C;
536 		}
537 	}
538 
539 	/*
540 	 * From the proc(4) manual page:
541 	 * When exit from a system call is being traced, the traced process
542 	 * stops on completion of the system call just prior to checking for
543 	 * signals and returning to user level.  At this point all return
544 	 * values have been stored into the traced process's saved registers.
545 	 */
546 	if (proc_stop) {
547 		mutex_enter(&p->p_lock);
548 		if (PTOU(p)->u_systrap &&
549 		    prismember(&PTOU(p)->u_exitmask, code))
550 			stop(PR_SYSEXIT, code);
551 		mutex_exit(&p->p_lock);
552 	}
553 
554 	/*
555 	 * If we are the parent returning from a successful
556 	 * vfork, wait for the child to exec or exit.
557 	 * This code must be here and not in the bowels of the system
558 	 * so that /proc can intercept exit from vfork in a timely way.
559 	 */
560 	if (t->t_flag & T_VFPARENT) {
561 		ASSERT(code == SYS_vfork || code == SYS_forksys);
562 		ASSERT(rp->r_r1 == 0 && error == 0);
563 		vfwait((pid_t)rval1);
564 		t->t_flag &= ~T_VFPARENT;
565 	}
566 
567 	/*
568 	 * If profiling is active, bill the current PC in user-land
569 	 * and keep reposting until profiling is disabled.
570 	 */
571 	if (p->p_prof.pr_scale) {
572 		if (lwp->lwp_oweupc)
573 			profil_tick(rp->r_pc);
574 		repost = 1;
575 	}
576 
577 sig_check:
578 	/*
579 	 * Reset flag for next time.
580 	 * We must do this after stopping on PR_SYSEXIT
581 	 * because /proc uses the information in lwp_eosys.
582 	 */
583 	lwp->lwp_eosys = NORMALRETURN;
584 	clear_stale_fd();
585 	t->t_flag &= ~T_FORKALL;
586 
587 	if (t->t_astflag | t->t_sig_check) {
588 		/*
589 		 * Turn off the AST flag before checking all the conditions that
590 		 * may have caused an AST.  This flag is on whenever a signal or
591 		 * unusual condition should be handled after the next trap or
592 		 * syscall.
593 		 */
594 		astoff(t);
595 		/*
596 		 * If a single-step trap occurred on a syscall (see trap())
597 		 * recognize it now.  Do this before checking for signals
598 		 * because deferred_singlestep_trap() may generate a SIGTRAP to
599 		 * the LWP or may otherwise mark the LWP to call issig(FORREAL).
600 		 */
601 		if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
602 			deferred_singlestep_trap((caddr_t)rp->r_pc);
603 
604 		t->t_sig_check = 0;
605 
606 		/*
607 		 * The following check is legal for the following reasons:
608 		 *	1) The thread we are checking, is ourselves, so there is
609 		 *	   no way the proc can go away.
610 		 *	2) The only time we need to be protected by the
611 		 *	   lock is if the binding is changed.
612 		 *
613 		 *	Note we will still take the lock and check the binding
614 		 *	if the condition was true without the lock held.  This
615 		 *	prevents lock contention among threads owned by the
616 		 * 	same proc.
617 		 */
618 
619 		if (curthread->t_proc_flag & TP_CHANGEBIND) {
620 			mutex_enter(&p->p_lock);
621 			if (curthread->t_proc_flag & TP_CHANGEBIND) {
622 				timer_lwpbind();
623 				curthread->t_proc_flag &= ~TP_CHANGEBIND;
624 			}
625 			mutex_exit(&p->p_lock);
626 		}
627 
628 		/*
629 		 * for kaio requests on the special kaio poll queue,
630 		 * copyout their results to user memory.
631 		 */
632 		if (p->p_aio)
633 			aio_cleanup(0);
634 		/*
635 		 * If this LWP was asked to hold, call holdlwp(), which will
636 		 * stop.  holdlwps() sets this up and calls pokelwps() which
637 		 * sets the AST flag.
638 		 *
639 		 * Also check TP_EXITLWP, since this is used by fresh new LWPs
640 		 * through lwp_rtt().  That flag is set if the lwp_create(2)
641 		 * syscall failed after creating the LWP.
642 		 */
643 		if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
644 			holdlwp();
645 
646 		/*
647 		 * All code that sets signals and makes ISSIG_PENDING
648 		 * evaluate true must set t_sig_check afterwards.
649 		 */
650 		if (ISSIG_PENDING(t, lwp, p)) {
651 			if (issig(FORREAL))
652 				psig();
653 			t->t_sig_check = 1;	/* recheck next time */
654 		}
655 
656 		if (sigprof) {
657 			int nargs = (code > 0 && code < NSYSCALL)?
658 			    LWP_GETSYSENT(lwp)[code].sy_narg : 0;
659 			realsigprof(code, nargs, error);
660 			t->t_sig_check = 1;	/* recheck next time */
661 		}
662 
663 		/*
664 		 * If a performance counter overflow interrupt was
665 		 * delivered *during* the syscall, then re-enable the
666 		 * AST so that we take a trip through trap() to cause
667 		 * the SIGEMT to be delivered.
668 		 */
669 		if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
670 			aston(t);
671 
672 		/*
673 		 * /proc can't enable/disable the trace bit itself
674 		 * because that could race with the call gate used by
675 		 * system calls via "lcall". If that happened, an
676 		 * invalid EFLAGS would result. prstep()/prnostep()
677 		 * therefore schedule an AST for the purpose.
678 		 */
679 		if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
680 			lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
681 			rp->r_ps |= PS_T;
682 		}
683 		if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
684 			lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
685 			rp->r_ps &= ~PS_T;
686 		}
687 	}
688 
689 	lwp->lwp_errno = 0;		/* clear error for next time */
690 
691 #ifndef NPROBE
692 	/* Kernel probe */
693 	if (tnf_tracing_active) {
694 		TNF_PROBE_3(syscall_end, "syscall thread", /* CSTYLED */,
695 		    tnf_long,	rval1,		rval1,
696 		    tnf_long,	rval2,		rval2,
697 		    tnf_long,	errno,		(long)error);
698 		repost = 1;
699 	}
700 #endif /* NPROBE */
701 
702 	/*
703 	 * Set state to LWP_USER here so preempt won't give us a kernel
704 	 * priority if it occurs after this point.  Call CL_TRAPRET() to
705 	 * restore the user-level priority.
706 	 *
707 	 * It is important that no locks (other than spinlocks) be entered
708 	 * after this point before returning to user mode (unless lwp_state
709 	 * is set back to LWP_SYS).
710 	 *
711 	 * XXX Sampled times past this point are charged to the user.
712 	 */
713 	lwp->lwp_state = LWP_USER;
714 
715 	if (t->t_trapret) {
716 		t->t_trapret = 0;
717 		thread_lock(t);
718 		CL_TRAPRET(t);
719 		thread_unlock(t);
720 	}
721 	if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ)
722 		preempt();
723 	prunstop();
724 
725 	lwp->lwp_errno = 0;		/* clear error for next time */
726 
727 	/*
728 	 * The thread lock must be held in order to clear sysnum and reset
729 	 * lwp_ap atomically with respect to other threads in the system that
730 	 * may be looking at the args via lwp_ap from get_syscall_args().
731 	 */
732 
733 	thread_lock(t);
734 	t->t_sysnum = 0;		/* no longer in a system call */
735 
736 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
737 #if defined(_LP64)
738 		/*
739 		 * In case the args were copied to the lwp, reset the
740 		 * pointer so the next syscall will have the right
741 		 * lwp_ap pointer.
742 		 */
743 		lwp->lwp_ap = (long *)&rp->r_rdi;
744 	} else {
745 #endif
746 		lwp->lwp_ap = NULL;	/* reset on every syscall entry */
747 	}
748 	thread_unlock(t);
749 
750 	lwp->lwp_argsaved = 0;
751 
752 	/*
753 	 * If there was a continuing reason for post-syscall processing,
754 	 * set the t_post_sys flag for the next system call.
755 	 */
756 	if (repost)
757 		t->t_post_sys = 1;
758 
759 	/*
760 	 * If there is a ustack registered for this lwp, and the stack rlimit
761 	 * has been altered, read in the ustack. If the saved stack rlimit
762 	 * matches the bounds of the ustack, update the ustack to reflect
763 	 * the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
764 	 * stack checking by setting the size to 0.
765 	 */
766 	if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
767 		rlim64_t new_size;
768 		caddr_t top;
769 		stack_t stk;
770 		struct rlimit64 rl;
771 
772 		mutex_enter(&p->p_lock);
773 		new_size = p->p_stk_ctl;
774 		top = p->p_usrstack;
775 		(void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
776 		mutex_exit(&p->p_lock);
777 
778 		if (rl.rlim_cur == RLIM64_INFINITY)
779 			new_size = 0;
780 
781 		if (copyin((stack_t *)lwp->lwp_ustack, &stk,
782 		    sizeof (stack_t)) == 0 &&
783 		    (stk.ss_size == lwp->lwp_old_stk_ctl ||
784 		    stk.ss_size == 0) &&
785 		    stk.ss_sp == top - stk.ss_size) {
786 			stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
787 			    stk.ss_size - (uintptr_t)new_size);
788 			stk.ss_size = new_size;
789 
790 			(void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
791 			    sizeof (stack_t));
792 		}
793 
794 		lwp->lwp_old_stk_ctl = 0;
795 	}
796 }
797 
798 /*
799  * Called from post_syscall() when a deferred singlestep is to be taken.
800  */
801 void
802 deferred_singlestep_trap(caddr_t pc)
803 {
804 	proc_t *p = ttoproc(curthread);
805 	klwp_t *lwp = ttolwp(curthread);
806 	pcb_t *pcb = &lwp->lwp_pcb;
807 	uint_t fault = 0;
808 	k_siginfo_t siginfo;
809 
810 	bzero(&siginfo, sizeof (siginfo));
811 
812 	/*
813 	 * If both NORMAL_STEP and WATCH_STEP are in
814 	 * effect, give precedence to WATCH_STEP.
815 	 * If neither is set, user must have set the
816 	 * PS_T bit in %efl; treat this as NORMAL_STEP.
817 	 */
818 	if ((fault = undo_watch_step(&siginfo)) == 0 &&
819 	    ((pcb->pcb_flags & NORMAL_STEP) ||
820 	    !(pcb->pcb_flags & WATCH_STEP))) {
821 		siginfo.si_signo = SIGTRAP;
822 		siginfo.si_code = TRAP_TRACE;
823 		siginfo.si_addr  = pc;
824 		fault = FLTTRACE;
825 	}
826 	pcb->pcb_flags &= ~(DEBUG_PENDING|NORMAL_STEP|WATCH_STEP);
827 
828 	if (fault) {
829 		/*
830 		 * Remember the fault and fault adddress
831 		 * for real-time (SIGPROF) profiling.
832 		 */
833 		lwp->lwp_lastfault = fault;
834 		lwp->lwp_lastfaddr = siginfo.si_addr;
835 		/*
836 		 * If a debugger has declared this fault to be an
837 		 * event of interest, stop the lwp.  Otherwise just
838 		 * deliver the associated signal.
839 		 */
840 		if (prismember(&p->p_fltmask, fault) &&
841 		    stop_on_fault(fault, &siginfo) == 0)
842 			siginfo.si_signo = 0;
843 	}
844 
845 	if (siginfo.si_signo)
846 		trapsig(&siginfo, 1);
847 }
848 
849 /*
850  * nonexistent system call-- signal lwp (may want to handle it)
851  * flag error if lwp won't see signal immediately
852  */
853 int64_t
854 nosys()
855 {
856 	tsignal(curthread, SIGSYS);
857 	return (set_errno(ENOSYS));
858 }
859 
860 /*
861  * Execute a 32-bit system call on behalf of the current thread.
862  */
863 void
864 dosyscall(void)
865 {
866 	/*
867 	 * Need space on the stack to store syscall arguments.
868 	 */
869 	long		syscall_args[MAXSYSARGS];
870 	struct sysent	*se;
871 	int64_t		ret;
872 
873 	syscall_mstate(LMS_TRAP, LMS_SYSTEM);
874 
875 	ASSERT(curproc->p_model == DATAMODEL_ILP32);
876 
877 	CPU_STATS_ENTER_K();
878 	CPU_STATS_ADDQ(CPU, sys, syscall, 1);
879 	CPU_STATS_EXIT_K();
880 
881 	se = syscall_entry(curthread, syscall_args);
882 
883 	/*
884 	 * syscall_entry() copied all 8 arguments into syscall_args.
885 	 */
886 	ret = se->sy_callc(syscall_args[0], syscall_args[1], syscall_args[2],
887 	    syscall_args[3], syscall_args[4], syscall_args[5], syscall_args[6],
888 	    syscall_args[7]);
889 
890 	syscall_exit(curthread, (int)ret & 0xffffffffu, (int)(ret >> 32));
891 	syscall_mstate(LMS_SYSTEM, LMS_TRAP);
892 }
893 
894 /*
895  * Get the arguments to the current system call. See comment atop
896  * save_syscall_args() regarding lwp_ap usage.
897  */
898 
899 uint_t
900 get_syscall_args(klwp_t *lwp, long *argp, int *nargsp)
901 {
902 	kthread_t	*t = lwptot(lwp);
903 	ulong_t	mask = 0xfffffffful;
904 	uint_t	code;
905 	long	*ap;
906 	int	nargs;
907 
908 #if defined(_LP64)
909 	if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
910 		mask = 0xfffffffffffffffful;
911 #endif
912 
913 	/*
914 	 * The thread lock must be held while looking at the arguments to ensure
915 	 * they don't go away via post_syscall().
916 	 * get_syscall_args() is the only routine to read them which is callable
917 	 * outside the LWP in question and hence the only one that must be
918 	 * synchronized in this manner.
919 	 */
920 	thread_lock(t);
921 
922 	code = t->t_sysnum;
923 	ap = lwp->lwp_ap;
924 
925 	thread_unlock(t);
926 
927 	if (code != 0 && code < NSYSCALL) {
928 		nargs = LWP_GETSYSENT(lwp)[code].sy_narg;
929 
930 		ASSERT(nargs <= MAXSYSARGS);
931 
932 		*nargsp = nargs;
933 		while (nargs-- > 0)
934 			*argp++ = *ap++ & mask;
935 	} else {
936 		*nargsp = 0;
937 	}
938 
939 	return (code);
940 }
941 
942 #ifdef _SYSCALL32_IMPL
943 /*
944  * Get the arguments to the current 32-bit system call.
945  */
946 uint_t
947 get_syscall32_args(klwp_t *lwp, int *argp, int *nargsp)
948 {
949 	long args[MAXSYSARGS];
950 	uint_t i, code;
951 
952 	code = get_syscall_args(lwp, args, nargsp);
953 
954 	for (i = 0; i != *nargsp; i++)
955 		*argp++ = (int)args[i];
956 	return (code);
957 }
958 #endif
959 
960 /*
961  * Save the system call arguments in a safe place.
962  *
963  * On the i386 kernel:
964  *
965  *	Copy the users args prior to changing the stack or stack pointer.
966  *	This is so /proc will be able to get a valid copy of the
967  *	args from the user stack even after the user stack has been changed.
968  *	Note that the kernel stack copy of the args may also have been
969  *	changed by a system call handler which takes C-style arguments.
970  *
971  *	Note that this may be called by stop() from trap().  In that case
972  *	t_sysnum will be zero (syscall_exit clears it), so no args will be
973  *	copied.
974  *
975  * On the amd64 kernel:
976  *
977  *	For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9
978  *	in the reg structure. If the user is going to change the argument
979  *	registers, rax, or the stack and might want to get the args (for
980  *	/proc tracing), it must copy the args elsewhere via save_syscall_args().
981  *
982  *	For 32-bit applications, lwp->lwp_ap normally points to a copy of
983  *	the system call arguments on the kernel stack made from the user
984  *	stack.  Copy the args prior to change the stack or stack pointer.
985  *	This is so /proc will be able to get a valid copy of the args
986  *	from the user stack even after that stack has been changed.
987  *
988  *	This may be called from stop() even when we're not in a system call.
989  *	Since there's no easy way to tell, this must be safe (not panic).
990  *	If the copyins get data faults, return non-zero.
991  */
992 int
993 save_syscall_args()
994 {
995 	kthread_t	*t = curthread;
996 	klwp_t		*lwp = ttolwp(t);
997 	uint_t		code = t->t_sysnum;
998 	uint_t		nargs;
999 
1000 	if (lwp->lwp_argsaved || code == 0)
1001 		return (0);		/* args already saved or not needed */
1002 
1003 	if (code >= NSYSCALL) {
1004 		nargs = 0;		/* illegal syscall */
1005 	} else {
1006 		struct sysent *se = LWP_GETSYSENT(lwp);
1007 		struct sysent *callp = se + code;
1008 
1009 		nargs = callp->sy_narg;
1010 		if (LOADABLE_SYSCALL(callp) && nargs == 0) {
1011 			krwlock_t	*module_lock;
1012 
1013 			/*
1014 			 * Find out how many arguments the system
1015 			 * call uses.
1016 			 *
1017 			 * We have the property that loaded syscalls
1018 			 * never change the number of arguments they
1019 			 * use after they've been loaded once.  This
1020 			 * allows us to stop for /proc tracing without
1021 			 * holding the module lock.
1022 			 * /proc is assured that sy_narg is valid.
1023 			 */
1024 			module_lock = lock_syscall(se, code);
1025 			nargs = callp->sy_narg;
1026 			rw_exit(module_lock);
1027 		}
1028 	}
1029 
1030 	/*
1031 	 * Fetch the system call arguments.
1032 	 */
1033 	if (nargs == 0)
1034 		goto out;
1035 
1036 	ASSERT(nargs <= MAXSYSARGS);
1037 
1038 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1039 #if defined(_LP64)
1040 		struct regs *rp = lwptoregs(lwp);
1041 
1042 		lwp->lwp_arg[0] = rp->r_rdi;
1043 		lwp->lwp_arg[1] = rp->r_rsi;
1044 		lwp->lwp_arg[2] = rp->r_rdx;
1045 		lwp->lwp_arg[3] = rp->r_rcx;
1046 		lwp->lwp_arg[4] = rp->r_r8;
1047 		lwp->lwp_arg[5] = rp->r_r9;
1048 		if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6))
1049 			return (-1);
1050 	} else {
1051 #endif
1052 		if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs))
1053 			return (-1);
1054 	}
1055 out:
1056 	lwp->lwp_ap = lwp->lwp_arg;
1057 	lwp->lwp_argsaved = 1;
1058 	t->t_post_sys = 1;	/* so lwp_ap will be reset */
1059 	return (0);
1060 }
1061 
1062 void
1063 reset_syscall_args(void)
1064 {
1065 	ttolwp(curthread)->lwp_argsaved = 0;
1066 }
1067 
1068 /*
1069  * Call a system call which takes a pointer to the user args struct and
1070  * a pointer to the return values.  This is a bit slower than the standard
1071  * C arg-passing method in some cases.
1072  */
1073 int64_t
1074 syscall_ap(void)
1075 {
1076 	uint_t	error;
1077 	struct sysent *callp;
1078 	rval_t	rval;
1079 	kthread_t *t = curthread;
1080 	klwp_t	*lwp = ttolwp(t);
1081 	struct regs *rp = lwptoregs(lwp);
1082 
1083 	callp = LWP_GETSYSENT(lwp) + t->t_sysnum;
1084 
1085 #if defined(__amd64)
1086 	/*
1087 	 * If the arguments don't fit in registers %rdi-%r9, make sure they
1088 	 * have been copied to the lwp_arg array.
1089 	 */
1090 	if (callp->sy_narg > 6 && save_syscall_args())
1091 		return ((int64_t)set_errno(EFAULT));
1092 #endif
1093 
1094 	rval.r_val1 = 0;
1095 	rval.r_val2 = rp->r_r1;
1096 	lwp->lwp_error = 0;	/* for old drivers */
1097 	error = (*(callp->sy_call))(lwp->lwp_ap, &rval);
1098 	if (error)
1099 		return ((longlong_t)set_errno(error));
1100 	return (rval.r_vals);
1101 }
1102 
1103 /*
1104  * Load system call module.
1105  *	Returns with pointer to held read lock for module.
1106  */
1107 static krwlock_t *
1108 lock_syscall(struct sysent *table, uint_t code)
1109 {
1110 	krwlock_t	*module_lock;
1111 	struct modctl	*modp;
1112 	int		id;
1113 	struct sysent   *callp;
1114 
1115 	callp = table + code;
1116 	module_lock = callp->sy_lock;
1117 
1118 	/*
1119 	 * Optimization to only call modload if we don't have a loaded
1120 	 * syscall.
1121 	 */
1122 	rw_enter(module_lock, RW_READER);
1123 	if (LOADED_SYSCALL(callp))
1124 		return (module_lock);
1125 	rw_exit(module_lock);
1126 
1127 	for (;;) {
1128 		if ((id = modload("sys", syscallnames[code])) == -1)
1129 			break;
1130 
1131 		/*
1132 		 * If we loaded successfully at least once, the modctl
1133 		 * will still be valid, so we try to grab it by filename.
1134 		 * If this call fails, it's because the mod_filename
1135 		 * was changed after the call to modload() (mod_hold_by_name()
1136 		 * is the likely culprit).  We can safely just take
1137 		 * another lap if this is the case;  the modload() will
1138 		 * change the mod_filename back to one by which we can
1139 		 * find the modctl.
1140 		 */
1141 		modp = mod_find_by_filename("sys", syscallnames[code]);
1142 
1143 		if (modp == NULL)
1144 			continue;
1145 
1146 		mutex_enter(&mod_lock);
1147 
1148 		if (!modp->mod_installed) {
1149 			mutex_exit(&mod_lock);
1150 			continue;
1151 		}
1152 		break;
1153 	}
1154 	rw_enter(module_lock, RW_READER);
1155 
1156 	if (id != -1)
1157 		mutex_exit(&mod_lock);
1158 
1159 	return (module_lock);
1160 }
1161 
1162 /*
1163  * Loadable syscall support.
1164  *	If needed, load the module, then reserve it by holding a read
1165  *	lock for the duration of the call.
1166  *	Later, if the syscall is not unloadable, it could patch the vector.
1167  */
1168 /*ARGSUSED*/
1169 int64_t
1170 loadable_syscall(
1171     long a0, long a1, long a2, long a3,
1172     long a4, long a5, long a6, long a7)
1173 {
1174 	klwp_t *lwp = ttolwp(curthread);
1175 	int64_t	rval;
1176 	struct sysent *callp;
1177 	struct sysent *se = LWP_GETSYSENT(lwp);
1178 	krwlock_t *module_lock;
1179 	int code, error = 0;
1180 	int64_t (*sy_call)();
1181 
1182 	code = curthread->t_sysnum;
1183 	callp = se + code;
1184 
1185 	/*
1186 	 * Try to autoload the system call if necessary
1187 	 */
1188 	module_lock = lock_syscall(se, code);
1189 	THREAD_KPRI_RELEASE();	/* drop priority given by rw_enter */
1190 
1191 	/*
1192 	 * we've locked either the loaded syscall or nosys
1193 	 */
1194 
1195 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1196 #if defined(_LP64)
1197 		if (callp->sy_flags & SE_ARGC) {
1198 			sy_call = (int64_t (*)())callp->sy_call;
1199 			rval = (*sy_call)(a0, a1, a2, a3, a4, a5);
1200 		} else
1201 			rval = syscall_ap();
1202 	} else {
1203 #endif
1204 		/*
1205 		 * Now that it's loaded, make sure enough args were copied.
1206 		 */
1207 		if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg))
1208 			error = EFAULT;
1209 		if (error) {
1210 			rval = set_errno(error);
1211 		} else if (callp->sy_flags & SE_ARGC) {
1212 			sy_call = (int64_t (*)())callp->sy_call;
1213 			rval = (*sy_call)(lwp->lwp_ap[0], lwp->lwp_ap[1],
1214 			    lwp->lwp_ap[2], lwp->lwp_ap[3], lwp->lwp_ap[4],
1215 			    lwp->lwp_ap[5]);
1216 		} else
1217 			rval = syscall_ap();
1218 	}
1219 
1220 	THREAD_KPRI_REQUEST();	/* regain priority from read lock */
1221 	rw_exit(module_lock);
1222 	return (rval);
1223 }
1224 
1225 /*
1226  * Indirect syscall handled in libc on x86 architectures
1227  */
1228 int64_t
1229 indir()
1230 {
1231 	return (nosys());
1232 }
1233 
1234 /*
1235  * set_errno - set an error return from the current system call.
1236  *	This could be a macro.
1237  *	This returns the value it is passed, so that the caller can
1238  *	use tail-recursion-elimination and do return (set_errno(ERRNO));
1239  */
1240 uint_t
1241 set_errno(uint_t error)
1242 {
1243 	ASSERT(error != 0);		/* must not be used to clear errno */
1244 
1245 	curthread->t_post_sys = 1;	/* have post_syscall do error return */
1246 	return (ttolwp(curthread)->lwp_errno = error);
1247 }
1248 
1249 /*
1250  * set_proc_pre_sys - Set pre-syscall processing for entire process.
1251  */
1252 void
1253 set_proc_pre_sys(proc_t *p)
1254 {
1255 	kthread_t	*t;
1256 	kthread_t	*first;
1257 
1258 	ASSERT(MUTEX_HELD(&p->p_lock));
1259 
1260 	t = first = p->p_tlist;
1261 	do {
1262 		t->t_pre_sys = 1;
1263 	} while ((t = t->t_forw) != first);
1264 }
1265 
1266 /*
1267  * set_proc_post_sys - Set post-syscall processing for entire process.
1268  */
1269 void
1270 set_proc_post_sys(proc_t *p)
1271 {
1272 	kthread_t	*t;
1273 	kthread_t	*first;
1274 
1275 	ASSERT(MUTEX_HELD(&p->p_lock));
1276 
1277 	t = first = p->p_tlist;
1278 	do {
1279 		t->t_post_sys = 1;
1280 	} while ((t = t->t_forw) != first);
1281 }
1282 
1283 /*
1284  * set_proc_sys - Set pre- and post-syscall processing for entire process.
1285  */
1286 void
1287 set_proc_sys(proc_t *p)
1288 {
1289 	kthread_t	*t;
1290 	kthread_t	*first;
1291 
1292 	ASSERT(MUTEX_HELD(&p->p_lock));
1293 
1294 	t = first = p->p_tlist;
1295 	do {
1296 		t->t_pre_sys = 1;
1297 		t->t_post_sys = 1;
1298 	} while ((t = t->t_forw) != first);
1299 }
1300 
1301 /*
1302  * set_all_proc_sys - set pre- and post-syscall processing flags for all
1303  * user processes.
1304  *
1305  * This is needed when auditing, tracing, or other facilities which affect
1306  * all processes are turned on.
1307  */
1308 void
1309 set_all_proc_sys()
1310 {
1311 	kthread_t	*t;
1312 	kthread_t	*first;
1313 
1314 	mutex_enter(&pidlock);
1315 	t = first = curthread;
1316 	do {
1317 		t->t_pre_sys = 1;
1318 		t->t_post_sys = 1;
1319 	} while ((t = t->t_next) != first);
1320 	mutex_exit(&pidlock);
1321 }
1322 
1323 /*
1324  * set_proc_ast - Set asynchronous service trap (AST) flag for all
1325  * threads in process.
1326  */
1327 void
1328 set_proc_ast(proc_t *p)
1329 {
1330 	kthread_t	*t;
1331 	kthread_t	*first;
1332 
1333 	ASSERT(MUTEX_HELD(&p->p_lock));
1334 
1335 	t = first = p->p_tlist;
1336 	do {
1337 		aston(t);
1338 	} while ((t = t->t_forw) != first);
1339 }
1340