xref: /titanic_44/usr/src/uts/intel/ia32/os/syscall.c (revision 45916cd2fec6e79bca5dee0421bd39e3c2910d1e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 
23 /*
24  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
25  * Use is subject to license terms.
26  */
27 
28 #pragma ident	"%Z%%M%	%I%	%E% SMI"
29 
30 #include <sys/param.h>
31 #include <sys/vmparam.h>
32 #include <sys/types.h>
33 #include <sys/sysmacros.h>
34 #include <sys/systm.h>
35 #include <sys/signal.h>
36 #include <sys/stack.h>
37 #include <sys/cred.h>
38 #include <sys/cmn_err.h>
39 #include <sys/user.h>
40 #include <sys/privregs.h>
41 #include <sys/psw.h>
42 #include <sys/debug.h>
43 #include <sys/errno.h>
44 #include <sys/proc.h>
45 #include <sys/modctl.h>
46 #include <sys/var.h>
47 #include <sys/inline.h>
48 #include <sys/syscall.h>
49 #include <sys/ucontext.h>
50 #include <sys/cpuvar.h>
51 #include <sys/siginfo.h>
52 #include <sys/trap.h>
53 #include <sys/vtrace.h>
54 #include <sys/sysinfo.h>
55 #include <sys/procfs.h>
56 #include <c2/audit.h>
57 #include <sys/modctl.h>
58 #include <sys/aio_impl.h>
59 #include <sys/tnf.h>
60 #include <sys/tnf_probe.h>
61 #include <sys/copyops.h>
62 #include <sys/priv.h>
63 #include <sys/msacct.h>
64 
65 int syscalltrace = 0;
66 #ifdef SYSCALLTRACE
67 static kmutex_t systrace_lock;		/* syscall tracing lock */
68 #else
69 #define	syscalltrace 0
70 #endif /* SYSCALLTRACE */
71 
72 typedef	int64_t (*llfcn_t)();	/* function returning long long */
73 
74 int pre_syscall(void);
75 void post_syscall(long rval1, long rval2);
76 static krwlock_t *lock_syscall(struct sysent *, uint_t);
77 static void deferred_singlestep_trap(caddr_t);
78 
79 #ifdef _SYSCALL32_IMPL
80 #define	LWP_GETSYSENT(lwp)	\
81 	(lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? sysent : sysent32)
82 #else
83 #define	LWP_GETSYSENT(lwp)	(sysent)
84 #endif
85 
86 /*
87  * Arrange for the real time profiling signal to be dispatched.
88  */
89 void
90 realsigprof(int sysnum, int error)
91 {
92 	proc_t *p;
93 	klwp_t *lwp;
94 
95 	if (curthread->t_rprof->rp_anystate == 0)
96 		return;
97 	p = ttoproc(curthread);
98 	lwp = ttolwp(curthread);
99 	mutex_enter(&p->p_lock);
100 	if (sigismember(&p->p_ignore, SIGPROF) ||
101 	    signal_is_blocked(curthread, SIGPROF)) {
102 		mutex_exit(&p->p_lock);
103 		return;
104 	}
105 	lwp->lwp_siginfo.si_signo = SIGPROF;
106 	lwp->lwp_siginfo.si_code = PROF_SIG;
107 	lwp->lwp_siginfo.si_errno = error;
108 	hrt2ts(gethrtime(), &lwp->lwp_siginfo.si_tstamp);
109 	lwp->lwp_siginfo.si_syscall = sysnum;
110 	lwp->lwp_siginfo.si_nsysarg = (sysnum > 0 && sysnum < NSYSCALL) ?
111 		LWP_GETSYSENT(lwp)[sysnum].sy_narg : 0;
112 	lwp->lwp_siginfo.si_fault = lwp->lwp_lastfault;
113 	lwp->lwp_siginfo.si_faddr = lwp->lwp_lastfaddr;
114 	lwp->lwp_lastfault = 0;
115 	lwp->lwp_lastfaddr = NULL;
116 	sigtoproc(p, curthread, SIGPROF);
117 	mutex_exit(&p->p_lock);
118 	ASSERT(lwp->lwp_cursig == 0);
119 	if (issig(FORREAL))
120 		psig();
121 	mutex_enter(&p->p_lock);
122 	lwp->lwp_siginfo.si_signo = 0;
123 	bzero(curthread->t_rprof, sizeof (*curthread->t_rprof));
124 	mutex_exit(&p->p_lock);
125 }
126 
127 /*
128  * If watchpoints are active, don't make copying in of
129  * system call arguments take a read watchpoint trap.
130  */
131 static int
132 copyin_args(struct regs *rp, long *ap, uint_t nargs)
133 {
134 	greg_t *sp = 1 + (greg_t *)rp->r_sp;		/* skip ret addr */
135 
136 	ASSERT(nargs <= MAXSYSARGS);
137 
138 	return (copyin_nowatch(sp, ap, nargs * sizeof (*sp)));
139 }
140 
141 #if defined(_SYSCALL32_IMPL)
142 static int
143 copyin_args32(struct regs *rp, long *ap, uint_t nargs)
144 {
145 	greg32_t *sp = 1 + (greg32_t *)rp->r_sp;	/* skip ret addr */
146 	uint32_t a32[MAXSYSARGS];
147 	int rc;
148 
149 	ASSERT(nargs <= MAXSYSARGS);
150 
151 	if ((rc = copyin_nowatch(sp, a32, nargs * sizeof (*sp))) == 0) {
152 		uint32_t *a32p = &a32[0];
153 
154 		while (nargs--)
155 			*ap++ = (ulong_t)*a32p++;
156 	}
157 	return (rc);
158 }
159 #define	COPYIN_ARGS32	copyin_args32
160 #else
161 #define	COPYIN_ARGS32	copyin_args
162 #endif
163 
164 /*
165  * Error handler for system calls where arg copy gets fault.
166  */
167 static longlong_t
168 syscall_err()
169 {
170 	return (0);
171 }
172 
173 /*
174  * Corresponding sysent entry to allow syscall_entry caller
175  * to invoke syscall_err.
176  */
177 static struct sysent sysent_err =  {
178 	0, SE_32RVAL1, NULL, NULL, (llfcn_t)syscall_err
179 };
180 
181 /*
182  * Called from syscall() when a non-trivial 32-bit system call occurs.
183  * 	Sets up the args and returns a pointer to the handler.
184  */
185 struct sysent *
186 syscall_entry(kthread_t *t, long *argp)
187 {
188 	klwp_t *lwp = ttolwp(t);
189 	struct regs *rp = lwptoregs(lwp);
190 	unsigned int code;
191 	struct sysent *callp;
192 	struct sysent *se = LWP_GETSYSENT(lwp);
193 	int error = 0;
194 	uint_t nargs;
195 
196 	ASSERT(t == curthread && curthread->t_schedflag & TS_DONT_SWAP);
197 
198 	lwp->lwp_ru.sysc++;
199 	lwp->lwp_eosys = NORMALRETURN;	/* assume this will be normal */
200 
201 	/*
202 	 * Set lwp_ap to point to the args, even if none are needed for this
203 	 * system call.  This is for the loadable-syscall case where the
204 	 * number of args won't be known until the system call is loaded, and
205 	 * also maintains a non-NULL lwp_ap setup for get_syscall_args(). Note
206 	 * that lwp_ap MUST be set to a non-NULL value _BEFORE_ t_sysnum is
207 	 * set to non-zero; otherwise get_syscall_args(), seeing a non-zero
208 	 * t_sysnum for this thread, will charge ahead and dereference lwp_ap.
209 	 */
210 	lwp->lwp_ap = argp;		/* for get_syscall_args */
211 
212 	code = rp->r_r0;
213 	t->t_sysnum = (short)code;
214 	callp = code >= NSYSCALL ? &nosys_ent : se + code;
215 
216 	if ((t->t_pre_sys | syscalltrace) != 0) {
217 		error = pre_syscall();
218 		/*
219 		 * Reset lwp_ap so that the args will be refetched if
220 		 * the lwp stopped for /proc purposes in pre_syscall().
221 		 */
222 		lwp->lwp_argsaved = 0;
223 		lwp->lwp_ap = argp;
224 		if (error)
225 			return (&sysent_err);	/* use dummy handler */
226 	}
227 
228 	/*
229 	 * Fetch the system call arguments.
230 	 * Note: for loadable system calls the number of arguments required
231 	 * may not be known at this point, and will be zero if the system call
232 	 * was never loaded.  Once the system call has been loaded, the number
233 	 * of args is not allowed to be changed.
234 	 */
235 	if ((nargs = (uint_t)callp->sy_narg) != 0 &&
236 	    COPYIN_ARGS32(rp, argp, nargs)) {
237 		(void) set_errno(EFAULT);
238 		return (&sysent_err);	/* use dummy handler */
239 	}
240 
241 	return (callp);		/* return sysent entry for caller */
242 }
243 
244 void
245 syscall_exit(kthread_t *t, long rval1, long rval2)
246 {
247 	/*
248 	 * Handle signals and other post-call events if necessary.
249 	 */
250 	if ((t->t_post_sys_ast | syscalltrace) == 0) {
251 		klwp_t *lwp = ttolwp(t);
252 		struct regs *rp = lwptoregs(lwp);
253 
254 		/*
255 		 * Normal return.
256 		 * Clear error indication and set return values.
257 		 */
258 		rp->r_ps &= ~PS_C;	/* reset carry bit */
259 		rp->r_r0 = rval1;
260 		rp->r_r1 = rval2;
261 		lwp->lwp_state = LWP_USER;
262 	} else
263 		post_syscall(rval1, rval2);
264 	t->t_sysnum = 0;		/* invalidate args */
265 }
266 
267 /*
268  * Perform pre-system-call processing, including stopping for tracing,
269  * auditing, etc.
270  *
271  * This routine is called only if the t_pre_sys flag is set. Any condition
272  * requiring pre-syscall handling must set the t_pre_sys flag. If the
273  * condition is persistent, this routine will repost t_pre_sys.
274  */
275 int
276 pre_syscall()
277 {
278 	kthread_t *t = curthread;
279 	unsigned code = t->t_sysnum;
280 	klwp_t *lwp = ttolwp(t);
281 	proc_t *p = ttoproc(t);
282 	int	repost;
283 
284 	t->t_pre_sys = repost = 0;	/* clear pre-syscall processing flag */
285 
286 	ASSERT(t->t_schedflag & TS_DONT_SWAP);
287 
288 #if defined(DEBUG)
289 	/*
290 	 * On the i386 kernel, lwp_ap points at the piece of the thread
291 	 * stack that we copy the users arguments into.
292 	 *
293 	 * On the amd64 kernel, the syscall arguments in the rdi..r9
294 	 * registers should be pointed at by lwp_ap.  If the args need to
295 	 * be copied so that those registers can be changed without losing
296 	 * the ability to get the args for /proc, they can be saved by
297 	 * save_syscall_args(), and lwp_ap will be restored by post_syscall().
298 	 */
299 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
300 #if defined(_LP64)
301 		ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi);
302 	} else {
303 #endif
304 		ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase &&
305 			(caddr_t)lwp->lwp_ap < t->t_stk);
306 	}
307 #endif	/* DEBUG */
308 
309 	/*
310 	 * Make sure the thread is holding the latest credentials for the
311 	 * process.  The credentials in the process right now apply to this
312 	 * thread for the entire system call.
313 	 */
314 	if (t->t_cred != p->p_cred) {
315 		cred_t *oldcred = t->t_cred;
316 		/*
317 		 * DTrace accesses t_cred in probe context.  t_cred must
318 		 * always be either NULL, or point to a valid, allocated cred
319 		 * structure.
320 		 */
321 		t->t_cred = crgetcred();
322 		crfree(oldcred);
323 	}
324 
325 	/*
326 	 * From the proc(4) manual page:
327 	 * When entry to a system call is being traced, the traced process
328 	 * stops after having begun the call to the system but before the
329 	 * system call arguments have been fetched from the process.
330 	 */
331 	if (PTOU(p)->u_systrap) {
332 		if (prismember(&PTOU(p)->u_entrymask, code)) {
333 			mutex_enter(&p->p_lock);
334 			/*
335 			 * Recheck stop condition, now that lock is held.
336 			 */
337 			if (PTOU(p)->u_systrap &&
338 			    prismember(&PTOU(p)->u_entrymask, code)) {
339 				stop(PR_SYSENTRY, code);
340 #if defined(_LP64)
341 				/*
342 				 * Must refetch args since they were
343 				 * possibly modified by /proc.
344 				 * Indicate that a valid copy is in registers.
345 				 */
346 				if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
347 					lwp->lwp_argsaved = 0;
348 					lwp->lwp_ap =
349 					    (long *)&lwptoregs(lwp)->r_rdi;
350 				}
351 #endif
352 			}
353 			mutex_exit(&p->p_lock);
354 		}
355 		repost = 1;
356 	}
357 
358 	if (lwp->lwp_sysabort) {
359 		/*
360 		 * lwp_sysabort may have been set via /proc while the process
361 		 * was stopped on PR_SYSENTRY.  If so, abort the system call.
362 		 * Override any error from the copyin() of the arguments.
363 		 */
364 		lwp->lwp_sysabort = 0;
365 		(void) set_errno(EINTR);	/* forces post_sys */
366 		t->t_pre_sys = 1;	/* repost anyway */
367 		return (1);		/* don't do system call, return EINTR */
368 	}
369 
370 #ifdef C2_AUDIT
371 	if (audit_active) {	/* begin auditing for this syscall */
372 		int error;
373 		if (error = audit_start(T_SYSCALL, code, 0, lwp)) {
374 			t->t_pre_sys = 1;	/* repost anyway */
375 			(void) set_errno(error);
376 			return (1);
377 		}
378 		repost = 1;
379 	}
380 #endif /* C2_AUDIT */
381 
382 #ifndef NPROBE
383 	/* Kernel probe */
384 	if (tnf_tracing_active) {
385 		TNF_PROBE_1(syscall_start, "syscall thread", /* CSTYLED */,
386 			tnf_sysnum,	sysnum,		t->t_sysnum);
387 		t->t_post_sys = 1;	/* make sure post_syscall runs */
388 		repost = 1;
389 	}
390 #endif /* NPROBE */
391 
392 #ifdef SYSCALLTRACE
393 	if (syscalltrace) {
394 		int i;
395 		long *ap;
396 		char *cp;
397 		char *sysname;
398 		struct sysent *callp;
399 
400 		if (code >= NSYSCALL)
401 			callp = &nosys_ent;	/* nosys has no args */
402 		else
403 			callp = LWP_GETSYSENT(lwp) + code;
404 		(void) save_syscall_args();
405 		mutex_enter(&systrace_lock);
406 		printf("%d: ", p->p_pid);
407 		if (code >= NSYSCALL)
408 			printf("0x%x", code);
409 		else {
410 			sysname = mod_getsysname(code);
411 			printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" :
412 			    sysname, code, callp->sy_callc);
413 		}
414 		cp = "(";
415 		for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) {
416 			printf("%s%lx", cp, *ap);
417 			cp = ", ";
418 		}
419 		if (i)
420 			printf(")");
421 		printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread);
422 		mutex_exit(&systrace_lock);
423 	}
424 #endif /* SYSCALLTRACE */
425 
426 	/*
427 	 * If there was a continuing reason for pre-syscall processing,
428 	 * set the t_pre_sys flag for the next system call.
429 	 */
430 	if (repost)
431 		t->t_pre_sys = 1;
432 	lwp->lwp_error = 0;	/* for old drivers */
433 	lwp->lwp_badpriv = PRIV_NONE;
434 	return (0);
435 }
436 
437 
438 /*
439  * Post-syscall processing.  Perform abnormal system call completion
440  * actions such as /proc tracing, profiling, signals, preemption, etc.
441  *
442  * This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
443  * Any condition requiring pre-syscall handling must set one of these.
444  * If the condition is persistent, this routine will repost t_post_sys.
445  */
446 void
447 post_syscall(long rval1, long rval2)
448 {
449 	kthread_t *t = curthread;
450 	klwp_t *lwp = ttolwp(t);
451 	proc_t *p = ttoproc(t);
452 	struct regs *rp = lwptoregs(lwp);
453 	uint_t	error;
454 	uint_t	code = t->t_sysnum;
455 	int	repost = 0;
456 	int	proc_stop = 0;		/* non-zero if stopping */
457 	int	sigprof = 0;		/* non-zero if sending SIGPROF */
458 
459 	t->t_post_sys = 0;
460 
461 	error = lwp->lwp_errno;
462 
463 	/*
464 	 * Code can be zero if this is a new LWP returning after a forkall(),
465 	 * other than the one which matches the one in the parent which called
466 	 * forkall().  In these LWPs, skip most of post-syscall activity.
467 	 */
468 	if (code == 0)
469 		goto sig_check;
470 
471 #ifdef C2_AUDIT
472 	if (audit_active) {	/* put out audit record for this syscall */
473 		rval_t	rval;
474 
475 		/* XX64 -- truncation of 64-bit return values? */
476 		rval.r_val1 = (int)rval1;
477 		rval.r_val2 = (int)rval2;
478 		audit_finish(T_SYSCALL, code, error, &rval);
479 		repost = 1;
480 	}
481 #endif /* C2_AUDIT */
482 
483 	if (curthread->t_pdmsg != NULL) {
484 		char *m = curthread->t_pdmsg;
485 
486 		uprintf("%s", m);
487 		kmem_free(m, strlen(m) + 1);
488 		curthread->t_pdmsg = NULL;
489 	}
490 
491 	/*
492 	 * If we're going to stop for /proc tracing, set the flag and
493 	 * save the arguments so that the return values don't smash them.
494 	 */
495 	if (PTOU(p)->u_systrap) {
496 		if (prismember(&PTOU(p)->u_exitmask, code)) {
497 			if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
498 				(void) save_syscall_args();
499 			proc_stop = 1;
500 		}
501 		repost = 1;
502 	}
503 
504 	/*
505 	 * Similarly check to see if SIGPROF might be sent.
506 	 */
507 	if (curthread->t_rprof != NULL &&
508 	    curthread->t_rprof->rp_anystate != 0) {
509 		if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
510 			(void) save_syscall_args();
511 		sigprof = 1;
512 	}
513 
514 	if (lwp->lwp_eosys == NORMALRETURN) {
515 		if (error == 0) {
516 #ifdef SYSCALLTRACE
517 			if (syscalltrace) {
518 				mutex_enter(&systrace_lock);
519 				printf(
520 				    "%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
521 				    p->p_pid, rval1, rval2, curthread);
522 				mutex_exit(&systrace_lock);
523 			}
524 #endif /* SYSCALLTRACE */
525 			rp->r_ps &= ~PS_C;
526 			rp->r_r0 = rval1;
527 			rp->r_r1 = rval2;
528 		} else {
529 			int sig;
530 #ifdef SYSCALLTRACE
531 			if (syscalltrace) {
532 				mutex_enter(&systrace_lock);
533 				printf("%d: error=%d, id 0x%p\n",
534 				    p->p_pid, error, curthread);
535 				mutex_exit(&systrace_lock);
536 			}
537 #endif /* SYSCALLTRACE */
538 			if (error == EINTR && t->t_activefd.a_stale)
539 				error = EBADF;
540 			if (error == EINTR &&
541 			    (sig = lwp->lwp_cursig) != 0 &&
542 			    sigismember(&PTOU(p)->u_sigrestart, sig) &&
543 			    PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
544 			    PTOU(p)->u_signal[sig - 1] != SIG_IGN)
545 				error = ERESTART;
546 			rp->r_r0 = error;
547 			rp->r_ps |= PS_C;
548 		}
549 	}
550 
551 	/*
552 	 * From the proc(4) manual page:
553 	 * When exit from a system call is being traced, the traced process
554 	 * stops on completion of the system call just prior to checking for
555 	 * signals and returning to user level.  At this point all return
556 	 * values have been stored into the traced process's saved registers.
557 	 */
558 	if (proc_stop) {
559 		mutex_enter(&p->p_lock);
560 		if (PTOU(p)->u_systrap &&
561 		    prismember(&PTOU(p)->u_exitmask, code))
562 			stop(PR_SYSEXIT, code);
563 		mutex_exit(&p->p_lock);
564 	}
565 
566 	/*
567 	 * If we are the parent returning from a successful
568 	 * vfork, wait for the child to exec or exit.
569 	 * This code must be here and not in the bowels of the system
570 	 * so that /proc can intercept exit from vfork in a timely way.
571 	 */
572 	if (code == SYS_vfork && rp->r_r1 == 0 && error == 0)
573 		vfwait((pid_t)rval1);
574 
575 	/*
576 	 * If profiling is active, bill the current PC in user-land
577 	 * and keep reposting until profiling is disabled.
578 	 */
579 	if (p->p_prof.pr_scale) {
580 		if (lwp->lwp_oweupc)
581 			profil_tick(rp->r_pc);
582 		repost = 1;
583 	}
584 
585 sig_check:
586 	/*
587 	 * Reset flag for next time.
588 	 * We must do this after stopping on PR_SYSEXIT
589 	 * because /proc uses the information in lwp_eosys.
590 	 */
591 	lwp->lwp_eosys = NORMALRETURN;
592 	clear_stale_fd();
593 	t->t_flag &= ~T_FORKALL;
594 
595 	/*
596 	 * If a single-step trap occurred on a syscall (see trap())
597 	 * recognize it now.  Do this before checking for signals
598 	 * because deferred_singlestep_trap() may generate a SIGTRAP to
599 	 * the LWP or may otherwise mark the LWP to call issig(FORREAL).
600 	 */
601 	if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
602 		deferred_singlestep_trap((caddr_t)rp->r_pc);
603 
604 	if (t->t_astflag | t->t_sig_check) {
605 		/*
606 		 * Turn off the AST flag before checking all the conditions that
607 		 * may have caused an AST.  This flag is on whenever a signal or
608 		 * unusual condition should be handled after the next trap or
609 		 * syscall.
610 		 */
611 		astoff(t);
612 		t->t_sig_check = 0;
613 
614 		mutex_enter(&p->p_lock);
615 		if (curthread->t_proc_flag & TP_CHANGEBIND) {
616 			timer_lwpbind();
617 			curthread->t_proc_flag &= ~TP_CHANGEBIND;
618 		}
619 		mutex_exit(&p->p_lock);
620 
621 		/*
622 		 * for kaio requests on the special kaio poll queue,
623 		 * copyout their results to user memory.
624 		 */
625 		if (p->p_aio)
626 			aio_cleanup(0);
627 		/*
628 		 * If this LWP was asked to hold, call holdlwp(), which will
629 		 * stop.  holdlwps() sets this up and calls pokelwps() which
630 		 * sets the AST flag.
631 		 *
632 		 * Also check TP_EXITLWP, since this is used by fresh new LWPs
633 		 * through lwp_rtt().  That flag is set if the lwp_create(2)
634 		 * syscall failed after creating the LWP.
635 		 */
636 		if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
637 			holdlwp();
638 
639 		/*
640 		 * All code that sets signals and makes ISSIG_PENDING
641 		 * evaluate true must set t_sig_check afterwards.
642 		 */
643 		if (ISSIG_PENDING(t, lwp, p)) {
644 			if (issig(FORREAL))
645 				psig();
646 			t->t_sig_check = 1;	/* recheck next time */
647 		}
648 
649 		if (sigprof) {
650 			realsigprof(code, error);
651 			t->t_sig_check = 1;	/* recheck next time */
652 		}
653 
654 		/*
655 		 * If a performance counter overflow interrupt was
656 		 * delivered *during* the syscall, then re-enable the
657 		 * AST so that we take a trip through trap() to cause
658 		 * the SIGEMT to be delivered.
659 		 */
660 		if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
661 			aston(t);
662 
663 		/*
664 		 * /proc can't enable/disable the trace bit itself
665 		 * because that could race with the call gate used by
666 		 * system calls via "lcall". If that happened, an
667 		 * invalid EFLAGS would result. prstep()/prnostep()
668 		 * therefore schedule an AST for the purpose.
669 		 */
670 		if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
671 			lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
672 			rp->r_ps |= PS_T;
673 		}
674 		if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
675 			lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
676 			rp->r_ps &= ~PS_T;
677 		}
678 	}
679 
680 	lwp->lwp_errno = 0;		/* clear error for next time */
681 
682 #ifndef NPROBE
683 	/* Kernel probe */
684 	if (tnf_tracing_active) {
685 		TNF_PROBE_3(syscall_end, "syscall thread", /* CSTYLED */,
686 			tnf_long,	rval1,		rval1,
687 			tnf_long,	rval2,		rval2,
688 			tnf_long,	errno,		(long)error);
689 		repost = 1;
690 	}
691 #endif /* NPROBE */
692 
693 	/*
694 	 * Set state to LWP_USER here so preempt won't give us a kernel
695 	 * priority if it occurs after this point.  Call CL_TRAPRET() to
696 	 * restore the user-level priority.
697 	 *
698 	 * It is important that no locks (other than spinlocks) be entered
699 	 * after this point before returning to user mode (unless lwp_state
700 	 * is set back to LWP_SYS).
701 	 *
702 	 * XXX Sampled times past this point are charged to the user.
703 	 */
704 	lwp->lwp_state = LWP_USER;
705 
706 	if (t->t_trapret) {
707 		t->t_trapret = 0;
708 		thread_lock(t);
709 		CL_TRAPRET(t);
710 		thread_unlock(t);
711 	}
712 	if (CPU->cpu_runrun)
713 		preempt();
714 
715 	lwp->lwp_errno = 0;		/* clear error for next time */
716 
717 	/*
718 	 * The thread lock must be held in order to clear sysnum and reset
719 	 * lwp_ap atomically with respect to other threads in the system that
720 	 * may be looking at the args via lwp_ap from get_syscall_args().
721 	 */
722 
723 	thread_lock(t);
724 	t->t_sysnum = 0;		/* no longer in a system call */
725 
726 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
727 #if defined(_LP64)
728 		/*
729 		 * In case the args were copied to the lwp, reset the
730 		 * pointer so the next syscall will have the right
731 		 * lwp_ap pointer.
732 		 */
733 		lwp->lwp_ap = (long *)&rp->r_rdi;
734 	} else {
735 #endif
736 		lwp->lwp_ap = NULL;	/* reset on every syscall entry */
737 	}
738 	thread_unlock(t);
739 
740 	lwp->lwp_argsaved = 0;
741 
742 	/*
743 	 * If there was a continuing reason for post-syscall processing,
744 	 * set the t_post_sys flag for the next system call.
745 	 */
746 	if (repost)
747 		t->t_post_sys = 1;
748 
749 	/*
750 	 * If there is a ustack registered for this lwp, and the stack rlimit
751 	 * has been altered, read in the ustack. If the saved stack rlimit
752 	 * matches the bounds of the ustack, update the ustack to reflect
753 	 * the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
754 	 * stack checking by setting the size to 0.
755 	 */
756 	if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
757 		rlim64_t new_size;
758 		caddr_t top;
759 		stack_t stk;
760 		struct rlimit64 rl;
761 
762 		mutex_enter(&p->p_lock);
763 		new_size = p->p_stk_ctl;
764 		top = p->p_usrstack;
765 		(void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
766 		mutex_exit(&p->p_lock);
767 
768 		if (rl.rlim_cur == RLIM64_INFINITY)
769 			new_size = 0;
770 
771 		if (copyin((stack_t *)lwp->lwp_ustack, &stk,
772 		    sizeof (stack_t)) == 0 &&
773 		    (stk.ss_size == lwp->lwp_old_stk_ctl ||
774 			stk.ss_size == 0) &&
775 		    stk.ss_sp == top - stk.ss_size) {
776 			stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
777 			    stk.ss_size - (uintptr_t)new_size);
778 			stk.ss_size = new_size;
779 
780 			(void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
781 			    sizeof (stack_t));
782 		}
783 
784 		lwp->lwp_old_stk_ctl = 0;
785 	}
786 }
787 
788 /*
789  * Called from post_syscall() when a deferred singlestep is to be taken.
790  */
791 static void
792 deferred_singlestep_trap(caddr_t pc)
793 {
794 	proc_t *p = ttoproc(curthread);
795 	klwp_t *lwp = ttolwp(curthread);
796 	pcb_t *pcb = &lwp->lwp_pcb;
797 	uint_t fault = 0;
798 	k_siginfo_t siginfo;
799 
800 	bzero(&siginfo, sizeof (siginfo));
801 
802 	/*
803 	 * If both NORMAL_STEP and WATCH_STEP are in
804 	 * effect, give precedence to NORMAL_STEP.
805 	 * If neither is set, user must have set the
806 	 * PS_T bit in %efl; treat this as NORMAL_STEP.
807 	 */
808 	if ((pcb->pcb_flags & NORMAL_STEP) ||
809 	    !(pcb->pcb_flags & WATCH_STEP)) {
810 		siginfo.si_signo = SIGTRAP;
811 		siginfo.si_code = TRAP_TRACE;
812 		siginfo.si_addr  = pc;
813 		fault = FLTTRACE;
814 		if (pcb->pcb_flags & WATCH_STEP)
815 			(void) undo_watch_step(NULL);
816 	} else {
817 		fault = undo_watch_step(&siginfo);
818 	}
819 	pcb->pcb_flags &= ~(DEBUG_PENDING|NORMAL_STEP|WATCH_STEP);
820 
821 	if (fault) {
822 		/*
823 		 * Remember the fault and fault adddress
824 		 * for real-time (SIGPROF) profiling.
825 		 */
826 		lwp->lwp_lastfault = fault;
827 		lwp->lwp_lastfaddr = siginfo.si_addr;
828 		/*
829 		 * If a debugger has declared this fault to be an
830 		 * event of interest, stop the lwp.  Otherwise just
831 		 * deliver the associated signal.
832 		 */
833 		if (prismember(&p->p_fltmask, fault) &&
834 		    stop_on_fault(fault, &siginfo) == 0)
835 			siginfo.si_signo = 0;
836 	}
837 
838 	if (siginfo.si_signo)
839 		trapsig(&siginfo, 1);
840 }
841 
842 /*
843  * nonexistent system call-- signal lwp (may want to handle it)
844  * flag error if lwp won't see signal immediately
845  */
846 int64_t
847 nosys()
848 {
849 	tsignal(curthread, SIGSYS);
850 	return (set_errno(ENOSYS));
851 }
852 
853 /*
854  * Execute a 32-bit system call on behalf of the current thread.
855  */
856 void
857 dosyscall(void)
858 {
859 	/*
860 	 * Need space on the stack to store syscall arguments.
861 	 */
862 	long		syscall_args[MAXSYSARGS];
863 	struct sysent	*se;
864 	int64_t		ret;
865 
866 	syscall_mstate(LMS_TRAP, LMS_SYSTEM);
867 
868 	ASSERT(curproc->p_model == DATAMODEL_ILP32);
869 
870 	CPU_STATS_ENTER_K();
871 	CPU_STATS_ADDQ(CPU, sys, syscall, 1);
872 	CPU_STATS_EXIT_K();
873 
874 	se = syscall_entry(curthread, syscall_args);
875 
876 	/*
877 	 * syscall_entry() copied all 8 arguments into syscall_args.
878 	 */
879 	ret = se->sy_callc(syscall_args[0], syscall_args[1], syscall_args[2],
880 	    syscall_args[3], syscall_args[4], syscall_args[5], syscall_args[6],
881 	    syscall_args[7]);
882 
883 	syscall_exit(curthread, (int)ret & 0xffffffffu, (int)(ret >> 32));
884 	syscall_mstate(LMS_SYSTEM, LMS_TRAP);
885 }
886 
887 /*
888  * Get the arguments to the current system call. See comment atop
889  * save_syscall_args() regarding lwp_ap usage.
890  */
891 
892 uint_t
893 get_syscall_args(klwp_t *lwp, long *argp, int *nargsp)
894 {
895 	kthread_t	*t = lwptot(lwp);
896 	ulong_t	mask = 0xfffffffful;
897 	uint_t	code;
898 	long	*ap;
899 	int	nargs;
900 
901 #if defined(_LP64)
902 	if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
903 		mask = 0xfffffffffffffffful;
904 #endif
905 
906 	/*
907 	 * The thread lock must be held while looking at the arguments to ensure
908 	 * they don't go away via post_syscall().
909 	 * get_syscall_args() is the only routine to read them which is callable
910 	 * outside the LWP in question and hence the only one that must be
911 	 * synchronized in this manner.
912 	 */
913 	thread_lock(t);
914 
915 	code = t->t_sysnum;
916 	ap = lwp->lwp_ap;
917 
918 	thread_unlock(t);
919 
920 	if (code != 0 && code < NSYSCALL) {
921 		nargs = LWP_GETSYSENT(lwp)[code].sy_narg;
922 
923 		ASSERT(nargs <= MAXSYSARGS);
924 
925 		*nargsp = nargs;
926 		while (nargs-- > 0)
927 			*argp++ = *ap++ & mask;
928 	} else {
929 		*nargsp = 0;
930 	}
931 
932 	return (code);
933 }
934 
935 #ifdef _SYSCALL32_IMPL
936 /*
937  * Get the arguments to the current 32-bit system call.
938  */
939 uint_t
940 get_syscall32_args(klwp_t *lwp, int *argp, int *nargsp)
941 {
942 	long args[MAXSYSARGS];
943 	uint_t i, code;
944 
945 	code = get_syscall_args(lwp, args, nargsp);
946 
947 	for (i = 0; i != *nargsp; i++)
948 		*argp++ = (int)args[i];
949 	return (code);
950 }
951 #endif
952 
953 /*
954  * Save the system call arguments in a safe place.
955  *
956  * On the i386 kernel:
957  *
958  *	Copy the users args prior to changing the stack or stack pointer.
959  *	This is so /proc will be able to get a valid copy of the
960  *	args from the user stack even after the user stack has been changed.
961  *	Note that the kernel stack copy of the args may also have been
962  *	changed by a system call handler which takes C-style arguments.
963  *
964  *	Note that this may be called by stop() from trap().  In that case
965  *	t_sysnum will be zero (syscall_exit clears it), so no args will be
966  *	copied.
967  *
968  * On the amd64 kernel:
969  *
970  *	For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9
971  *	in the reg structure. If the user is going to change the argument
972  *	registers, rax, or the stack and might want to get the args (for
973  *	/proc tracing), it must copy the args elsewhere via save_syscall_args().
974  *
975  *	For 32-bit applications, lwp->lwp_ap normally points to a copy of
976  *	the system call arguments on the kernel stack made from the user
977  *	stack.  Copy the args prior to change the stack or stack pointer.
978  *	This is so /proc will be able to get a valid copy of the args
979  *	from the user stack even after that stack has been changed.
980  *
981  *	This may be called from stop() even when we're not in a system call.
982  *	Since there's no easy way to tell, this must be safe (not panic).
983  *	If the copyins get data faults, return non-zero.
984  */
985 int
986 save_syscall_args()
987 {
988 	kthread_t	*t = curthread;
989 	klwp_t		*lwp = ttolwp(t);
990 	uint_t		code = t->t_sysnum;
991 	uint_t		nargs;
992 
993 	if (lwp->lwp_argsaved || code == 0)
994 		return (0);		/* args already saved or not needed */
995 
996 	if (code >= NSYSCALL) {
997 		nargs = 0;		/* illegal syscall */
998 	} else {
999 		struct sysent *se = LWP_GETSYSENT(lwp);
1000 		struct sysent *callp = se + code;
1001 
1002 		nargs = callp->sy_narg;
1003 		if (LOADABLE_SYSCALL(callp) && nargs == 0) {
1004 			krwlock_t	*module_lock;
1005 
1006 			/*
1007 			 * Find out how many arguments the system
1008 			 * call uses.
1009 			 *
1010 			 * We have the property that loaded syscalls
1011 			 * never change the number of arguments they
1012 			 * use after they've been loaded once.  This
1013 			 * allows us to stop for /proc tracing without
1014 			 * holding the module lock.
1015 			 * /proc is assured that sy_narg is valid.
1016 			 */
1017 			module_lock = lock_syscall(se, code);
1018 			nargs = callp->sy_narg;
1019 			rw_exit(module_lock);
1020 		}
1021 	}
1022 
1023 	/*
1024 	 * Fetch the system call arguments.
1025 	 */
1026 	if (nargs == 0)
1027 		goto out;
1028 
1029 	ASSERT(nargs <= MAXSYSARGS);
1030 
1031 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1032 #if defined(_LP64)
1033 		struct regs *rp = lwptoregs(lwp);
1034 
1035 		lwp->lwp_arg[0] = rp->r_rdi;
1036 		lwp->lwp_arg[1] = rp->r_rsi;
1037 		lwp->lwp_arg[2] = rp->r_rdx;
1038 		lwp->lwp_arg[3] = rp->r_rcx;
1039 		lwp->lwp_arg[4] = rp->r_r8;
1040 		lwp->lwp_arg[5] = rp->r_r9;
1041 		if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6))
1042 			return (-1);
1043 	} else {
1044 #endif
1045 		if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs))
1046 			return (-1);
1047 	}
1048 out:
1049 	lwp->lwp_ap = lwp->lwp_arg;
1050 	lwp->lwp_argsaved = 1;
1051 	t->t_post_sys = 1;	/* so lwp_ap will be reset */
1052 	return (0);
1053 }
1054 
1055 void
1056 reset_syscall_args(void)
1057 {
1058 	ttolwp(curthread)->lwp_argsaved = 0;
1059 }
1060 
1061 /*
1062  * Call a system call which takes a pointer to the user args struct and
1063  * a pointer to the return values.  This is a bit slower than the standard
1064  * C arg-passing method in some cases.
1065  */
1066 int64_t
1067 syscall_ap(void)
1068 {
1069 	uint_t	error;
1070 	struct sysent *callp;
1071 	rval_t	rval;
1072 	kthread_t *t = curthread;
1073 	klwp_t	*lwp = ttolwp(t);
1074 	struct regs *rp = lwptoregs(lwp);
1075 
1076 	callp = LWP_GETSYSENT(lwp) + t->t_sysnum;
1077 
1078 #if defined(__amd64)
1079 	/*
1080 	 * If the arguments don't fit in registers %rdi-%r9, make sure they
1081 	 * have been copied to the lwp_arg array.
1082 	 */
1083 	if (callp->sy_narg > 6 && save_syscall_args())
1084 		return ((int64_t)set_errno(EFAULT));
1085 #endif
1086 
1087 	rval.r_val1 = 0;
1088 	rval.r_val2 = rp->r_r1;
1089 	lwp->lwp_error = 0;	/* for old drivers */
1090 	error = (*(callp->sy_call))(lwp->lwp_ap, &rval);
1091 	if (error)
1092 		return ((longlong_t)set_errno(error));
1093 	return (rval.r_vals);
1094 }
1095 
1096 /*
1097  * Load system call module.
1098  *	Returns with pointer to held read lock for module.
1099  */
1100 static krwlock_t *
1101 lock_syscall(struct sysent *table, uint_t code)
1102 {
1103 	krwlock_t	*module_lock;
1104 	struct modctl	*modp;
1105 	int		id;
1106 	struct sysent   *callp;
1107 
1108 	callp = table + code;
1109 	module_lock = callp->sy_lock;
1110 
1111 	/*
1112 	 * Optimization to only call modload if we don't have a loaded
1113 	 * syscall.
1114 	 */
1115 	rw_enter(module_lock, RW_READER);
1116 	if (LOADED_SYSCALL(callp))
1117 		return (module_lock);
1118 	rw_exit(module_lock);
1119 
1120 	for (;;) {
1121 		if ((id = modload("sys", syscallnames[code])) == -1)
1122 			break;
1123 
1124 		/*
1125 		 * If we loaded successfully at least once, the modctl
1126 		 * will still be valid, so we try to grab it by filename.
1127 		 * If this call fails, it's because the mod_filename
1128 		 * was changed after the call to modload() (mod_hold_by_name()
1129 		 * is the likely culprit).  We can safely just take
1130 		 * another lap if this is the case;  the modload() will
1131 		 * change the mod_filename back to one by which we can
1132 		 * find the modctl.
1133 		 */
1134 		modp = mod_find_by_filename("sys", syscallnames[code]);
1135 
1136 		if (modp == NULL)
1137 			continue;
1138 
1139 		mutex_enter(&mod_lock);
1140 
1141 		if (!modp->mod_installed) {
1142 			mutex_exit(&mod_lock);
1143 			continue;
1144 		}
1145 		break;
1146 	}
1147 	rw_enter(module_lock, RW_READER);
1148 
1149 	if (id != -1)
1150 		mutex_exit(&mod_lock);
1151 
1152 	return (module_lock);
1153 }
1154 
1155 /*
1156  * Loadable syscall support.
1157  *	If needed, load the module, then reserve it by holding a read
1158  *	lock for the duration of the call.
1159  *	Later, if the syscall is not unloadable, it could patch the vector.
1160  */
1161 /*ARGSUSED*/
1162 int64_t
1163 loadable_syscall(
1164     long a0, long a1, long a2, long a3,
1165     long a4, long a5, long a6, long a7)
1166 {
1167 	klwp_t *lwp = ttolwp(curthread);
1168 	int64_t	rval;
1169 	struct sysent *callp;
1170 	struct sysent *se = LWP_GETSYSENT(lwp);
1171 	krwlock_t *module_lock;
1172 	int code, error = 0;
1173 	int64_t (*sy_call)();
1174 
1175 	code = curthread->t_sysnum;
1176 	callp = se + code;
1177 
1178 	/*
1179 	 * Try to autoload the system call if necessary
1180 	 */
1181 	module_lock = lock_syscall(se, code);
1182 	THREAD_KPRI_RELEASE();	/* drop priority given by rw_enter */
1183 
1184 	/*
1185 	 * we've locked either the loaded syscall or nosys
1186 	 */
1187 
1188 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1189 #if defined(_LP64)
1190 		if (callp->sy_flags & SE_ARGC) {
1191 			sy_call = (int64_t (*)())callp->sy_call;
1192 			rval = (*sy_call)(a0, a1, a2, a3, a4, a5);
1193 		} else
1194 			rval = syscall_ap();
1195 	} else {
1196 #endif
1197 		/*
1198 		 * Now that it's loaded, make sure enough args were copied.
1199 		 */
1200 		if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg))
1201 			error = EFAULT;
1202 		if (error) {
1203 			rval = set_errno(error);
1204 		} else if (callp->sy_flags & SE_ARGC) {
1205 			sy_call = (int64_t (*)())callp->sy_call;
1206 			rval = (*sy_call)(lwp->lwp_ap[0], lwp->lwp_ap[1],
1207 			    lwp->lwp_ap[2], lwp->lwp_ap[3], lwp->lwp_ap[4],
1208 			    lwp->lwp_ap[5]);
1209 		} else
1210 			rval = syscall_ap();
1211 	}
1212 
1213 	THREAD_KPRI_REQUEST();	/* regain priority from read lock */
1214 	rw_exit(module_lock);
1215 	return (rval);
1216 }
1217 
1218 /*
1219  * Indirect syscall handled in libc on x86 architectures
1220  */
1221 int64_t
1222 indir()
1223 {
1224 	return (nosys());
1225 }
1226 
1227 /*
1228  * set_errno - set an error return from the current system call.
1229  *	This could be a macro.
1230  *	This returns the value it is passed, so that the caller can
1231  *	use tail-recursion-elimination and do return (set_errno(ERRNO));
1232  */
1233 uint_t
1234 set_errno(uint_t error)
1235 {
1236 	ASSERT(error != 0);		/* must not be used to clear errno */
1237 
1238 	curthread->t_post_sys = 1;	/* have post_syscall do error return */
1239 	return (ttolwp(curthread)->lwp_errno = error);
1240 }
1241 
1242 /*
1243  * set_proc_pre_sys - Set pre-syscall processing for entire process.
1244  */
1245 void
1246 set_proc_pre_sys(proc_t *p)
1247 {
1248 	kthread_t	*t;
1249 	kthread_t	*first;
1250 
1251 	ASSERT(MUTEX_HELD(&p->p_lock));
1252 
1253 	t = first = p->p_tlist;
1254 	do {
1255 		t->t_pre_sys = 1;
1256 	} while ((t = t->t_forw) != first);
1257 }
1258 
1259 /*
1260  * set_proc_post_sys - Set post-syscall processing for entire process.
1261  */
1262 void
1263 set_proc_post_sys(proc_t *p)
1264 {
1265 	kthread_t	*t;
1266 	kthread_t	*first;
1267 
1268 	ASSERT(MUTEX_HELD(&p->p_lock));
1269 
1270 	t = first = p->p_tlist;
1271 	do {
1272 		t->t_post_sys = 1;
1273 	} while ((t = t->t_forw) != first);
1274 }
1275 
1276 /*
1277  * set_proc_sys - Set pre- and post-syscall processing for entire process.
1278  */
1279 void
1280 set_proc_sys(proc_t *p)
1281 {
1282 	kthread_t	*t;
1283 	kthread_t	*first;
1284 
1285 	ASSERT(MUTEX_HELD(&p->p_lock));
1286 
1287 	t = first = p->p_tlist;
1288 	do {
1289 		t->t_pre_sys = 1;
1290 		t->t_post_sys = 1;
1291 	} while ((t = t->t_forw) != first);
1292 }
1293 
1294 /*
1295  * set_all_proc_sys - set pre- and post-syscall processing flags for all
1296  * user processes.
1297  *
1298  * This is needed when auditing, tracing, or other facilities which affect
1299  * all processes are turned on.
1300  */
1301 void
1302 set_all_proc_sys()
1303 {
1304 	kthread_t	*t;
1305 	kthread_t	*first;
1306 
1307 	mutex_enter(&pidlock);
1308 	t = first = curthread;
1309 	do {
1310 		t->t_pre_sys = 1;
1311 		t->t_post_sys = 1;
1312 	} while ((t = t->t_next) != first);
1313 	mutex_exit(&pidlock);
1314 }
1315 
1316 /*
1317  * set_proc_ast - Set asynchronous service trap (AST) flag for all
1318  * threads in process.
1319  */
1320 void
1321 set_proc_ast(proc_t *p)
1322 {
1323 	kthread_t	*t;
1324 	kthread_t	*first;
1325 
1326 	ASSERT(MUTEX_HELD(&p->p_lock));
1327 
1328 	t = first = p->p_tlist;
1329 	do {
1330 		aston(t);
1331 	} while ((t = t->t_forw) != first);
1332 }
1333