1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 #include <sys/param.h>
28 #include <sys/vmparam.h>
29 #include <sys/types.h>
30 #include <sys/sysmacros.h>
31 #include <sys/systm.h>
32 #include <sys/signal.h>
33 #include <sys/stack.h>
34 #include <sys/cred.h>
35 #include <sys/cmn_err.h>
36 #include <sys/user.h>
37 #include <sys/privregs.h>
38 #include <sys/psw.h>
39 #include <sys/debug.h>
40 #include <sys/errno.h>
41 #include <sys/proc.h>
42 #include <sys/modctl.h>
43 #include <sys/var.h>
44 #include <sys/inline.h>
45 #include <sys/syscall.h>
46 #include <sys/ucontext.h>
47 #include <sys/cpuvar.h>
48 #include <sys/siginfo.h>
49 #include <sys/trap.h>
50 #include <sys/vtrace.h>
51 #include <sys/sysinfo.h>
52 #include <sys/procfs.h>
53 #include <sys/prsystm.h>
54 #include <c2/audit.h>
55 #include <sys/modctl.h>
56 #include <sys/aio_impl.h>
57 #include <sys/copyops.h>
58 #include <sys/priv.h>
59 #include <sys/msacct.h>
60
61 int syscalltrace = 0;
62 #ifdef SYSCALLTRACE
63 static kmutex_t systrace_lock; /* syscall tracing lock */
64 #else
65 #define syscalltrace 0
66 #endif /* SYSCALLTRACE */
67
68 typedef int64_t (*llfcn_t)(); /* function returning long long */
69
70 int pre_syscall(void);
71 void post_syscall(long rval1, long rval2);
72 static krwlock_t *lock_syscall(struct sysent *, uint_t);
73 void deferred_singlestep_trap(caddr_t);
74
75 #ifdef _SYSCALL32_IMPL
76 #define LWP_GETSYSENT(lwp) \
77 (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? sysent : sysent32)
78 #else
79 #define LWP_GETSYSENT(lwp) (sysent)
80 #endif
81
82 /*
83 * If watchpoints are active, don't make copying in of
84 * system call arguments take a read watchpoint trap.
85 */
86 static int
copyin_args(struct regs * rp,long * ap,uint_t nargs)87 copyin_args(struct regs *rp, long *ap, uint_t nargs)
88 {
89 greg_t *sp = 1 + (greg_t *)rp->r_sp; /* skip ret addr */
90
91 ASSERT(nargs <= MAXSYSARGS);
92
93 return (copyin_nowatch(sp, ap, nargs * sizeof (*sp)));
94 }
95
96 #if defined(_SYSCALL32_IMPL)
97 static int
copyin_args32(struct regs * rp,long * ap,uint_t nargs)98 copyin_args32(struct regs *rp, long *ap, uint_t nargs)
99 {
100 greg32_t *sp = 1 + (greg32_t *)rp->r_sp; /* skip ret addr */
101 uint32_t a32[MAXSYSARGS];
102 int rc;
103
104 ASSERT(nargs <= MAXSYSARGS);
105
106 if ((rc = copyin_nowatch(sp, a32, nargs * sizeof (*sp))) == 0) {
107 uint32_t *a32p = &a32[0];
108
109 while (nargs--)
110 *ap++ = (ulong_t)*a32p++;
111 }
112 return (rc);
113 }
114 #define COPYIN_ARGS32 copyin_args32
115 #else
116 #define COPYIN_ARGS32 copyin_args
117 #endif
118
119 /*
120 * Error handler for system calls where arg copy gets fault.
121 */
122 static longlong_t
syscall_err()123 syscall_err()
124 {
125 return (0);
126 }
127
128 /*
129 * Corresponding sysent entry to allow syscall_entry caller
130 * to invoke syscall_err.
131 */
132 static struct sysent sysent_err = {
133 0, SE_32RVAL1, NULL, NULL, (llfcn_t)syscall_err
134 };
135
136 /*
137 * Called from syscall() when a non-trivial 32-bit system call occurs.
138 * Sets up the args and returns a pointer to the handler.
139 */
140 struct sysent *
syscall_entry(kthread_t * t,long * argp)141 syscall_entry(kthread_t *t, long *argp)
142 {
143 klwp_t *lwp = ttolwp(t);
144 struct regs *rp = lwptoregs(lwp);
145 unsigned int code;
146 struct sysent *callp;
147 struct sysent *se = LWP_GETSYSENT(lwp);
148 int error = 0;
149 uint_t nargs;
150
151 ASSERT(t == curthread && curthread->t_schedflag & TS_DONT_SWAP);
152
153 lwp->lwp_ru.sysc++;
154 lwp->lwp_eosys = NORMALRETURN; /* assume this will be normal */
155
156 /*
157 * Set lwp_ap to point to the args, even if none are needed for this
158 * system call. This is for the loadable-syscall case where the
159 * number of args won't be known until the system call is loaded, and
160 * also maintains a non-NULL lwp_ap setup for get_syscall_args(). Note
161 * that lwp_ap MUST be set to a non-NULL value _BEFORE_ t_sysnum is
162 * set to non-zero; otherwise get_syscall_args(), seeing a non-zero
163 * t_sysnum for this thread, will charge ahead and dereference lwp_ap.
164 */
165 lwp->lwp_ap = argp; /* for get_syscall_args */
166
167 code = rp->r_r0;
168 t->t_sysnum = (short)code;
169 callp = code >= NSYSCALL ? &nosys_ent : se + code;
170
171 if ((t->t_pre_sys | syscalltrace) != 0) {
172 error = pre_syscall();
173
174 /*
175 * pre_syscall() has taken care so that lwp_ap is current;
176 * it either points to syscall-entry-saved amd64 regs,
177 * or it points to lwp_arg[], which has been re-copied from
178 * the ia32 ustack, but either way, it's a current copy after
179 * /proc has possibly mucked with the syscall args.
180 */
181
182 if (error)
183 return (&sysent_err); /* use dummy handler */
184 }
185
186 /*
187 * Fetch the system call arguments to the kernel stack copy used
188 * for syscall handling.
189 * Note: for loadable system calls the number of arguments required
190 * may not be known at this point, and will be zero if the system call
191 * was never loaded. Once the system call has been loaded, the number
192 * of args is not allowed to be changed.
193 */
194 if ((nargs = (uint_t)callp->sy_narg) != 0 &&
195 COPYIN_ARGS32(rp, argp, nargs)) {
196 (void) set_errno(EFAULT);
197 return (&sysent_err); /* use dummy handler */
198 }
199
200 return (callp); /* return sysent entry for caller */
201 }
202
203 void
syscall_exit(kthread_t * t,long rval1,long rval2)204 syscall_exit(kthread_t *t, long rval1, long rval2)
205 {
206 /*
207 * Handle signals and other post-call events if necessary.
208 */
209 if ((t->t_post_sys_ast | syscalltrace) == 0) {
210 klwp_t *lwp = ttolwp(t);
211 struct regs *rp = lwptoregs(lwp);
212
213 /*
214 * Normal return.
215 * Clear error indication and set return values.
216 */
217 rp->r_ps &= ~PS_C; /* reset carry bit */
218 rp->r_r0 = rval1;
219 rp->r_r1 = rval2;
220 lwp->lwp_state = LWP_USER;
221 } else {
222 post_syscall(rval1, rval2);
223 }
224 t->t_sysnum = 0; /* invalidate args */
225 }
226
227 /*
228 * Perform pre-system-call processing, including stopping for tracing,
229 * auditing, etc.
230 *
231 * This routine is called only if the t_pre_sys flag is set. Any condition
232 * requiring pre-syscall handling must set the t_pre_sys flag. If the
233 * condition is persistent, this routine will repost t_pre_sys.
234 */
235 int
pre_syscall()236 pre_syscall()
237 {
238 kthread_t *t = curthread;
239 unsigned code = t->t_sysnum;
240 klwp_t *lwp = ttolwp(t);
241 proc_t *p = ttoproc(t);
242 int repost;
243
244 t->t_pre_sys = repost = 0; /* clear pre-syscall processing flag */
245
246 ASSERT(t->t_schedflag & TS_DONT_SWAP);
247
248 #if defined(DEBUG)
249 /*
250 * On the i386 kernel, lwp_ap points at the piece of the thread
251 * stack that we copy the users arguments into.
252 *
253 * On the amd64 kernel, the syscall arguments in the rdi..r9
254 * registers should be pointed at by lwp_ap. If the args need to
255 * be copied so that those registers can be changed without losing
256 * the ability to get the args for /proc, they can be saved by
257 * save_syscall_args(), and lwp_ap will be restored by post_syscall().
258 */
259 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
260 #if defined(_LP64)
261 ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi);
262 } else {
263 #endif
264 ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase &&
265 (caddr_t)lwp->lwp_ap < t->t_stk);
266 }
267 #endif /* DEBUG */
268
269 /*
270 * Make sure the thread is holding the latest credentials for the
271 * process. The credentials in the process right now apply to this
272 * thread for the entire system call.
273 */
274 if (t->t_cred != p->p_cred) {
275 cred_t *oldcred = t->t_cred;
276 /*
277 * DTrace accesses t_cred in probe context. t_cred must
278 * always be either NULL, or point to a valid, allocated cred
279 * structure.
280 */
281 t->t_cred = crgetcred();
282 crfree(oldcred);
283 }
284
285 /*
286 * From the proc(5) manual page:
287 * When entry to a system call is being traced, the traced process
288 * stops after having begun the call to the system but before the
289 * system call arguments have been fetched from the process.
290 */
291 if (PTOU(p)->u_systrap) {
292 if (prismember(&PTOU(p)->u_entrymask, code)) {
293 mutex_enter(&p->p_lock);
294 /*
295 * Recheck stop condition, now that lock is held.
296 */
297 if (PTOU(p)->u_systrap &&
298 prismember(&PTOU(p)->u_entrymask, code)) {
299 stop(PR_SYSENTRY, code);
300
301 /*
302 * /proc may have modified syscall args,
303 * either in regs for amd64 or on ustack
304 * for ia32. Either way, arrange to
305 * copy them again, both for the syscall
306 * handler and for other consumers in
307 * post_syscall (like audit). Here, we
308 * only do amd64, and just set lwp_ap
309 * back to the kernel-entry stack copy;
310 * the syscall ml code redoes
311 * move-from-regs to set up for the
312 * syscall handler after we return. For
313 * ia32, save_syscall_args() below makes
314 * an lwp_ap-accessible copy.
315 */
316 #if defined(_LP64)
317 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
318 lwp->lwp_argsaved = 0;
319 lwp->lwp_ap =
320 (long *)&lwptoregs(lwp)->r_rdi;
321 }
322 #endif
323 }
324 mutex_exit(&p->p_lock);
325 }
326 repost = 1;
327 }
328
329 /*
330 * ia32 kernel, or ia32 proc on amd64 kernel: keep args in
331 * lwp_arg for post-syscall processing, regardless of whether
332 * they might have been changed in /proc above.
333 */
334 #if defined(_LP64)
335 if (lwp_getdatamodel(lwp) != DATAMODEL_NATIVE)
336 #endif
337 (void) save_syscall_args();
338
339 if (lwp->lwp_sysabort) {
340 /*
341 * lwp_sysabort may have been set via /proc while the process
342 * was stopped on PR_SYSENTRY. If so, abort the system call.
343 * Override any error from the copyin() of the arguments.
344 */
345 lwp->lwp_sysabort = 0;
346 (void) set_errno(EINTR); /* forces post_sys */
347 t->t_pre_sys = 1; /* repost anyway */
348 return (1); /* don't do system call, return EINTR */
349 }
350
351 /*
352 * begin auditing for this syscall if the c2audit module is loaded
353 * and auditing is enabled
354 */
355 if (audit_active == C2AUDIT_LOADED) {
356 uint32_t auditing = au_zone_getstate(NULL);
357
358 if (auditing & AU_AUDIT_MASK) {
359 int error;
360 if (error = audit_start(T_SYSCALL, code, auditing, \
361 0, lwp)) {
362 t->t_pre_sys = 1; /* repost anyway */
363 (void) set_errno(error);
364 return (1);
365 }
366 repost = 1;
367 }
368 }
369
370 #ifdef SYSCALLTRACE
371 if (syscalltrace) {
372 int i;
373 long *ap;
374 char *cp;
375 char *sysname;
376 struct sysent *callp;
377
378 if (code >= NSYSCALL)
379 callp = &nosys_ent; /* nosys has no args */
380 else
381 callp = LWP_GETSYSENT(lwp) + code;
382 (void) save_syscall_args();
383 mutex_enter(&systrace_lock);
384 printf("%d: ", p->p_pid);
385 if (code >= NSYSCALL) {
386 printf("0x%x", code);
387 } else {
388 sysname = mod_getsysname(code);
389 printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" :
390 sysname, code, callp->sy_callc);
391 }
392 cp = "(";
393 for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) {
394 printf("%s%lx", cp, *ap);
395 cp = ", ";
396 }
397 if (i)
398 printf(")");
399 printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread);
400 mutex_exit(&systrace_lock);
401 }
402 #endif /* SYSCALLTRACE */
403
404 /*
405 * If there was a continuing reason for pre-syscall processing,
406 * set the t_pre_sys flag for the next system call.
407 */
408 if (repost)
409 t->t_pre_sys = 1;
410 lwp->lwp_error = 0; /* for old drivers */
411 lwp->lwp_badpriv = PRIV_NONE;
412 return (0);
413 }
414
415
416 /*
417 * Post-syscall processing. Perform abnormal system call completion
418 * actions such as /proc tracing, profiling, signals, preemption, etc.
419 *
420 * This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
421 * Any condition requiring pre-syscall handling must set one of these.
422 * If the condition is persistent, this routine will repost t_post_sys.
423 */
424 void
post_syscall(long rval1,long rval2)425 post_syscall(long rval1, long rval2)
426 {
427 kthread_t *t = curthread;
428 klwp_t *lwp = ttolwp(t);
429 proc_t *p = ttoproc(t);
430 struct regs *rp = lwptoregs(lwp);
431 uint_t error;
432 uint_t code = t->t_sysnum;
433 int repost = 0;
434 int proc_stop = 0; /* non-zero if stopping */
435 int sigprof = 0; /* non-zero if sending SIGPROF */
436
437 t->t_post_sys = 0;
438
439 error = lwp->lwp_errno;
440
441 /*
442 * Code can be zero if this is a new LWP returning after a forkall(),
443 * other than the one which matches the one in the parent which called
444 * forkall(). In these LWPs, skip most of post-syscall activity.
445 */
446 if (code == 0)
447 goto sig_check;
448 /*
449 * If the trace flag is set, mark the lwp to take a single-step trap
450 * on return to user level (below). The x86 lcall interface and
451 * sysenter has already done this, and turned off the flag, but
452 * amd64 syscall interface has not.
453 */
454 if (rp->r_ps & PS_T) {
455 lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
456 rp->r_ps &= ~PS_T;
457 aston(curthread);
458 }
459
460 /* put out audit record for this syscall */
461 if (AU_AUDITING()) {
462 rval_t rval;
463
464 /* XX64 -- truncation of 64-bit return values? */
465 rval.r_val1 = (int)rval1;
466 rval.r_val2 = (int)rval2;
467 audit_finish(T_SYSCALL, code, error, &rval);
468 repost = 1;
469 }
470
471 if (curthread->t_pdmsg != NULL) {
472 char *m = curthread->t_pdmsg;
473
474 uprintf("%s", m);
475 kmem_free(m, strlen(m) + 1);
476 curthread->t_pdmsg = NULL;
477 }
478
479 /*
480 * If we're going to stop for /proc tracing, set the flag and
481 * save the arguments so that the return values don't smash them.
482 */
483 if (PTOU(p)->u_systrap) {
484 if (prismember(&PTOU(p)->u_exitmask, code)) {
485 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
486 (void) save_syscall_args();
487 proc_stop = 1;
488 }
489 repost = 1;
490 }
491
492 /*
493 * Similarly check to see if SIGPROF might be sent.
494 */
495 if (curthread->t_rprof != NULL &&
496 curthread->t_rprof->rp_anystate != 0) {
497 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
498 (void) save_syscall_args();
499 sigprof = 1;
500 }
501
502 if (lwp->lwp_eosys == NORMALRETURN) {
503 if (error == 0) {
504 #ifdef SYSCALLTRACE
505 if (syscalltrace) {
506 mutex_enter(&systrace_lock);
507 printf(
508 "%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
509 p->p_pid, rval1, rval2, curthread);
510 mutex_exit(&systrace_lock);
511 }
512 #endif /* SYSCALLTRACE */
513 rp->r_ps &= ~PS_C;
514 rp->r_r0 = rval1;
515 rp->r_r1 = rval2;
516 } else {
517 int sig;
518 #ifdef SYSCALLTRACE
519 if (syscalltrace) {
520 mutex_enter(&systrace_lock);
521 printf("%d: error=%d, id 0x%p\n",
522 p->p_pid, error, curthread);
523 mutex_exit(&systrace_lock);
524 }
525 #endif /* SYSCALLTRACE */
526 if (error == EINTR && t->t_activefd.a_stale)
527 error = EBADF;
528 if (error == EINTR &&
529 (sig = lwp->lwp_cursig) != 0 &&
530 sigismember(&PTOU(p)->u_sigrestart, sig) &&
531 PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
532 PTOU(p)->u_signal[sig - 1] != SIG_IGN)
533 error = ERESTART;
534 rp->r_r0 = error;
535 rp->r_ps |= PS_C;
536 }
537 }
538
539 /*
540 * From the proc(5) manual page:
541 * When exit from a system call is being traced, the traced process
542 * stops on completion of the system call just prior to checking for
543 * signals and returning to user level. At this point all return
544 * values have been stored into the traced process's saved registers.
545 */
546 if (proc_stop) {
547 mutex_enter(&p->p_lock);
548 if (PTOU(p)->u_systrap &&
549 prismember(&PTOU(p)->u_exitmask, code))
550 stop(PR_SYSEXIT, code);
551 mutex_exit(&p->p_lock);
552 }
553
554 /*
555 * If we are the parent returning from a successful
556 * vfork, wait for the child to exec or exit.
557 * This code must be here and not in the bowels of the system
558 * so that /proc can intercept exit from vfork in a timely way.
559 */
560 if (t->t_flag & T_VFPARENT) {
561 ASSERT(code == SYS_vfork || code == SYS_forksys);
562 ASSERT(rp->r_r1 == 0 && error == 0);
563 vfwait((pid_t)rval1);
564 t->t_flag &= ~T_VFPARENT;
565 }
566
567 /*
568 * If profiling is active, bill the current PC in user-land
569 * and keep reposting until profiling is disabled.
570 */
571 if (p->p_prof.pr_scale) {
572 if (lwp->lwp_oweupc)
573 profil_tick(rp->r_pc);
574 repost = 1;
575 }
576
577 sig_check:
578 /*
579 * Reset flag for next time.
580 * We must do this after stopping on PR_SYSEXIT
581 * because /proc uses the information in lwp_eosys.
582 */
583 lwp->lwp_eosys = NORMALRETURN;
584 clear_stale_fd();
585 t->t_flag &= ~T_FORKALL;
586
587 if (t->t_astflag | t->t_sig_check) {
588 /*
589 * Turn off the AST flag before checking all the conditions that
590 * may have caused an AST. This flag is on whenever a signal or
591 * unusual condition should be handled after the next trap or
592 * syscall.
593 */
594 astoff(t);
595 /*
596 * If a single-step trap occurred on a syscall (see trap())
597 * recognize it now. Do this before checking for signals
598 * because deferred_singlestep_trap() may generate a SIGTRAP to
599 * the LWP or may otherwise mark the LWP to call issig(FORREAL).
600 */
601 if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
602 deferred_singlestep_trap((caddr_t)rp->r_pc);
603
604 t->t_sig_check = 0;
605
606 /*
607 * The following check is legal for the following reasons:
608 * 1) The thread we are checking, is ourselves, so there is
609 * no way the proc can go away.
610 * 2) The only time we need to be protected by the
611 * lock is if the binding is changed.
612 *
613 * Note we will still take the lock and check the binding
614 * if the condition was true without the lock held. This
615 * prevents lock contention among threads owned by the
616 * same proc.
617 */
618
619 if (curthread->t_proc_flag & TP_CHANGEBIND) {
620 mutex_enter(&p->p_lock);
621 if (curthread->t_proc_flag & TP_CHANGEBIND) {
622 timer_lwpbind();
623 curthread->t_proc_flag &= ~TP_CHANGEBIND;
624 }
625 mutex_exit(&p->p_lock);
626 }
627
628 /*
629 * for kaio requests on the special kaio poll queue,
630 * copyout their results to user memory.
631 */
632 if (p->p_aio)
633 aio_cleanup(0);
634 /*
635 * If this LWP was asked to hold, call holdlwp(), which will
636 * stop. holdlwps() sets this up and calls pokelwps() which
637 * sets the AST flag.
638 *
639 * Also check TP_EXITLWP, since this is used by fresh new LWPs
640 * through lwp_rtt(). That flag is set if the lwp_create(2)
641 * syscall failed after creating the LWP.
642 */
643 if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
644 holdlwp();
645
646 /*
647 * All code that sets signals and makes ISSIG_PENDING
648 * evaluate true must set t_sig_check afterwards.
649 */
650 if (ISSIG_PENDING(t, lwp, p)) {
651 if (issig(FORREAL))
652 psig();
653 t->t_sig_check = 1; /* recheck next time */
654 }
655
656 if (sigprof) {
657 int nargs = (code > 0 && code < NSYSCALL)?
658 LWP_GETSYSENT(lwp)[code].sy_narg : 0;
659 realsigprof(code, nargs, error);
660 t->t_sig_check = 1; /* recheck next time */
661 }
662
663 /*
664 * If a performance counter overflow interrupt was
665 * delivered *during* the syscall, then re-enable the
666 * AST so that we take a trip through trap() to cause
667 * the SIGEMT to be delivered.
668 */
669 if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
670 aston(t);
671
672 /*
673 * /proc can't enable/disable the trace bit itself
674 * because that could race with the call gate used by
675 * system calls via "lcall". If that happened, an
676 * invalid EFLAGS would result. prstep()/prnostep()
677 * therefore schedule an AST for the purpose.
678 */
679 if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
680 lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
681 rp->r_ps |= PS_T;
682 }
683 if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
684 lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
685 rp->r_ps &= ~PS_T;
686 }
687 }
688
689 lwp->lwp_errno = 0; /* clear error for next time */
690
691 /*
692 * Set state to LWP_USER here so preempt won't give us a kernel
693 * priority if it occurs after this point. Call CL_TRAPRET() to
694 * restore the user-level priority.
695 *
696 * It is important that no locks (other than spinlocks) be entered
697 * after this point before returning to user mode (unless lwp_state
698 * is set back to LWP_SYS).
699 *
700 * XXX Sampled times past this point are charged to the user.
701 */
702 lwp->lwp_state = LWP_USER;
703
704 if (t->t_trapret) {
705 t->t_trapret = 0;
706 thread_lock(t);
707 CL_TRAPRET(t);
708 thread_unlock(t);
709 }
710 if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ)
711 preempt();
712 prunstop();
713
714 lwp->lwp_errno = 0; /* clear error for next time */
715
716 /*
717 * The thread lock must be held in order to clear sysnum and reset
718 * lwp_ap atomically with respect to other threads in the system that
719 * may be looking at the args via lwp_ap from get_syscall_args().
720 */
721
722 thread_lock(t);
723 t->t_sysnum = 0; /* no longer in a system call */
724
725 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
726 #if defined(_LP64)
727 /*
728 * In case the args were copied to the lwp, reset the
729 * pointer so the next syscall will have the right
730 * lwp_ap pointer.
731 */
732 lwp->lwp_ap = (long *)&rp->r_rdi;
733 } else {
734 #endif
735 lwp->lwp_ap = NULL; /* reset on every syscall entry */
736 }
737 thread_unlock(t);
738
739 lwp->lwp_argsaved = 0;
740
741 /*
742 * If there was a continuing reason for post-syscall processing,
743 * set the t_post_sys flag for the next system call.
744 */
745 if (repost)
746 t->t_post_sys = 1;
747
748 /*
749 * If there is a ustack registered for this lwp, and the stack rlimit
750 * has been altered, read in the ustack. If the saved stack rlimit
751 * matches the bounds of the ustack, update the ustack to reflect
752 * the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
753 * stack checking by setting the size to 0.
754 */
755 if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
756 rlim64_t new_size;
757 caddr_t top;
758 stack_t stk;
759 struct rlimit64 rl;
760
761 mutex_enter(&p->p_lock);
762 new_size = p->p_stk_ctl;
763 top = p->p_usrstack;
764 (void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
765 mutex_exit(&p->p_lock);
766
767 if (rl.rlim_cur == RLIM64_INFINITY)
768 new_size = 0;
769
770 if (copyin((stack_t *)lwp->lwp_ustack, &stk,
771 sizeof (stack_t)) == 0 &&
772 (stk.ss_size == lwp->lwp_old_stk_ctl ||
773 stk.ss_size == 0) &&
774 stk.ss_sp == top - stk.ss_size) {
775 stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
776 stk.ss_size - (uintptr_t)new_size);
777 stk.ss_size = new_size;
778
779 (void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
780 sizeof (stack_t));
781 }
782
783 lwp->lwp_old_stk_ctl = 0;
784 }
785 }
786
787 /*
788 * Called from post_syscall() when a deferred singlestep is to be taken.
789 */
790 void
deferred_singlestep_trap(caddr_t pc)791 deferred_singlestep_trap(caddr_t pc)
792 {
793 proc_t *p = ttoproc(curthread);
794 klwp_t *lwp = ttolwp(curthread);
795 pcb_t *pcb = &lwp->lwp_pcb;
796 uint_t fault = 0;
797 k_siginfo_t siginfo;
798
799 bzero(&siginfo, sizeof (siginfo));
800
801 /*
802 * If both NORMAL_STEP and WATCH_STEP are in
803 * effect, give precedence to WATCH_STEP.
804 * If neither is set, user must have set the
805 * PS_T bit in %efl; treat this as NORMAL_STEP.
806 */
807 if ((fault = undo_watch_step(&siginfo)) == 0 &&
808 ((pcb->pcb_flags & NORMAL_STEP) ||
809 !(pcb->pcb_flags & WATCH_STEP))) {
810 siginfo.si_signo = SIGTRAP;
811 siginfo.si_code = TRAP_TRACE;
812 siginfo.si_addr = pc;
813 fault = FLTTRACE;
814 }
815 pcb->pcb_flags &= ~(DEBUG_PENDING|NORMAL_STEP|WATCH_STEP);
816
817 if (fault) {
818 /*
819 * Remember the fault and fault adddress
820 * for real-time (SIGPROF) profiling.
821 */
822 lwp->lwp_lastfault = fault;
823 lwp->lwp_lastfaddr = siginfo.si_addr;
824 /*
825 * If a debugger has declared this fault to be an
826 * event of interest, stop the lwp. Otherwise just
827 * deliver the associated signal.
828 */
829 if (prismember(&p->p_fltmask, fault) &&
830 stop_on_fault(fault, &siginfo) == 0)
831 siginfo.si_signo = 0;
832 }
833
834 if (siginfo.si_signo)
835 trapsig(&siginfo, 1);
836 }
837
838 /*
839 * nonexistent system call-- signal lwp (may want to handle it)
840 * flag error if lwp won't see signal immediately
841 */
842 int64_t
nosys(void)843 nosys(void)
844 {
845 tsignal(curthread, SIGSYS);
846 return (set_errno(ENOSYS));
847 }
848
849 int
nosys32(void)850 nosys32(void)
851 {
852 return (nosys());
853 }
854
855 /*
856 * Execute a 32-bit system call on behalf of the current thread.
857 */
858 void
dosyscall(void)859 dosyscall(void)
860 {
861 /*
862 * Need space on the stack to store syscall arguments.
863 */
864 long syscall_args[MAXSYSARGS];
865 struct sysent *se;
866 int64_t ret;
867
868 syscall_mstate(LMS_TRAP, LMS_SYSTEM);
869
870 ASSERT(curproc->p_model == DATAMODEL_ILP32);
871
872 CPU_STATS_ENTER_K();
873 CPU_STATS_ADDQ(CPU, sys, syscall, 1);
874 CPU_STATS_EXIT_K();
875
876 se = syscall_entry(curthread, syscall_args);
877
878 /*
879 * syscall_entry() copied all 8 arguments into syscall_args.
880 */
881 ret = se->sy_callc(syscall_args[0], syscall_args[1], syscall_args[2],
882 syscall_args[3], syscall_args[4], syscall_args[5], syscall_args[6],
883 syscall_args[7]);
884
885 syscall_exit(curthread, (int)ret & 0xffffffffu, (int)(ret >> 32));
886 syscall_mstate(LMS_SYSTEM, LMS_TRAP);
887 }
888
889 /*
890 * Get the arguments to the current system call. See comment atop
891 * save_syscall_args() regarding lwp_ap usage.
892 */
893
894 uint_t
get_syscall_args(klwp_t * lwp,long * argp,int * nargsp)895 get_syscall_args(klwp_t *lwp, long *argp, int *nargsp)
896 {
897 kthread_t *t = lwptot(lwp);
898 ulong_t mask = 0xfffffffful;
899 uint_t code;
900 long *ap;
901 int nargs;
902
903 #if defined(_LP64)
904 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
905 mask = 0xfffffffffffffffful;
906 #endif
907
908 /*
909 * The thread lock must be held while looking at the arguments to ensure
910 * they don't go away via post_syscall().
911 * get_syscall_args() is the only routine to read them which is callable
912 * outside the LWP in question and hence the only one that must be
913 * synchronized in this manner.
914 */
915 thread_lock(t);
916
917 code = t->t_sysnum;
918 ap = lwp->lwp_ap;
919
920 thread_unlock(t);
921
922 if (code != 0 && code < NSYSCALL) {
923 nargs = LWP_GETSYSENT(lwp)[code].sy_narg;
924
925 ASSERT(nargs <= MAXSYSARGS);
926
927 *nargsp = nargs;
928 while (nargs-- > 0)
929 *argp++ = *ap++ & mask;
930 } else {
931 *nargsp = 0;
932 }
933
934 return (code);
935 }
936
937 #ifdef _SYSCALL32_IMPL
938 /*
939 * Get the arguments to the current 32-bit system call.
940 */
941 uint_t
get_syscall32_args(klwp_t * lwp,int * argp,int * nargsp)942 get_syscall32_args(klwp_t *lwp, int *argp, int *nargsp)
943 {
944 long args[MAXSYSARGS];
945 uint_t i, code;
946
947 code = get_syscall_args(lwp, args, nargsp);
948
949 for (i = 0; i != *nargsp; i++)
950 *argp++ = (int)args[i];
951 return (code);
952 }
953 #endif
954
955 /*
956 * Save the system call arguments in a safe place.
957 *
958 * On the i386 kernel:
959 *
960 * Copy the users args prior to changing the stack or stack pointer.
961 * This is so /proc will be able to get a valid copy of the
962 * args from the user stack even after the user stack has been changed.
963 * Note that the kernel stack copy of the args may also have been
964 * changed by a system call handler which takes C-style arguments.
965 *
966 * Note that this may be called by stop() from trap(). In that case
967 * t_sysnum will be zero (syscall_exit clears it), so no args will be
968 * copied.
969 *
970 * On the amd64 kernel:
971 *
972 * For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9
973 * in the reg structure. If the user is going to change the argument
974 * registers, rax, or the stack and might want to get the args (for
975 * /proc tracing), it must copy the args elsewhere via save_syscall_args().
976 *
977 * For 32-bit applications, lwp->lwp_ap normally points to a copy of
978 * the system call arguments on the kernel stack made from the user
979 * stack. Copy the args prior to change the stack or stack pointer.
980 * This is so /proc will be able to get a valid copy of the args
981 * from the user stack even after that stack has been changed.
982 *
983 * This may be called from stop() even when we're not in a system call.
984 * Since there's no easy way to tell, this must be safe (not panic).
985 * If the copyins get data faults, return non-zero.
986 */
987 int
save_syscall_args()988 save_syscall_args()
989 {
990 kthread_t *t = curthread;
991 klwp_t *lwp = ttolwp(t);
992 uint_t code = t->t_sysnum;
993 uint_t nargs;
994
995 if (lwp->lwp_argsaved || code == 0)
996 return (0); /* args already saved or not needed */
997
998 if (code >= NSYSCALL) {
999 nargs = 0; /* illegal syscall */
1000 } else {
1001 struct sysent *se = LWP_GETSYSENT(lwp);
1002 struct sysent *callp = se + code;
1003
1004 nargs = callp->sy_narg;
1005 if (LOADABLE_SYSCALL(callp) && nargs == 0) {
1006 krwlock_t *module_lock;
1007
1008 /*
1009 * Find out how many arguments the system
1010 * call uses.
1011 *
1012 * We have the property that loaded syscalls
1013 * never change the number of arguments they
1014 * use after they've been loaded once. This
1015 * allows us to stop for /proc tracing without
1016 * holding the module lock.
1017 * /proc is assured that sy_narg is valid.
1018 */
1019 module_lock = lock_syscall(se, code);
1020 nargs = callp->sy_narg;
1021 rw_exit(module_lock);
1022 }
1023 }
1024
1025 /*
1026 * Fetch the system call arguments.
1027 */
1028 if (nargs == 0)
1029 goto out;
1030
1031 ASSERT(nargs <= MAXSYSARGS);
1032
1033 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1034 #if defined(_LP64)
1035 struct regs *rp = lwptoregs(lwp);
1036
1037 lwp->lwp_arg[0] = rp->r_rdi;
1038 lwp->lwp_arg[1] = rp->r_rsi;
1039 lwp->lwp_arg[2] = rp->r_rdx;
1040 lwp->lwp_arg[3] = rp->r_rcx;
1041 lwp->lwp_arg[4] = rp->r_r8;
1042 lwp->lwp_arg[5] = rp->r_r9;
1043 if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6))
1044 return (-1);
1045 } else {
1046 #endif
1047 if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs))
1048 return (-1);
1049 }
1050 out:
1051 lwp->lwp_ap = lwp->lwp_arg;
1052 lwp->lwp_argsaved = 1;
1053 t->t_post_sys = 1; /* so lwp_ap will be reset */
1054 return (0);
1055 }
1056
1057 void
reset_syscall_args(void)1058 reset_syscall_args(void)
1059 {
1060 ttolwp(curthread)->lwp_argsaved = 0;
1061 }
1062
1063 /*
1064 * Call a system call which takes a pointer to the user args struct and
1065 * a pointer to the return values. This is a bit slower than the standard
1066 * C arg-passing method in some cases.
1067 */
1068 int64_t
syscall_ap(void)1069 syscall_ap(void)
1070 {
1071 uint_t error;
1072 struct sysent *callp;
1073 rval_t rval;
1074 kthread_t *t = curthread;
1075 klwp_t *lwp = ttolwp(t);
1076 struct regs *rp = lwptoregs(lwp);
1077
1078 callp = LWP_GETSYSENT(lwp) + t->t_sysnum;
1079
1080 /*
1081 * If the arguments don't fit in registers %rdi-%r9, make sure they
1082 * have been copied to the lwp_arg array.
1083 */
1084 if (callp->sy_narg > 6 && save_syscall_args())
1085 return ((int64_t)set_errno(EFAULT));
1086
1087 rval.r_val1 = 0;
1088 rval.r_val2 = rp->r_r1;
1089 lwp->lwp_error = 0; /* for old drivers */
1090 error = (*(callp->sy_call))(lwp->lwp_ap, &rval);
1091 if (error)
1092 return ((longlong_t)set_errno(error));
1093 return (rval.r_vals);
1094 }
1095
1096 /*
1097 * Load system call module.
1098 * Returns with pointer to held read lock for module.
1099 */
1100 static krwlock_t *
lock_syscall(struct sysent * table,uint_t code)1101 lock_syscall(struct sysent *table, uint_t code)
1102 {
1103 krwlock_t *module_lock;
1104 struct modctl *modp;
1105 int id;
1106 struct sysent *callp;
1107
1108 callp = table + code;
1109 module_lock = callp->sy_lock;
1110
1111 /*
1112 * Optimization to only call modload if we don't have a loaded
1113 * syscall.
1114 */
1115 rw_enter(module_lock, RW_READER);
1116 if (LOADED_SYSCALL(callp))
1117 return (module_lock);
1118 rw_exit(module_lock);
1119
1120 for (;;) {
1121 if ((id = modload("sys", syscallnames[code])) == -1)
1122 break;
1123
1124 /*
1125 * If we loaded successfully at least once, the modctl
1126 * will still be valid, so we try to grab it by filename.
1127 * If this call fails, it's because the mod_filename
1128 * was changed after the call to modload() (mod_hold_by_name()
1129 * is the likely culprit). We can safely just take
1130 * another lap if this is the case; the modload() will
1131 * change the mod_filename back to one by which we can
1132 * find the modctl.
1133 */
1134 modp = mod_find_by_filename("sys", syscallnames[code]);
1135
1136 if (modp == NULL)
1137 continue;
1138
1139 mutex_enter(&mod_lock);
1140
1141 if (!modp->mod_installed) {
1142 mutex_exit(&mod_lock);
1143 continue;
1144 }
1145 break;
1146 }
1147 rw_enter(module_lock, RW_READER);
1148
1149 if (id != -1)
1150 mutex_exit(&mod_lock);
1151
1152 return (module_lock);
1153 }
1154
1155 /*
1156 * Loadable syscall support.
1157 * If needed, load the module, then reserve it by holding a read
1158 * lock for the duration of the call.
1159 * Later, if the syscall is not unloadable, it could patch the vector.
1160 */
1161 /*ARGSUSED*/
1162 int64_t
loadable_syscall(long a0,long a1,long a2,long a3,long a4,long a5,long a6,long a7)1163 loadable_syscall(
1164 long a0, long a1, long a2, long a3,
1165 long a4, long a5, long a6, long a7)
1166 {
1167 klwp_t *lwp = ttolwp(curthread);
1168 int64_t rval;
1169 struct sysent *callp;
1170 struct sysent *se = LWP_GETSYSENT(lwp);
1171 krwlock_t *module_lock;
1172 int code, error = 0;
1173
1174 code = curthread->t_sysnum;
1175 callp = se + code;
1176
1177 /*
1178 * Try to autoload the system call if necessary
1179 */
1180 module_lock = lock_syscall(se, code);
1181
1182 /*
1183 * we've locked either the loaded syscall or nosys
1184 */
1185
1186 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1187 #if defined(_LP64)
1188 if (callp->sy_flags & SE_ARGC) {
1189 rval = (int64_t)(*callp->sy_call)(a0, a1, a2, a3,
1190 a4, a5);
1191 } else {
1192 rval = syscall_ap();
1193 }
1194 } else {
1195 #endif
1196 /*
1197 * Now that it's loaded, make sure enough args were copied.
1198 */
1199 if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg))
1200 error = EFAULT;
1201 if (error) {
1202 rval = set_errno(error);
1203 } else if (callp->sy_flags & SE_ARGC) {
1204 rval = (int64_t)(*callp->sy_call)(lwp->lwp_ap[0],
1205 lwp->lwp_ap[1], lwp->lwp_ap[2], lwp->lwp_ap[3],
1206 lwp->lwp_ap[4], lwp->lwp_ap[5]);
1207 } else {
1208 rval = syscall_ap();
1209 }
1210 }
1211
1212 rw_exit(module_lock);
1213 return (rval);
1214 }
1215
1216 /*
1217 * Indirect syscall handled in libc on x86 architectures
1218 */
1219 int64_t
indir()1220 indir()
1221 {
1222 return (nosys());
1223 }
1224
1225 /*
1226 * set_errno - set an error return from the current system call.
1227 * This could be a macro.
1228 * This returns the value it is passed, so that the caller can
1229 * use tail-recursion-elimination and do return (set_errno(ERRNO));
1230 */
1231 uint_t
set_errno(uint_t error)1232 set_errno(uint_t error)
1233 {
1234 ASSERT(error != 0); /* must not be used to clear errno */
1235
1236 curthread->t_post_sys = 1; /* have post_syscall do error return */
1237 return (ttolwp(curthread)->lwp_errno = error);
1238 }
1239
1240 /*
1241 * set_proc_pre_sys - Set pre-syscall processing for entire process.
1242 */
1243 void
set_proc_pre_sys(proc_t * p)1244 set_proc_pre_sys(proc_t *p)
1245 {
1246 kthread_t *t;
1247 kthread_t *first;
1248
1249 ASSERT(MUTEX_HELD(&p->p_lock));
1250
1251 t = first = p->p_tlist;
1252 do {
1253 t->t_pre_sys = 1;
1254 } while ((t = t->t_forw) != first);
1255 }
1256
1257 /*
1258 * set_proc_post_sys - Set post-syscall processing for entire process.
1259 */
1260 void
set_proc_post_sys(proc_t * p)1261 set_proc_post_sys(proc_t *p)
1262 {
1263 kthread_t *t;
1264 kthread_t *first;
1265
1266 ASSERT(MUTEX_HELD(&p->p_lock));
1267
1268 t = first = p->p_tlist;
1269 do {
1270 t->t_post_sys = 1;
1271 } while ((t = t->t_forw) != first);
1272 }
1273
1274 /*
1275 * set_proc_sys - Set pre- and post-syscall processing for entire process.
1276 */
1277 void
set_proc_sys(proc_t * p)1278 set_proc_sys(proc_t *p)
1279 {
1280 kthread_t *t;
1281 kthread_t *first;
1282
1283 ASSERT(MUTEX_HELD(&p->p_lock));
1284
1285 t = first = p->p_tlist;
1286 do {
1287 t->t_pre_sys = 1;
1288 t->t_post_sys = 1;
1289 } while ((t = t->t_forw) != first);
1290 }
1291
1292 /*
1293 * set_all_proc_sys - set pre- and post-syscall processing flags for all
1294 * user processes.
1295 *
1296 * This is needed when auditing, tracing, or other facilities which affect
1297 * all processes are turned on.
1298 */
1299 void
set_all_proc_sys()1300 set_all_proc_sys()
1301 {
1302 kthread_t *t;
1303 kthread_t *first;
1304
1305 mutex_enter(&pidlock);
1306 t = first = curthread;
1307 do {
1308 t->t_pre_sys = 1;
1309 t->t_post_sys = 1;
1310 } while ((t = t->t_next) != first);
1311 mutex_exit(&pidlock);
1312 }
1313
1314 /*
1315 * set_all_zone_usr_proc_sys - set pre- and post-syscall processing flags for
1316 * all user processes running in the zone of the current process
1317 *
1318 * This is needed when auditing, tracing, or other facilities which affect
1319 * all processes are turned on.
1320 */
1321 void
set_all_zone_usr_proc_sys(zoneid_t zoneid)1322 set_all_zone_usr_proc_sys(zoneid_t zoneid)
1323 {
1324 proc_t *p;
1325 kthread_t *t;
1326
1327 mutex_enter(&pidlock);
1328 for (p = practive; p != NULL; p = p->p_next) {
1329 /* skip kernel and incomplete processes */
1330 if (p->p_exec == NULLVP || p->p_as == &kas ||
1331 p->p_stat == SIDL || p->p_stat == SZOMB ||
1332 (p->p_flag & (SSYS | SEXITING | SEXITLWPS)))
1333 continue;
1334 /*
1335 * Only processes in the given zone (eventually in
1336 * all zones) are taken into account
1337 */
1338 if (zoneid == ALL_ZONES || p->p_zone->zone_id == zoneid) {
1339 mutex_enter(&p->p_lock);
1340 if ((t = p->p_tlist) == NULL) {
1341 mutex_exit(&p->p_lock);
1342 continue;
1343 }
1344 /*
1345 * Set pre- and post-syscall processing flags
1346 * for all threads of the process
1347 */
1348 do {
1349 t->t_pre_sys = 1;
1350 t->t_post_sys = 1;
1351 } while (p->p_tlist != (t = t->t_forw));
1352 mutex_exit(&p->p_lock);
1353 }
1354 }
1355 mutex_exit(&pidlock);
1356 }
1357
1358 /*
1359 * set_proc_ast - Set asynchronous service trap (AST) flag for all
1360 * threads in process.
1361 */
1362 void
set_proc_ast(proc_t * p)1363 set_proc_ast(proc_t *p)
1364 {
1365 kthread_t *t;
1366 kthread_t *first;
1367
1368 ASSERT(MUTEX_HELD(&p->p_lock));
1369
1370 t = first = p->p_tlist;
1371 do {
1372 aston(t);
1373 } while ((t = t->t_forw) != first);
1374 }
1375