1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27 #include <sys/param.h>
28 #include <sys/vmparam.h>
29 #include <sys/types.h>
30 #include <sys/sysmacros.h>
31 #include <sys/systm.h>
32 #include <sys/signal.h>
33 #include <sys/stack.h>
34 #include <sys/cred.h>
35 #include <sys/cmn_err.h>
36 #include <sys/user.h>
37 #include <sys/privregs.h>
38 #include <sys/psw.h>
39 #include <sys/debug.h>
40 #include <sys/errno.h>
41 #include <sys/proc.h>
42 #include <sys/modctl.h>
43 #include <sys/var.h>
44 #include <sys/inline.h>
45 #include <sys/syscall.h>
46 #include <sys/ucontext.h>
47 #include <sys/cpuvar.h>
48 #include <sys/siginfo.h>
49 #include <sys/trap.h>
50 #include <sys/vtrace.h>
51 #include <sys/sysinfo.h>
52 #include <sys/procfs.h>
53 #include <sys/prsystm.h>
54 #include <c2/audit.h>
55 #include <sys/modctl.h>
56 #include <sys/aio_impl.h>
57 #include <sys/tnf.h>
58 #include <sys/tnf_probe.h>
59 #include <sys/copyops.h>
60 #include <sys/priv.h>
61 #include <sys/msacct.h>
62
63 int syscalltrace = 0;
64 #ifdef SYSCALLTRACE
65 static kmutex_t systrace_lock; /* syscall tracing lock */
66 #else
67 #define syscalltrace 0
68 #endif /* SYSCALLTRACE */
69
70 typedef int64_t (*llfcn_t)(); /* function returning long long */
71
72 int pre_syscall(void);
73 void post_syscall(long rval1, long rval2);
74 static krwlock_t *lock_syscall(struct sysent *, uint_t);
75 void deferred_singlestep_trap(caddr_t);
76
77 #ifdef _SYSCALL32_IMPL
78 #define LWP_GETSYSENT(lwp) \
79 (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? sysent : sysent32)
80 #else
81 #define LWP_GETSYSENT(lwp) (sysent)
82 #endif
83
84 /*
85 * If watchpoints are active, don't make copying in of
86 * system call arguments take a read watchpoint trap.
87 */
88 static int
copyin_args(struct regs * rp,long * ap,uint_t nargs)89 copyin_args(struct regs *rp, long *ap, uint_t nargs)
90 {
91 greg_t *sp = 1 + (greg_t *)rp->r_sp; /* skip ret addr */
92
93 ASSERT(nargs <= MAXSYSARGS);
94
95 return (copyin_nowatch(sp, ap, nargs * sizeof (*sp)));
96 }
97
98 #if defined(_SYSCALL32_IMPL)
99 static int
copyin_args32(struct regs * rp,long * ap,uint_t nargs)100 copyin_args32(struct regs *rp, long *ap, uint_t nargs)
101 {
102 greg32_t *sp = 1 + (greg32_t *)rp->r_sp; /* skip ret addr */
103 uint32_t a32[MAXSYSARGS];
104 int rc;
105
106 ASSERT(nargs <= MAXSYSARGS);
107
108 if ((rc = copyin_nowatch(sp, a32, nargs * sizeof (*sp))) == 0) {
109 uint32_t *a32p = &a32[0];
110
111 while (nargs--)
112 *ap++ = (ulong_t)*a32p++;
113 }
114 return (rc);
115 }
116 #define COPYIN_ARGS32 copyin_args32
117 #else
118 #define COPYIN_ARGS32 copyin_args
119 #endif
120
121 /*
122 * Error handler for system calls where arg copy gets fault.
123 */
124 static longlong_t
syscall_err()125 syscall_err()
126 {
127 return (0);
128 }
129
130 /*
131 * Corresponding sysent entry to allow syscall_entry caller
132 * to invoke syscall_err.
133 */
134 static struct sysent sysent_err = {
135 0, SE_32RVAL1, NULL, NULL, (llfcn_t)syscall_err
136 };
137
138 /*
139 * Called from syscall() when a non-trivial 32-bit system call occurs.
140 * Sets up the args and returns a pointer to the handler.
141 */
142 struct sysent *
syscall_entry(kthread_t * t,long * argp)143 syscall_entry(kthread_t *t, long *argp)
144 {
145 klwp_t *lwp = ttolwp(t);
146 struct regs *rp = lwptoregs(lwp);
147 unsigned int code;
148 struct sysent *callp;
149 struct sysent *se = LWP_GETSYSENT(lwp);
150 int error = 0;
151 uint_t nargs;
152
153 ASSERT(t == curthread && curthread->t_schedflag & TS_DONT_SWAP);
154
155 lwp->lwp_ru.sysc++;
156 lwp->lwp_eosys = NORMALRETURN; /* assume this will be normal */
157
158 /*
159 * Set lwp_ap to point to the args, even if none are needed for this
160 * system call. This is for the loadable-syscall case where the
161 * number of args won't be known until the system call is loaded, and
162 * also maintains a non-NULL lwp_ap setup for get_syscall_args(). Note
163 * that lwp_ap MUST be set to a non-NULL value _BEFORE_ t_sysnum is
164 * set to non-zero; otherwise get_syscall_args(), seeing a non-zero
165 * t_sysnum for this thread, will charge ahead and dereference lwp_ap.
166 */
167 lwp->lwp_ap = argp; /* for get_syscall_args */
168
169 code = rp->r_r0;
170 t->t_sysnum = (short)code;
171 callp = code >= NSYSCALL ? &nosys_ent : se + code;
172
173 if ((t->t_pre_sys | syscalltrace) != 0) {
174 error = pre_syscall();
175
176 /*
177 * pre_syscall() has taken care so that lwp_ap is current;
178 * it either points to syscall-entry-saved amd64 regs,
179 * or it points to lwp_arg[], which has been re-copied from
180 * the ia32 ustack, but either way, it's a current copy after
181 * /proc has possibly mucked with the syscall args.
182 */
183
184 if (error)
185 return (&sysent_err); /* use dummy handler */
186 }
187
188 /*
189 * Fetch the system call arguments to the kernel stack copy used
190 * for syscall handling.
191 * Note: for loadable system calls the number of arguments required
192 * may not be known at this point, and will be zero if the system call
193 * was never loaded. Once the system call has been loaded, the number
194 * of args is not allowed to be changed.
195 */
196 if ((nargs = (uint_t)callp->sy_narg) != 0 &&
197 COPYIN_ARGS32(rp, argp, nargs)) {
198 (void) set_errno(EFAULT);
199 return (&sysent_err); /* use dummy handler */
200 }
201
202 return (callp); /* return sysent entry for caller */
203 }
204
205 void
syscall_exit(kthread_t * t,long rval1,long rval2)206 syscall_exit(kthread_t *t, long rval1, long rval2)
207 {
208 /*
209 * Handle signals and other post-call events if necessary.
210 */
211 if ((t->t_post_sys_ast | syscalltrace) == 0) {
212 klwp_t *lwp = ttolwp(t);
213 struct regs *rp = lwptoregs(lwp);
214
215 /*
216 * Normal return.
217 * Clear error indication and set return values.
218 */
219 rp->r_ps &= ~PS_C; /* reset carry bit */
220 rp->r_r0 = rval1;
221 rp->r_r1 = rval2;
222 lwp->lwp_state = LWP_USER;
223 } else
224 post_syscall(rval1, rval2);
225 t->t_sysnum = 0; /* invalidate args */
226 }
227
228 /*
229 * Perform pre-system-call processing, including stopping for tracing,
230 * auditing, etc.
231 *
232 * This routine is called only if the t_pre_sys flag is set. Any condition
233 * requiring pre-syscall handling must set the t_pre_sys flag. If the
234 * condition is persistent, this routine will repost t_pre_sys.
235 */
236 int
pre_syscall()237 pre_syscall()
238 {
239 kthread_t *t = curthread;
240 unsigned code = t->t_sysnum;
241 klwp_t *lwp = ttolwp(t);
242 proc_t *p = ttoproc(t);
243 int repost;
244
245 t->t_pre_sys = repost = 0; /* clear pre-syscall processing flag */
246
247 ASSERT(t->t_schedflag & TS_DONT_SWAP);
248
249 #if defined(DEBUG)
250 /*
251 * On the i386 kernel, lwp_ap points at the piece of the thread
252 * stack that we copy the users arguments into.
253 *
254 * On the amd64 kernel, the syscall arguments in the rdi..r9
255 * registers should be pointed at by lwp_ap. If the args need to
256 * be copied so that those registers can be changed without losing
257 * the ability to get the args for /proc, they can be saved by
258 * save_syscall_args(), and lwp_ap will be restored by post_syscall().
259 */
260 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
261 #if defined(_LP64)
262 ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi);
263 } else {
264 #endif
265 ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase &&
266 (caddr_t)lwp->lwp_ap < t->t_stk);
267 }
268 #endif /* DEBUG */
269
270 /*
271 * Make sure the thread is holding the latest credentials for the
272 * process. The credentials in the process right now apply to this
273 * thread for the entire system call.
274 */
275 if (t->t_cred != p->p_cred) {
276 cred_t *oldcred = t->t_cred;
277 /*
278 * DTrace accesses t_cred in probe context. t_cred must
279 * always be either NULL, or point to a valid, allocated cred
280 * structure.
281 */
282 t->t_cred = crgetcred();
283 crfree(oldcred);
284 }
285
286 /*
287 * From the proc(4) manual page:
288 * When entry to a system call is being traced, the traced process
289 * stops after having begun the call to the system but before the
290 * system call arguments have been fetched from the process.
291 */
292 if (PTOU(p)->u_systrap) {
293 if (prismember(&PTOU(p)->u_entrymask, code)) {
294 mutex_enter(&p->p_lock);
295 /*
296 * Recheck stop condition, now that lock is held.
297 */
298 if (PTOU(p)->u_systrap &&
299 prismember(&PTOU(p)->u_entrymask, code)) {
300 stop(PR_SYSENTRY, code);
301
302 /*
303 * /proc may have modified syscall args,
304 * either in regs for amd64 or on ustack
305 * for ia32. Either way, arrange to
306 * copy them again, both for the syscall
307 * handler and for other consumers in
308 * post_syscall (like audit). Here, we
309 * only do amd64, and just set lwp_ap
310 * back to the kernel-entry stack copy;
311 * the syscall ml code redoes
312 * move-from-regs to set up for the
313 * syscall handler after we return. For
314 * ia32, save_syscall_args() below makes
315 * an lwp_ap-accessible copy.
316 */
317 #if defined(_LP64)
318 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
319 lwp->lwp_argsaved = 0;
320 lwp->lwp_ap =
321 (long *)&lwptoregs(lwp)->r_rdi;
322 }
323 #endif
324 }
325 mutex_exit(&p->p_lock);
326 }
327 repost = 1;
328 }
329
330 /*
331 * ia32 kernel, or ia32 proc on amd64 kernel: keep args in
332 * lwp_arg for post-syscall processing, regardless of whether
333 * they might have been changed in /proc above.
334 */
335 #if defined(_LP64)
336 if (lwp_getdatamodel(lwp) != DATAMODEL_NATIVE)
337 #endif
338 (void) save_syscall_args();
339
340 if (lwp->lwp_sysabort) {
341 /*
342 * lwp_sysabort may have been set via /proc while the process
343 * was stopped on PR_SYSENTRY. If so, abort the system call.
344 * Override any error from the copyin() of the arguments.
345 */
346 lwp->lwp_sysabort = 0;
347 (void) set_errno(EINTR); /* forces post_sys */
348 t->t_pre_sys = 1; /* repost anyway */
349 return (1); /* don't do system call, return EINTR */
350 }
351
352 /*
353 * begin auditing for this syscall if the c2audit module is loaded
354 * and auditing is enabled
355 */
356 if (audit_active == C2AUDIT_LOADED) {
357 uint32_t auditing = au_zone_getstate(NULL);
358
359 if (auditing & AU_AUDIT_MASK) {
360 int error;
361 if (error = audit_start(T_SYSCALL, code, auditing, \
362 0, lwp)) {
363 t->t_pre_sys = 1; /* repost anyway */
364 (void) set_errno(error);
365 return (1);
366 }
367 repost = 1;
368 }
369 }
370
371 #ifndef NPROBE
372 /* Kernel probe */
373 if (tnf_tracing_active) {
374 TNF_PROBE_1(syscall_start, "syscall thread", /* CSTYLED */,
375 tnf_sysnum, sysnum, t->t_sysnum);
376 t->t_post_sys = 1; /* make sure post_syscall runs */
377 repost = 1;
378 }
379 #endif /* NPROBE */
380
381 #ifdef SYSCALLTRACE
382 if (syscalltrace) {
383 int i;
384 long *ap;
385 char *cp;
386 char *sysname;
387 struct sysent *callp;
388
389 if (code >= NSYSCALL)
390 callp = &nosys_ent; /* nosys has no args */
391 else
392 callp = LWP_GETSYSENT(lwp) + code;
393 (void) save_syscall_args();
394 mutex_enter(&systrace_lock);
395 printf("%d: ", p->p_pid);
396 if (code >= NSYSCALL)
397 printf("0x%x", code);
398 else {
399 sysname = mod_getsysname(code);
400 printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" :
401 sysname, code, callp->sy_callc);
402 }
403 cp = "(";
404 for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) {
405 printf("%s%lx", cp, *ap);
406 cp = ", ";
407 }
408 if (i)
409 printf(")");
410 printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread);
411 mutex_exit(&systrace_lock);
412 }
413 #endif /* SYSCALLTRACE */
414
415 /*
416 * If there was a continuing reason for pre-syscall processing,
417 * set the t_pre_sys flag for the next system call.
418 */
419 if (repost)
420 t->t_pre_sys = 1;
421 lwp->lwp_error = 0; /* for old drivers */
422 lwp->lwp_badpriv = PRIV_NONE;
423 return (0);
424 }
425
426
427 /*
428 * Post-syscall processing. Perform abnormal system call completion
429 * actions such as /proc tracing, profiling, signals, preemption, etc.
430 *
431 * This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
432 * Any condition requiring pre-syscall handling must set one of these.
433 * If the condition is persistent, this routine will repost t_post_sys.
434 */
435 void
post_syscall(long rval1,long rval2)436 post_syscall(long rval1, long rval2)
437 {
438 kthread_t *t = curthread;
439 klwp_t *lwp = ttolwp(t);
440 proc_t *p = ttoproc(t);
441 struct regs *rp = lwptoregs(lwp);
442 uint_t error;
443 uint_t code = t->t_sysnum;
444 int repost = 0;
445 int proc_stop = 0; /* non-zero if stopping */
446 int sigprof = 0; /* non-zero if sending SIGPROF */
447
448 t->t_post_sys = 0;
449
450 error = lwp->lwp_errno;
451
452 /*
453 * Code can be zero if this is a new LWP returning after a forkall(),
454 * other than the one which matches the one in the parent which called
455 * forkall(). In these LWPs, skip most of post-syscall activity.
456 */
457 if (code == 0)
458 goto sig_check;
459 /*
460 * If the trace flag is set, mark the lwp to take a single-step trap
461 * on return to user level (below). The x86 lcall interface and
462 * sysenter has already done this, and turned off the flag, but
463 * amd64 syscall interface has not.
464 */
465 if (rp->r_ps & PS_T) {
466 lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
467 rp->r_ps &= ~PS_T;
468 aston(curthread);
469 }
470
471 /* put out audit record for this syscall */
472 if (AU_AUDITING()) {
473 rval_t rval;
474
475 /* XX64 -- truncation of 64-bit return values? */
476 rval.r_val1 = (int)rval1;
477 rval.r_val2 = (int)rval2;
478 audit_finish(T_SYSCALL, code, error, &rval);
479 repost = 1;
480 }
481
482 if (curthread->t_pdmsg != NULL) {
483 char *m = curthread->t_pdmsg;
484
485 uprintf("%s", m);
486 kmem_free(m, strlen(m) + 1);
487 curthread->t_pdmsg = NULL;
488 }
489
490 /*
491 * If we're going to stop for /proc tracing, set the flag and
492 * save the arguments so that the return values don't smash them.
493 */
494 if (PTOU(p)->u_systrap) {
495 if (prismember(&PTOU(p)->u_exitmask, code)) {
496 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
497 (void) save_syscall_args();
498 proc_stop = 1;
499 }
500 repost = 1;
501 }
502
503 /*
504 * Similarly check to see if SIGPROF might be sent.
505 */
506 if (curthread->t_rprof != NULL &&
507 curthread->t_rprof->rp_anystate != 0) {
508 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
509 (void) save_syscall_args();
510 sigprof = 1;
511 }
512
513 if (lwp->lwp_eosys == NORMALRETURN) {
514 if (error == 0) {
515 #ifdef SYSCALLTRACE
516 if (syscalltrace) {
517 mutex_enter(&systrace_lock);
518 printf(
519 "%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
520 p->p_pid, rval1, rval2, curthread);
521 mutex_exit(&systrace_lock);
522 }
523 #endif /* SYSCALLTRACE */
524 rp->r_ps &= ~PS_C;
525 rp->r_r0 = rval1;
526 rp->r_r1 = rval2;
527 } else {
528 int sig;
529 #ifdef SYSCALLTRACE
530 if (syscalltrace) {
531 mutex_enter(&systrace_lock);
532 printf("%d: error=%d, id 0x%p\n",
533 p->p_pid, error, curthread);
534 mutex_exit(&systrace_lock);
535 }
536 #endif /* SYSCALLTRACE */
537 if (error == EINTR && t->t_activefd.a_stale)
538 error = EBADF;
539 if (error == EINTR &&
540 (sig = lwp->lwp_cursig) != 0 &&
541 sigismember(&PTOU(p)->u_sigrestart, sig) &&
542 PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
543 PTOU(p)->u_signal[sig - 1] != SIG_IGN)
544 error = ERESTART;
545 rp->r_r0 = error;
546 rp->r_ps |= PS_C;
547 }
548 }
549
550 /*
551 * From the proc(4) manual page:
552 * When exit from a system call is being traced, the traced process
553 * stops on completion of the system call just prior to checking for
554 * signals and returning to user level. At this point all return
555 * values have been stored into the traced process's saved registers.
556 */
557 if (proc_stop) {
558 mutex_enter(&p->p_lock);
559 if (PTOU(p)->u_systrap &&
560 prismember(&PTOU(p)->u_exitmask, code))
561 stop(PR_SYSEXIT, code);
562 mutex_exit(&p->p_lock);
563 }
564
565 /*
566 * If we are the parent returning from a successful
567 * vfork, wait for the child to exec or exit.
568 * This code must be here and not in the bowels of the system
569 * so that /proc can intercept exit from vfork in a timely way.
570 */
571 if (t->t_flag & T_VFPARENT) {
572 ASSERT(code == SYS_vfork || code == SYS_forksys);
573 ASSERT(rp->r_r1 == 0 && error == 0);
574 vfwait((pid_t)rval1);
575 t->t_flag &= ~T_VFPARENT;
576 }
577
578 /*
579 * If profiling is active, bill the current PC in user-land
580 * and keep reposting until profiling is disabled.
581 */
582 if (p->p_prof.pr_scale) {
583 if (lwp->lwp_oweupc)
584 profil_tick(rp->r_pc);
585 repost = 1;
586 }
587
588 sig_check:
589 /*
590 * Reset flag for next time.
591 * We must do this after stopping on PR_SYSEXIT
592 * because /proc uses the information in lwp_eosys.
593 */
594 lwp->lwp_eosys = NORMALRETURN;
595 clear_stale_fd();
596 t->t_flag &= ~T_FORKALL;
597
598 if (t->t_astflag | t->t_sig_check) {
599 /*
600 * Turn off the AST flag before checking all the conditions that
601 * may have caused an AST. This flag is on whenever a signal or
602 * unusual condition should be handled after the next trap or
603 * syscall.
604 */
605 astoff(t);
606 /*
607 * If a single-step trap occurred on a syscall (see trap())
608 * recognize it now. Do this before checking for signals
609 * because deferred_singlestep_trap() may generate a SIGTRAP to
610 * the LWP or may otherwise mark the LWP to call issig(FORREAL).
611 */
612 if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
613 deferred_singlestep_trap((caddr_t)rp->r_pc);
614
615 t->t_sig_check = 0;
616
617 /*
618 * The following check is legal for the following reasons:
619 * 1) The thread we are checking, is ourselves, so there is
620 * no way the proc can go away.
621 * 2) The only time we need to be protected by the
622 * lock is if the binding is changed.
623 *
624 * Note we will still take the lock and check the binding
625 * if the condition was true without the lock held. This
626 * prevents lock contention among threads owned by the
627 * same proc.
628 */
629
630 if (curthread->t_proc_flag & TP_CHANGEBIND) {
631 mutex_enter(&p->p_lock);
632 if (curthread->t_proc_flag & TP_CHANGEBIND) {
633 timer_lwpbind();
634 curthread->t_proc_flag &= ~TP_CHANGEBIND;
635 }
636 mutex_exit(&p->p_lock);
637 }
638
639 /*
640 * for kaio requests on the special kaio poll queue,
641 * copyout their results to user memory.
642 */
643 if (p->p_aio)
644 aio_cleanup(0);
645 /*
646 * If this LWP was asked to hold, call holdlwp(), which will
647 * stop. holdlwps() sets this up and calls pokelwps() which
648 * sets the AST flag.
649 *
650 * Also check TP_EXITLWP, since this is used by fresh new LWPs
651 * through lwp_rtt(). That flag is set if the lwp_create(2)
652 * syscall failed after creating the LWP.
653 */
654 if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
655 holdlwp();
656
657 /*
658 * All code that sets signals and makes ISSIG_PENDING
659 * evaluate true must set t_sig_check afterwards.
660 */
661 if (ISSIG_PENDING(t, lwp, p)) {
662 if (issig(FORREAL))
663 psig();
664 t->t_sig_check = 1; /* recheck next time */
665 }
666
667 if (sigprof) {
668 int nargs = (code > 0 && code < NSYSCALL)?
669 LWP_GETSYSENT(lwp)[code].sy_narg : 0;
670 realsigprof(code, nargs, error);
671 t->t_sig_check = 1; /* recheck next time */
672 }
673
674 /*
675 * If a performance counter overflow interrupt was
676 * delivered *during* the syscall, then re-enable the
677 * AST so that we take a trip through trap() to cause
678 * the SIGEMT to be delivered.
679 */
680 if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
681 aston(t);
682
683 /*
684 * /proc can't enable/disable the trace bit itself
685 * because that could race with the call gate used by
686 * system calls via "lcall". If that happened, an
687 * invalid EFLAGS would result. prstep()/prnostep()
688 * therefore schedule an AST for the purpose.
689 */
690 if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
691 lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
692 rp->r_ps |= PS_T;
693 }
694 if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
695 lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
696 rp->r_ps &= ~PS_T;
697 }
698 }
699
700 lwp->lwp_errno = 0; /* clear error for next time */
701
702 #ifndef NPROBE
703 /* Kernel probe */
704 if (tnf_tracing_active) {
705 TNF_PROBE_3(syscall_end, "syscall thread", /* CSTYLED */,
706 tnf_long, rval1, rval1,
707 tnf_long, rval2, rval2,
708 tnf_long, errno, (long)error);
709 repost = 1;
710 }
711 #endif /* NPROBE */
712
713 /*
714 * Set state to LWP_USER here so preempt won't give us a kernel
715 * priority if it occurs after this point. Call CL_TRAPRET() to
716 * restore the user-level priority.
717 *
718 * It is important that no locks (other than spinlocks) be entered
719 * after this point before returning to user mode (unless lwp_state
720 * is set back to LWP_SYS).
721 *
722 * XXX Sampled times past this point are charged to the user.
723 */
724 lwp->lwp_state = LWP_USER;
725
726 if (t->t_trapret) {
727 t->t_trapret = 0;
728 thread_lock(t);
729 CL_TRAPRET(t);
730 thread_unlock(t);
731 }
732 if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ)
733 preempt();
734 prunstop();
735
736 lwp->lwp_errno = 0; /* clear error for next time */
737
738 /*
739 * The thread lock must be held in order to clear sysnum and reset
740 * lwp_ap atomically with respect to other threads in the system that
741 * may be looking at the args via lwp_ap from get_syscall_args().
742 */
743
744 thread_lock(t);
745 t->t_sysnum = 0; /* no longer in a system call */
746
747 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
748 #if defined(_LP64)
749 /*
750 * In case the args were copied to the lwp, reset the
751 * pointer so the next syscall will have the right
752 * lwp_ap pointer.
753 */
754 lwp->lwp_ap = (long *)&rp->r_rdi;
755 } else {
756 #endif
757 lwp->lwp_ap = NULL; /* reset on every syscall entry */
758 }
759 thread_unlock(t);
760
761 lwp->lwp_argsaved = 0;
762
763 /*
764 * If there was a continuing reason for post-syscall processing,
765 * set the t_post_sys flag for the next system call.
766 */
767 if (repost)
768 t->t_post_sys = 1;
769
770 /*
771 * If there is a ustack registered for this lwp, and the stack rlimit
772 * has been altered, read in the ustack. If the saved stack rlimit
773 * matches the bounds of the ustack, update the ustack to reflect
774 * the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
775 * stack checking by setting the size to 0.
776 */
777 if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
778 rlim64_t new_size;
779 caddr_t top;
780 stack_t stk;
781 struct rlimit64 rl;
782
783 mutex_enter(&p->p_lock);
784 new_size = p->p_stk_ctl;
785 top = p->p_usrstack;
786 (void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
787 mutex_exit(&p->p_lock);
788
789 if (rl.rlim_cur == RLIM64_INFINITY)
790 new_size = 0;
791
792 if (copyin((stack_t *)lwp->lwp_ustack, &stk,
793 sizeof (stack_t)) == 0 &&
794 (stk.ss_size == lwp->lwp_old_stk_ctl ||
795 stk.ss_size == 0) &&
796 stk.ss_sp == top - stk.ss_size) {
797 stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
798 stk.ss_size - (uintptr_t)new_size);
799 stk.ss_size = new_size;
800
801 (void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
802 sizeof (stack_t));
803 }
804
805 lwp->lwp_old_stk_ctl = 0;
806 }
807 }
808
809 /*
810 * Called from post_syscall() when a deferred singlestep is to be taken.
811 */
812 void
deferred_singlestep_trap(caddr_t pc)813 deferred_singlestep_trap(caddr_t pc)
814 {
815 proc_t *p = ttoproc(curthread);
816 klwp_t *lwp = ttolwp(curthread);
817 pcb_t *pcb = &lwp->lwp_pcb;
818 uint_t fault = 0;
819 k_siginfo_t siginfo;
820
821 bzero(&siginfo, sizeof (siginfo));
822
823 /*
824 * If both NORMAL_STEP and WATCH_STEP are in
825 * effect, give precedence to WATCH_STEP.
826 * If neither is set, user must have set the
827 * PS_T bit in %efl; treat this as NORMAL_STEP.
828 */
829 if ((fault = undo_watch_step(&siginfo)) == 0 &&
830 ((pcb->pcb_flags & NORMAL_STEP) ||
831 !(pcb->pcb_flags & WATCH_STEP))) {
832 siginfo.si_signo = SIGTRAP;
833 siginfo.si_code = TRAP_TRACE;
834 siginfo.si_addr = pc;
835 fault = FLTTRACE;
836 }
837 pcb->pcb_flags &= ~(DEBUG_PENDING|NORMAL_STEP|WATCH_STEP);
838
839 if (fault) {
840 /*
841 * Remember the fault and fault adddress
842 * for real-time (SIGPROF) profiling.
843 */
844 lwp->lwp_lastfault = fault;
845 lwp->lwp_lastfaddr = siginfo.si_addr;
846 /*
847 * If a debugger has declared this fault to be an
848 * event of interest, stop the lwp. Otherwise just
849 * deliver the associated signal.
850 */
851 if (prismember(&p->p_fltmask, fault) &&
852 stop_on_fault(fault, &siginfo) == 0)
853 siginfo.si_signo = 0;
854 }
855
856 if (siginfo.si_signo)
857 trapsig(&siginfo, 1);
858 }
859
860 /*
861 * nonexistent system call-- signal lwp (may want to handle it)
862 * flag error if lwp won't see signal immediately
863 */
864 int64_t
nosys()865 nosys()
866 {
867 tsignal(curthread, SIGSYS);
868 return (set_errno(ENOSYS));
869 }
870
871 /*
872 * Execute a 32-bit system call on behalf of the current thread.
873 */
874 void
dosyscall(void)875 dosyscall(void)
876 {
877 /*
878 * Need space on the stack to store syscall arguments.
879 */
880 long syscall_args[MAXSYSARGS];
881 struct sysent *se;
882 int64_t ret;
883
884 syscall_mstate(LMS_TRAP, LMS_SYSTEM);
885
886 ASSERT(curproc->p_model == DATAMODEL_ILP32);
887
888 CPU_STATS_ENTER_K();
889 CPU_STATS_ADDQ(CPU, sys, syscall, 1);
890 CPU_STATS_EXIT_K();
891
892 se = syscall_entry(curthread, syscall_args);
893
894 /*
895 * syscall_entry() copied all 8 arguments into syscall_args.
896 */
897 ret = se->sy_callc(syscall_args[0], syscall_args[1], syscall_args[2],
898 syscall_args[3], syscall_args[4], syscall_args[5], syscall_args[6],
899 syscall_args[7]);
900
901 syscall_exit(curthread, (int)ret & 0xffffffffu, (int)(ret >> 32));
902 syscall_mstate(LMS_SYSTEM, LMS_TRAP);
903 }
904
905 /*
906 * Get the arguments to the current system call. See comment atop
907 * save_syscall_args() regarding lwp_ap usage.
908 */
909
910 uint_t
get_syscall_args(klwp_t * lwp,long * argp,int * nargsp)911 get_syscall_args(klwp_t *lwp, long *argp, int *nargsp)
912 {
913 kthread_t *t = lwptot(lwp);
914 ulong_t mask = 0xfffffffful;
915 uint_t code;
916 long *ap;
917 int nargs;
918
919 #if defined(_LP64)
920 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
921 mask = 0xfffffffffffffffful;
922 #endif
923
924 /*
925 * The thread lock must be held while looking at the arguments to ensure
926 * they don't go away via post_syscall().
927 * get_syscall_args() is the only routine to read them which is callable
928 * outside the LWP in question and hence the only one that must be
929 * synchronized in this manner.
930 */
931 thread_lock(t);
932
933 code = t->t_sysnum;
934 ap = lwp->lwp_ap;
935
936 thread_unlock(t);
937
938 if (code != 0 && code < NSYSCALL) {
939 nargs = LWP_GETSYSENT(lwp)[code].sy_narg;
940
941 ASSERT(nargs <= MAXSYSARGS);
942
943 *nargsp = nargs;
944 while (nargs-- > 0)
945 *argp++ = *ap++ & mask;
946 } else {
947 *nargsp = 0;
948 }
949
950 return (code);
951 }
952
953 #ifdef _SYSCALL32_IMPL
954 /*
955 * Get the arguments to the current 32-bit system call.
956 */
957 uint_t
get_syscall32_args(klwp_t * lwp,int * argp,int * nargsp)958 get_syscall32_args(klwp_t *lwp, int *argp, int *nargsp)
959 {
960 long args[MAXSYSARGS];
961 uint_t i, code;
962
963 code = get_syscall_args(lwp, args, nargsp);
964
965 for (i = 0; i != *nargsp; i++)
966 *argp++ = (int)args[i];
967 return (code);
968 }
969 #endif
970
971 /*
972 * Save the system call arguments in a safe place.
973 *
974 * On the i386 kernel:
975 *
976 * Copy the users args prior to changing the stack or stack pointer.
977 * This is so /proc will be able to get a valid copy of the
978 * args from the user stack even after the user stack has been changed.
979 * Note that the kernel stack copy of the args may also have been
980 * changed by a system call handler which takes C-style arguments.
981 *
982 * Note that this may be called by stop() from trap(). In that case
983 * t_sysnum will be zero (syscall_exit clears it), so no args will be
984 * copied.
985 *
986 * On the amd64 kernel:
987 *
988 * For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9
989 * in the reg structure. If the user is going to change the argument
990 * registers, rax, or the stack and might want to get the args (for
991 * /proc tracing), it must copy the args elsewhere via save_syscall_args().
992 *
993 * For 32-bit applications, lwp->lwp_ap normally points to a copy of
994 * the system call arguments on the kernel stack made from the user
995 * stack. Copy the args prior to change the stack or stack pointer.
996 * This is so /proc will be able to get a valid copy of the args
997 * from the user stack even after that stack has been changed.
998 *
999 * This may be called from stop() even when we're not in a system call.
1000 * Since there's no easy way to tell, this must be safe (not panic).
1001 * If the copyins get data faults, return non-zero.
1002 */
1003 int
save_syscall_args()1004 save_syscall_args()
1005 {
1006 kthread_t *t = curthread;
1007 klwp_t *lwp = ttolwp(t);
1008 uint_t code = t->t_sysnum;
1009 uint_t nargs;
1010
1011 if (lwp->lwp_argsaved || code == 0)
1012 return (0); /* args already saved or not needed */
1013
1014 if (code >= NSYSCALL) {
1015 nargs = 0; /* illegal syscall */
1016 } else {
1017 struct sysent *se = LWP_GETSYSENT(lwp);
1018 struct sysent *callp = se + code;
1019
1020 nargs = callp->sy_narg;
1021 if (LOADABLE_SYSCALL(callp) && nargs == 0) {
1022 krwlock_t *module_lock;
1023
1024 /*
1025 * Find out how many arguments the system
1026 * call uses.
1027 *
1028 * We have the property that loaded syscalls
1029 * never change the number of arguments they
1030 * use after they've been loaded once. This
1031 * allows us to stop for /proc tracing without
1032 * holding the module lock.
1033 * /proc is assured that sy_narg is valid.
1034 */
1035 module_lock = lock_syscall(se, code);
1036 nargs = callp->sy_narg;
1037 rw_exit(module_lock);
1038 }
1039 }
1040
1041 /*
1042 * Fetch the system call arguments.
1043 */
1044 if (nargs == 0)
1045 goto out;
1046
1047 ASSERT(nargs <= MAXSYSARGS);
1048
1049 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1050 #if defined(_LP64)
1051 struct regs *rp = lwptoregs(lwp);
1052
1053 lwp->lwp_arg[0] = rp->r_rdi;
1054 lwp->lwp_arg[1] = rp->r_rsi;
1055 lwp->lwp_arg[2] = rp->r_rdx;
1056 lwp->lwp_arg[3] = rp->r_rcx;
1057 lwp->lwp_arg[4] = rp->r_r8;
1058 lwp->lwp_arg[5] = rp->r_r9;
1059 if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6))
1060 return (-1);
1061 } else {
1062 #endif
1063 if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs))
1064 return (-1);
1065 }
1066 out:
1067 lwp->lwp_ap = lwp->lwp_arg;
1068 lwp->lwp_argsaved = 1;
1069 t->t_post_sys = 1; /* so lwp_ap will be reset */
1070 return (0);
1071 }
1072
1073 void
reset_syscall_args(void)1074 reset_syscall_args(void)
1075 {
1076 ttolwp(curthread)->lwp_argsaved = 0;
1077 }
1078
1079 /*
1080 * Call a system call which takes a pointer to the user args struct and
1081 * a pointer to the return values. This is a bit slower than the standard
1082 * C arg-passing method in some cases.
1083 */
1084 int64_t
syscall_ap(void)1085 syscall_ap(void)
1086 {
1087 uint_t error;
1088 struct sysent *callp;
1089 rval_t rval;
1090 kthread_t *t = curthread;
1091 klwp_t *lwp = ttolwp(t);
1092 struct regs *rp = lwptoregs(lwp);
1093
1094 callp = LWP_GETSYSENT(lwp) + t->t_sysnum;
1095
1096 #if defined(__amd64)
1097 /*
1098 * If the arguments don't fit in registers %rdi-%r9, make sure they
1099 * have been copied to the lwp_arg array.
1100 */
1101 if (callp->sy_narg > 6 && save_syscall_args())
1102 return ((int64_t)set_errno(EFAULT));
1103 #endif
1104
1105 rval.r_val1 = 0;
1106 rval.r_val2 = rp->r_r1;
1107 lwp->lwp_error = 0; /* for old drivers */
1108 error = (*(callp->sy_call))(lwp->lwp_ap, &rval);
1109 if (error)
1110 return ((longlong_t)set_errno(error));
1111 return (rval.r_vals);
1112 }
1113
1114 /*
1115 * Load system call module.
1116 * Returns with pointer to held read lock for module.
1117 */
1118 static krwlock_t *
lock_syscall(struct sysent * table,uint_t code)1119 lock_syscall(struct sysent *table, uint_t code)
1120 {
1121 krwlock_t *module_lock;
1122 struct modctl *modp;
1123 int id;
1124 struct sysent *callp;
1125
1126 callp = table + code;
1127 module_lock = callp->sy_lock;
1128
1129 /*
1130 * Optimization to only call modload if we don't have a loaded
1131 * syscall.
1132 */
1133 rw_enter(module_lock, RW_READER);
1134 if (LOADED_SYSCALL(callp))
1135 return (module_lock);
1136 rw_exit(module_lock);
1137
1138 for (;;) {
1139 if ((id = modload("sys", syscallnames[code])) == -1)
1140 break;
1141
1142 /*
1143 * If we loaded successfully at least once, the modctl
1144 * will still be valid, so we try to grab it by filename.
1145 * If this call fails, it's because the mod_filename
1146 * was changed after the call to modload() (mod_hold_by_name()
1147 * is the likely culprit). We can safely just take
1148 * another lap if this is the case; the modload() will
1149 * change the mod_filename back to one by which we can
1150 * find the modctl.
1151 */
1152 modp = mod_find_by_filename("sys", syscallnames[code]);
1153
1154 if (modp == NULL)
1155 continue;
1156
1157 mutex_enter(&mod_lock);
1158
1159 if (!modp->mod_installed) {
1160 mutex_exit(&mod_lock);
1161 continue;
1162 }
1163 break;
1164 }
1165 rw_enter(module_lock, RW_READER);
1166
1167 if (id != -1)
1168 mutex_exit(&mod_lock);
1169
1170 return (module_lock);
1171 }
1172
1173 /*
1174 * Loadable syscall support.
1175 * If needed, load the module, then reserve it by holding a read
1176 * lock for the duration of the call.
1177 * Later, if the syscall is not unloadable, it could patch the vector.
1178 */
1179 /*ARGSUSED*/
1180 int64_t
loadable_syscall(long a0,long a1,long a2,long a3,long a4,long a5,long a6,long a7)1181 loadable_syscall(
1182 long a0, long a1, long a2, long a3,
1183 long a4, long a5, long a6, long a7)
1184 {
1185 klwp_t *lwp = ttolwp(curthread);
1186 int64_t rval;
1187 struct sysent *callp;
1188 struct sysent *se = LWP_GETSYSENT(lwp);
1189 krwlock_t *module_lock;
1190 int code, error = 0;
1191 int64_t (*sy_call)();
1192
1193 code = curthread->t_sysnum;
1194 callp = se + code;
1195
1196 /*
1197 * Try to autoload the system call if necessary
1198 */
1199 module_lock = lock_syscall(se, code);
1200
1201 /*
1202 * we've locked either the loaded syscall or nosys
1203 */
1204
1205 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
1206 #if defined(_LP64)
1207 if (callp->sy_flags & SE_ARGC) {
1208 sy_call = (int64_t (*)())callp->sy_call;
1209 rval = (*sy_call)(a0, a1, a2, a3, a4, a5);
1210 } else
1211 rval = syscall_ap();
1212 } else {
1213 #endif
1214 /*
1215 * Now that it's loaded, make sure enough args were copied.
1216 */
1217 if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg))
1218 error = EFAULT;
1219 if (error) {
1220 rval = set_errno(error);
1221 } else if (callp->sy_flags & SE_ARGC) {
1222 sy_call = (int64_t (*)())callp->sy_call;
1223 rval = (*sy_call)(lwp->lwp_ap[0], lwp->lwp_ap[1],
1224 lwp->lwp_ap[2], lwp->lwp_ap[3], lwp->lwp_ap[4],
1225 lwp->lwp_ap[5]);
1226 } else
1227 rval = syscall_ap();
1228 }
1229
1230 rw_exit(module_lock);
1231 return (rval);
1232 }
1233
1234 /*
1235 * Indirect syscall handled in libc on x86 architectures
1236 */
1237 int64_t
indir()1238 indir()
1239 {
1240 return (nosys());
1241 }
1242
1243 /*
1244 * set_errno - set an error return from the current system call.
1245 * This could be a macro.
1246 * This returns the value it is passed, so that the caller can
1247 * use tail-recursion-elimination and do return (set_errno(ERRNO));
1248 */
1249 uint_t
set_errno(uint_t error)1250 set_errno(uint_t error)
1251 {
1252 ASSERT(error != 0); /* must not be used to clear errno */
1253
1254 curthread->t_post_sys = 1; /* have post_syscall do error return */
1255 return (ttolwp(curthread)->lwp_errno = error);
1256 }
1257
1258 /*
1259 * set_proc_pre_sys - Set pre-syscall processing for entire process.
1260 */
1261 void
set_proc_pre_sys(proc_t * p)1262 set_proc_pre_sys(proc_t *p)
1263 {
1264 kthread_t *t;
1265 kthread_t *first;
1266
1267 ASSERT(MUTEX_HELD(&p->p_lock));
1268
1269 t = first = p->p_tlist;
1270 do {
1271 t->t_pre_sys = 1;
1272 } while ((t = t->t_forw) != first);
1273 }
1274
1275 /*
1276 * set_proc_post_sys - Set post-syscall processing for entire process.
1277 */
1278 void
set_proc_post_sys(proc_t * p)1279 set_proc_post_sys(proc_t *p)
1280 {
1281 kthread_t *t;
1282 kthread_t *first;
1283
1284 ASSERT(MUTEX_HELD(&p->p_lock));
1285
1286 t = first = p->p_tlist;
1287 do {
1288 t->t_post_sys = 1;
1289 } while ((t = t->t_forw) != first);
1290 }
1291
1292 /*
1293 * set_proc_sys - Set pre- and post-syscall processing for entire process.
1294 */
1295 void
set_proc_sys(proc_t * p)1296 set_proc_sys(proc_t *p)
1297 {
1298 kthread_t *t;
1299 kthread_t *first;
1300
1301 ASSERT(MUTEX_HELD(&p->p_lock));
1302
1303 t = first = p->p_tlist;
1304 do {
1305 t->t_pre_sys = 1;
1306 t->t_post_sys = 1;
1307 } while ((t = t->t_forw) != first);
1308 }
1309
1310 /*
1311 * set_all_proc_sys - set pre- and post-syscall processing flags for all
1312 * user processes.
1313 *
1314 * This is needed when auditing, tracing, or other facilities which affect
1315 * all processes are turned on.
1316 */
1317 void
set_all_proc_sys()1318 set_all_proc_sys()
1319 {
1320 kthread_t *t;
1321 kthread_t *first;
1322
1323 mutex_enter(&pidlock);
1324 t = first = curthread;
1325 do {
1326 t->t_pre_sys = 1;
1327 t->t_post_sys = 1;
1328 } while ((t = t->t_next) != first);
1329 mutex_exit(&pidlock);
1330 }
1331
1332 /*
1333 * set_all_zone_usr_proc_sys - set pre- and post-syscall processing flags for
1334 * all user processes running in the zone of the current process
1335 *
1336 * This is needed when auditing, tracing, or other facilities which affect
1337 * all processes are turned on.
1338 */
1339 void
set_all_zone_usr_proc_sys(zoneid_t zoneid)1340 set_all_zone_usr_proc_sys(zoneid_t zoneid)
1341 {
1342 proc_t *p;
1343 kthread_t *t;
1344
1345 mutex_enter(&pidlock);
1346 for (p = practive; p != NULL; p = p->p_next) {
1347 /* skip kernel and incomplete processes */
1348 if (p->p_exec == NULLVP || p->p_as == &kas ||
1349 p->p_stat == SIDL || p->p_stat == SZOMB ||
1350 (p->p_flag & (SSYS | SEXITING | SEXITLWPS)))
1351 continue;
1352 /*
1353 * Only processes in the given zone (eventually in
1354 * all zones) are taken into account
1355 */
1356 if (zoneid == ALL_ZONES || p->p_zone->zone_id == zoneid) {
1357 mutex_enter(&p->p_lock);
1358 if ((t = p->p_tlist) == NULL) {
1359 mutex_exit(&p->p_lock);
1360 continue;
1361 }
1362 /*
1363 * Set pre- and post-syscall processing flags
1364 * for all threads of the process
1365 */
1366 do {
1367 t->t_pre_sys = 1;
1368 t->t_post_sys = 1;
1369 } while (p->p_tlist != (t = t->t_forw));
1370 mutex_exit(&p->p_lock);
1371 }
1372 }
1373 mutex_exit(&pidlock);
1374 }
1375
1376 /*
1377 * set_proc_ast - Set asynchronous service trap (AST) flag for all
1378 * threads in process.
1379 */
1380 void
set_proc_ast(proc_t * p)1381 set_proc_ast(proc_t *p)
1382 {
1383 kthread_t *t;
1384 kthread_t *first;
1385
1386 ASSERT(MUTEX_HELD(&p->p_lock));
1387
1388 t = first = p->p_tlist;
1389 do {
1390 aston(t);
1391 } while ((t = t->t_forw) != first);
1392 }
1393