1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#include <sys/asm_linkage.h> 27#include <sys/asm_misc.h> 28#include <sys/regset.h> 29#include <sys/privregs.h> 30#include <sys/psw.h> 31#include <sys/machbrand.h> 32 33#if defined(__lint) 34 35#include <sys/types.h> 36#include <sys/thread.h> 37#include <sys/systm.h> 38 39#else /* __lint */ 40 41#include <sys/segments.h> 42#include <sys/pcb.h> 43#include <sys/trap.h> 44#include <sys/ftrace.h> 45#include <sys/traptrace.h> 46#include <sys/clock.h> 47#include <sys/model.h> 48#include <sys/panic.h> 49 50#if defined(__xpv) 51#include <sys/hypervisor.h> 52#endif 53 54#include "assym.h" 55 56#endif /* __lint */ 57 58/* 59 * We implement five flavours of system call entry points 60 * 61 * - syscall/sysretq (amd64 generic) 62 * - syscall/sysretl (i386 plus SYSC bit) 63 * - sysenter/sysexit (i386 plus SEP bit) 64 * - int/iret (i386 generic) 65 * - lcall/iret (i386 generic) 66 * 67 * The current libc included in Solaris uses int/iret as the base unoptimized 68 * kernel entry method. Older libc implementations and legacy binaries may use 69 * the lcall call gate, so it must continue to be supported. 70 * 71 * System calls that use an lcall call gate are processed in trap() via a 72 * segment-not-present trap, i.e. lcalls are extremely slow(!). 73 * 74 * The basic pattern used in the 32-bit SYSC handler at this point in time is 75 * to have the bare minimum of assembler, and get to the C handlers as 76 * quickly as possible. 77 * 78 * The 64-bit handler is much closer to the sparcv9 handler; that's 79 * because of passing arguments in registers. The 32-bit world still 80 * passes arguments on the stack -- that makes that handler substantially 81 * more complex. 82 * 83 * The two handlers share a few code fragments which are broken 84 * out into preprocessor macros below. 85 * 86 * XX64 come back and speed all this up later. The 32-bit stuff looks 87 * especially easy to speed up the argument copying part .. 88 * 89 * 90 * Notes about segment register usage (c.f. the 32-bit kernel) 91 * 92 * In the 32-bit kernel, segment registers are dutifully saved and 93 * restored on all mode transitions because the kernel uses them directly. 94 * When the processor is running in 64-bit mode, segment registers are 95 * largely ignored. 96 * 97 * %cs and %ss 98 * controlled by the hardware mechanisms that make mode transitions 99 * 100 * The remaining segment registers have to either be pointing at a valid 101 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors 102 * 103 * %ds and %es 104 * always ignored 105 * 106 * %fs and %gs 107 * fsbase and gsbase are used to control the place they really point at. 108 * The kernel only depends on %gs, and controls its own gsbase via swapgs 109 * 110 * Note that loading segment registers is still costly because the GDT 111 * lookup still happens (this is because the hardware can't know that we're 112 * not setting up these segment registers for a 32-bit program). Thus we 113 * avoid doing this in the syscall path, and defer them to lwp context switch 114 * handlers, so the register values remain virtualized to the lwp. 115 */ 116 117#if defined(SYSCALLTRACE) 118#define ORL_SYSCALLTRACE(r32) \ 119 orl syscalltrace(%rip), r32 120#else 121#define ORL_SYSCALLTRACE(r32) 122#endif 123 124/* 125 * In the 32-bit kernel, we do absolutely nothing before getting into the 126 * brand callback checks. In 64-bit land, we do swapgs and then come here. 127 * We assume that the %rsp- and %r15-stashing fields in the CPU structure 128 * are still unused. 129 * 130 * When the callback is invoked, we will be on the user's %gs and 131 * the stack will look like this: 132 * 133 * stack: -------------------------------------- 134 * | callback pointer | 135 * | | user stack pointer | 136 * | | lwp pointer | 137 * v | userland return address | 138 * | callback wrapper return addr | 139 * -------------------------------------- 140 * 141 */ 142#define BRAND_CALLBACK(callback_id) \ 143 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\ 144 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\ 145 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\ 146 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\ 147 subq $16, %rsp /* save space for two pointers */ ;\ 148 pushq %r14 /* save %r14 */ ;\ 149 movq %gs:CPU_RTMP_RSP, %r14 ;\ 150 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\ 151 popq %r14 /* restore %r14 */ ;\ 152 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\ 153 pushq %r15 /* push the lwp pointer */ ;\ 154 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\ 155 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\ 156 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\ 157 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\ 158 cmpq $0, %r15 ;\ 159 je 1f ;\ 160 movq %r15, 16(%rsp) /* save the callback pointer */ ;\ 161 movq %gs:CPU_RTMP_RSP, %r15 /* grab the user stack pointer */ ;\ 162 pushq (%r15) /* push the return address */ ;\ 163 movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\ 164 SWAPGS /* user gsbase */ ;\ 165 call *24(%rsp) /* call callback */ ;\ 166 SWAPGS /* kernel gsbase */ ;\ 1671: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\ 168 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */ 169 170#define MSTATE_TRANSITION(from, to) \ 171 movl $from, %edi; \ 172 movl $to, %esi; \ 173 call syscall_mstate 174 175/* 176 * Check to see if a simple (direct) return is possible i.e. 177 * 178 * if (t->t_post_sys_ast | syscalltrace | 179 * lwp->lwp_pcb.pcb_rupdate == 1) 180 * do full version ; 181 * 182 * Preconditions: 183 * - t is curthread 184 * Postconditions: 185 * - condition code NE is set if post-sys is too complex 186 * - rtmp is zeroed if it isn't (we rely on this!) 187 * - ltmp is smashed 188 */ 189#define CHECK_POSTSYS_NE(t, ltmp, rtmp) \ 190 movq T_LWP(t), ltmp; \ 191 movzbl PCB_RUPDATE(ltmp), rtmp; \ 192 ORL_SYSCALLTRACE(rtmp); \ 193 orl T_POST_SYS_AST(t), rtmp; \ 194 cmpl $0, rtmp 195 196/* 197 * Fix up the lwp, thread, and eflags for a successful return 198 * 199 * Preconditions: 200 * - zwreg contains zero 201 */ 202#define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \ 203 movb $LWP_USER, LWP_STATE(lwp); \ 204 movw zwreg, T_SYSNUM(t); \ 205 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp) 206 207/* 208 * ASSERT(lwptoregs(lwp) == rp); 209 * 210 * This may seem obvious, but very odd things happen if this 211 * assertion is false 212 * 213 * Preconditions: 214 * (%rsp is ready for normal call sequence) 215 * Postconditions (if assertion is true): 216 * %r11 is smashed 217 * 218 * ASSERT(rp->r_cs == descnum) 219 * 220 * The code selector is written into the regs structure when the 221 * lwp stack is created. We use this ASSERT to validate that 222 * the regs structure really matches how we came in. 223 * 224 * Preconditions: 225 * (%rsp is ready for normal call sequence) 226 * Postconditions (if assertion is true): 227 * -none- 228 * 229 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0); 230 * 231 * If this is false, it meant that we returned to userland without 232 * updating the segment registers as we were supposed to. 233 * 234 * Note that we must ensure no interrupts or other traps intervene 235 * between entering privileged mode and performing the assertion, 236 * otherwise we may perform a context switch on the thread, which 237 * will end up setting pcb_rupdate to 1 again. 238 */ 239#if defined(DEBUG) 240 241#if !defined(__lint) 242 243__lwptoregs_msg: 244 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]" 245 246__codesel_msg: 247 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld" 248 249__no_rupdate_msg: 250 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0" 251 252#endif /* !__lint */ 253 254#define ASSERT_LWPTOREGS(lwp, rp) \ 255 movq LWP_REGS(lwp), %r11; \ 256 cmpq rp, %r11; \ 257 je 7f; \ 258 leaq __lwptoregs_msg(%rip), %rdi; \ 259 movl $__LINE__, %esi; \ 260 movq lwp, %rdx; \ 261 movq %r11, %rcx; \ 262 movq rp, %r8; \ 263 xorl %eax, %eax; \ 264 call panic; \ 2657: 266 267#define ASSERT_NO_RUPDATE_PENDING(lwp) \ 268 testb $0x1, PCB_RUPDATE(lwp); \ 269 je 8f; \ 270 movq lwp, %rdx; \ 271 leaq __no_rupdate_msg(%rip), %rdi; \ 272 movl $__LINE__, %esi; \ 273 xorl %eax, %eax; \ 274 call panic; \ 2758: 276 277#else 278#define ASSERT_LWPTOREGS(lwp, rp) 279#define ASSERT_NO_RUPDATE_PENDING(lwp) 280#endif 281 282/* 283 * Do the traptrace thing and restore any registers we used 284 * in situ. Assumes that %rsp is pointing at the base of 285 * the struct regs, obviously .. 286 */ 287#ifdef TRAPTRACE 288#define SYSCALL_TRAPTRACE(ttype) \ 289 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \ 290 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \ 291 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \ 292 movq REGOFF_RAX(%rsp), %rax; \ 293 movq REGOFF_RBX(%rsp), %rbx; \ 294 movq REGOFF_RCX(%rsp), %rcx; \ 295 movq REGOFF_RDX(%rsp), %rdx; \ 296 movl %eax, TTR_SYSNUM(%rdi); \ 297 movq REGOFF_RDI(%rsp), %rdi 298 299#define SYSCALL_TRAPTRACE32(ttype) \ 300 SYSCALL_TRAPTRACE(ttype); \ 301 /* paranoia: clean the top 32-bits of the registers */ \ 302 orl %eax, %eax; \ 303 orl %ebx, %ebx; \ 304 orl %ecx, %ecx; \ 305 orl %edx, %edx; \ 306 orl %edi, %edi 307#else /* TRAPTRACE */ 308#define SYSCALL_TRAPTRACE(ttype) 309#define SYSCALL_TRAPTRACE32(ttype) 310#endif /* TRAPTRACE */ 311 312/* 313 * The 64-bit libc syscall wrapper does this: 314 * 315 * fn(<args>) 316 * { 317 * movq %rcx, %r10 -- because syscall smashes %rcx 318 * movl $CODE, %eax 319 * syscall 320 * <error processing> 321 * } 322 * 323 * Thus when we come into the kernel: 324 * 325 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args 326 * %rax is the syscall number 327 * %r12-%r15 contain caller state 328 * 329 * The syscall instruction arranges that: 330 * 331 * %rcx contains the return %rip 332 * %r11d contains bottom 32-bits of %rflags 333 * %rflags is masked (as determined by the SFMASK msr) 334 * %cs is set to UCS_SEL (as determined by the STAR msr) 335 * %ss is set to UDS_SEL (as determined by the STAR msr) 336 * %rip is set to sys_syscall (as determined by the LSTAR msr) 337 * 338 * Or in other words, we have no registers available at all. 339 * Only swapgs can save us! 340 * 341 * Under the hypervisor, the swapgs has happened already. However, the 342 * state of the world is very different from that we're familiar with. 343 * 344 * In particular, we have a stack structure like that for interrupt 345 * gates, except that the %cs and %ss registers are modified for reasons 346 * that are not entirely clear. Critically, the %rcx/%r11 values do 347 * *not* reflect the usage of those registers under a 'real' syscall[1]; 348 * the stack, therefore, looks like this: 349 * 350 * 0x0(rsp) potentially junk %rcx 351 * 0x8(rsp) potentially junk %r11 352 * 0x10(rsp) user %rip 353 * 0x18(rsp) modified %cs 354 * 0x20(rsp) user %rflags 355 * 0x28(rsp) user %rsp 356 * 0x30(rsp) modified %ss 357 * 358 * 359 * and before continuing on, we must load the %rip into %rcx and the 360 * %rflags into %r11. 361 * 362 * [1] They used to, and we relied on it, but this was broken in 3.1.1. 363 * Sigh. 364 */ 365 366#if defined(__xpv) 367#define XPV_SYSCALL_PROD \ 368 XPV_TRAP_POP; \ 369 movq (%rsp), %rcx; \ 370 movq 0x10(%rsp), %r11 371#else 372#define XPV_SYSCALL_PROD /* nothing */ 373#endif 374 375#if defined(__lint) 376 377/*ARGSUSED*/ 378void 379sys_syscall() 380{} 381 382void 383_allsyscalls() 384{} 385 386size_t _allsyscalls_size; 387 388#else /* __lint */ 389 390 ENTRY_NP2(brand_sys_syscall,_allsyscalls) 391 SWAPGS /* kernel gsbase */ 392 XPV_SYSCALL_PROD 393 BRAND_CALLBACK(BRAND_CB_SYSCALL) 394 SWAPGS /* user gsbase */ 395 396#if defined(__xpv) 397 jmp noprod_sys_syscall 398#endif 399 400 ALTENTRY(sys_syscall) 401 SWAPGS /* kernel gsbase */ 402 XPV_SYSCALL_PROD 403 404noprod_sys_syscall: 405 ASSERT_UPCALL_MASK_IS_SET 406 407 movq %r15, %gs:CPU_RTMP_R15 408#if defined(__xpv) 409 movq 0x18(%rsp), %r15 /* save user stack */ 410 movq %r15, %gs:CPU_RTMP_RSP 411#else 412 movq %rsp, %gs:CPU_RTMP_RSP 413#endif /* __xpv */ 414 415 movq %gs:CPU_THREAD, %r15 416 movq T_STACK(%r15), %rsp 417 418 movl $UCS_SEL, REGOFF_CS(%rsp) 419 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 420 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 421 movl $UDS_SEL, REGOFF_SS(%rsp) 422 423 movl %eax, %eax /* wrapper: sysc# -> %eax */ 424 movq %rdi, REGOFF_RDI(%rsp) 425 movq %rsi, REGOFF_RSI(%rsp) 426 movq %rdx, REGOFF_RDX(%rsp) 427 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */ 428 movq %r10, %rcx /* arg[3] for direct calls */ 429 430 movq %r8, REGOFF_R8(%rsp) 431 movq %r9, REGOFF_R9(%rsp) 432 movq %rax, REGOFF_RAX(%rsp) 433 movq %rbx, REGOFF_RBX(%rsp) 434 435 movq %rbp, REGOFF_RBP(%rsp) 436 movq %r10, REGOFF_R10(%rsp) 437 movq %gs:CPU_RTMP_RSP, %r11 438 movq %r11, REGOFF_RSP(%rsp) 439 movq %r12, REGOFF_R12(%rsp) 440 441 movq %r13, REGOFF_R13(%rsp) 442 movq %r14, REGOFF_R14(%rsp) 443 movq %gs:CPU_RTMP_R15, %r10 444 movq %r10, REGOFF_R15(%rsp) 445 movq $0, REGOFF_SAVFP(%rsp) 446 movq $0, REGOFF_SAVPC(%rsp) 447 448 /* 449 * Copy these registers here in case we end up stopped with 450 * someone (like, say, /proc) messing with our register state. 451 * We don't -restore- them unless we have to in update_sregs. 452 * 453 * Since userland -can't- change fsbase or gsbase directly, 454 * and capturing them involves two serializing instructions, 455 * we don't bother to capture them here. 456 */ 457 xorl %ebx, %ebx 458 movw %ds, %bx 459 movq %rbx, REGOFF_DS(%rsp) 460 movw %es, %bx 461 movq %rbx, REGOFF_ES(%rsp) 462 movw %fs, %bx 463 movq %rbx, REGOFF_FS(%rsp) 464 movw %gs, %bx 465 movq %rbx, REGOFF_GS(%rsp) 466 467 /* 468 * Machine state saved in the regs structure on the stack 469 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9 470 * %eax is the syscall number 471 * %rsp is the thread's stack, %r15 is curthread 472 * REG_RSP(%rsp) is the user's stack 473 */ 474 475 SYSCALL_TRAPTRACE($TT_SYSC64) 476 477 movq %rsp, %rbp 478 479 movq T_LWP(%r15), %r14 480 ASSERT_NO_RUPDATE_PENDING(%r14) 481 ENABLE_INTR_FLAGS 482 483 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 484 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 485 486 ASSERT_LWPTOREGS(%r14, %rsp) 487 488 movb $LWP_SYS, LWP_STATE(%r14) 489 incq LWP_RU_SYSC(%r14) 490 movb $NORMALRETURN, LWP_EOSYS(%r14) 491 492 incq %gs:CPU_STATS_SYS_SYSCALL 493 494 movw %ax, T_SYSNUM(%r15) 495 movzbl T_PRE_SYS(%r15), %ebx 496 ORL_SYSCALLTRACE(%ebx) 497 testl %ebx, %ebx 498 jne _syscall_pre 499 500_syscall_invoke: 501 movq REGOFF_RDI(%rbp), %rdi 502 movq REGOFF_RSI(%rbp), %rsi 503 movq REGOFF_RDX(%rbp), %rdx 504 movq REGOFF_RCX(%rbp), %rcx 505 movq REGOFF_R8(%rbp), %r8 506 movq REGOFF_R9(%rbp), %r9 507 508 cmpl $NSYSCALL, %eax 509 jae _syscall_ill 510 shll $SYSENT_SIZE_SHIFT, %eax 511 leaq sysent(%rax), %rbx 512 513 call *SY_CALLC(%rbx) 514 515 movq %rax, %r12 516 movq %rdx, %r13 517 518 /* 519 * If the handler returns two ints, then we need to split the 520 * 64-bit return value into two 32-bit values. 521 */ 522 testw $SE_32RVAL2, SY_FLAGS(%rbx) 523 je 5f 524 movq %r12, %r13 525 shrq $32, %r13 /* upper 32-bits into %edx */ 526 movl %r12d, %r12d /* lower 32-bits into %eax */ 5275: 528 /* 529 * Optimistically assume that there's no post-syscall 530 * work to do. (This is to avoid having to call syscall_mstate() 531 * with interrupts disabled) 532 */ 533 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 534 535 /* 536 * We must protect ourselves from being descheduled here; 537 * If we were, and we ended up on another cpu, or another 538 * lwp got in ahead of us, it could change the segment 539 * registers without us noticing before we return to userland. 540 */ 541 CLI(%r14) 542 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 543 jne _syscall_post 544 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 545 546 movq %r12, REGOFF_RAX(%rsp) 547 movq %r13, REGOFF_RDX(%rsp) 548 549 /* 550 * To get back to userland, we need the return %rip in %rcx and 551 * the return %rfl in %r11d. The sysretq instruction also arranges 552 * to fix up %cs and %ss; everything else is our responsibility. 553 */ 554 movq REGOFF_RDI(%rsp), %rdi 555 movq REGOFF_RSI(%rsp), %rsi 556 movq REGOFF_RDX(%rsp), %rdx 557 /* %rcx used to restore %rip value */ 558 559 movq REGOFF_R8(%rsp), %r8 560 movq REGOFF_R9(%rsp), %r9 561 movq REGOFF_RAX(%rsp), %rax 562 movq REGOFF_RBX(%rsp), %rbx 563 564 movq REGOFF_RBP(%rsp), %rbp 565 movq REGOFF_R10(%rsp), %r10 566 /* %r11 used to restore %rfl value */ 567 movq REGOFF_R12(%rsp), %r12 568 569 movq REGOFF_R13(%rsp), %r13 570 movq REGOFF_R14(%rsp), %r14 571 movq REGOFF_R15(%rsp), %r15 572 573 movq REGOFF_RIP(%rsp), %rcx 574 movl REGOFF_RFL(%rsp), %r11d 575 576#if defined(__xpv) 577 addq $REGOFF_RIP, %rsp 578#else 579 movq REGOFF_RSP(%rsp), %rsp 580#endif 581 582 /* 583 * There can be no instructions between the ALTENTRY below and 584 * SYSRET or we could end up breaking brand support. See label usage 585 * in sn1_brand_syscall_callback for an example. 586 */ 587 ASSERT_UPCALL_MASK_IS_SET 588 SWAPGS /* user gsbase */ 589 ALTENTRY(nopop_sys_syscall_sysretq) 590 SYSRETQ 591 /*NOTREACHED*/ 592 SET_SIZE(nopop_sys_syscall_sysretq) 593 594_syscall_pre: 595 call pre_syscall 596 movl %eax, %r12d 597 testl %eax, %eax 598 jne _syscall_post_call 599 /* 600 * Didn't abort, so reload the syscall args and invoke the handler. 601 */ 602 movzwl T_SYSNUM(%r15), %eax 603 jmp _syscall_invoke 604 605_syscall_ill: 606 call nosys 607 movq %rax, %r12 608 movq %rdx, %r13 609 jmp _syscall_post_call 610 611_syscall_post: 612 STI 613 /* 614 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 615 * so that we can account for the extra work it takes us to finish. 616 */ 617 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 618_syscall_post_call: 619 movq %r12, %rdi 620 movq %r13, %rsi 621 call post_syscall 622 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 623 jmp _sys_rtt 624 SET_SIZE(sys_syscall) 625 SET_SIZE(brand_sys_syscall) 626 627#endif /* __lint */ 628 629#if defined(__lint) 630 631/*ARGSUSED*/ 632void 633sys_syscall32() 634{} 635 636#else /* __lint */ 637 638 ENTRY_NP(brand_sys_syscall32) 639 SWAPGS /* kernel gsbase */ 640 XPV_TRAP_POP 641 BRAND_CALLBACK(BRAND_CB_SYSCALL32) 642 SWAPGS /* user gsbase */ 643 644#if defined(__xpv) 645 jmp nopop_sys_syscall32 646#endif 647 648 ALTENTRY(sys_syscall32) 649 SWAPGS /* kernel gsbase */ 650 651#if defined(__xpv) 652 XPV_TRAP_POP 653nopop_sys_syscall32: 654#endif 655 656 movl %esp, %r10d 657 movq %gs:CPU_THREAD, %r15 658 movq T_STACK(%r15), %rsp 659 movl %eax, %eax 660 661 movl $U32CS_SEL, REGOFF_CS(%rsp) 662 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 663 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 664 movq %r10, REGOFF_RSP(%rsp) 665 movl $UDS_SEL, REGOFF_SS(%rsp) 666 667_syscall32_save: 668 movl %edi, REGOFF_RDI(%rsp) 669 movl %esi, REGOFF_RSI(%rsp) 670 movl %ebp, REGOFF_RBP(%rsp) 671 movl %ebx, REGOFF_RBX(%rsp) 672 movl %edx, REGOFF_RDX(%rsp) 673 movl %ecx, REGOFF_RCX(%rsp) 674 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 675 movq $0, REGOFF_SAVFP(%rsp) 676 movq $0, REGOFF_SAVPC(%rsp) 677 678 /* 679 * Copy these registers here in case we end up stopped with 680 * someone (like, say, /proc) messing with our register state. 681 * We don't -restore- them unless we have to in update_sregs. 682 * 683 * Since userland -can't- change fsbase or gsbase directly, 684 * we don't bother to capture them here. 685 */ 686 xorl %ebx, %ebx 687 movw %ds, %bx 688 movq %rbx, REGOFF_DS(%rsp) 689 movw %es, %bx 690 movq %rbx, REGOFF_ES(%rsp) 691 movw %fs, %bx 692 movq %rbx, REGOFF_FS(%rsp) 693 movw %gs, %bx 694 movq %rbx, REGOFF_GS(%rsp) 695 696 /* 697 * Application state saved in the regs structure on the stack 698 * %eax is the syscall number 699 * %rsp is the thread's stack, %r15 is curthread 700 * REG_RSP(%rsp) is the user's stack 701 */ 702 703 SYSCALL_TRAPTRACE32($TT_SYSC) 704 705 movq %rsp, %rbp 706 707 movq T_LWP(%r15), %r14 708 ASSERT_NO_RUPDATE_PENDING(%r14) 709 710 ENABLE_INTR_FLAGS 711 712 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 713 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 714 715 ASSERT_LWPTOREGS(%r14, %rsp) 716 717 incq %gs:CPU_STATS_SYS_SYSCALL 718 719 /* 720 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed 721 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or 722 * more succinctly: 723 * 724 * SA(MAXSYSARGS * sizeof (long)) == 64 725 */ 726#define SYS_DROP 64 /* drop for args */ 727 subq $SYS_DROP, %rsp 728 movb $LWP_SYS, LWP_STATE(%r14) 729 movq %r15, %rdi 730 movq %rsp, %rsi 731 call syscall_entry 732 733 /* 734 * Fetch the arguments copied onto the kernel stack and put 735 * them in the right registers to invoke a C-style syscall handler. 736 * %rax contains the handler address. 737 * 738 * Ideas for making all this go faster of course include simply 739 * forcibly fetching 6 arguments from the user stack under lofault 740 * protection, reverting to copyin_args only when watchpoints 741 * are in effect. 742 * 743 * (If we do this, make sure that exec and libthread leave 744 * enough space at the top of the stack to ensure that we'll 745 * never do a fetch from an invalid page.) 746 * 747 * Lots of ideas here, but they won't really help with bringup B-) 748 * Correctness can't wait, performance can wait a little longer .. 749 */ 750 751 movq %rax, %rbx 752 movl 0(%rsp), %edi 753 movl 8(%rsp), %esi 754 movl 0x10(%rsp), %edx 755 movl 0x18(%rsp), %ecx 756 movl 0x20(%rsp), %r8d 757 movl 0x28(%rsp), %r9d 758 759 call *SY_CALLC(%rbx) 760 761 movq %rbp, %rsp /* pop the args */ 762 763 /* 764 * amd64 syscall handlers -always- return a 64-bit value in %rax. 765 * On the 32-bit kernel, they always return that value in %eax:%edx 766 * as required by the 32-bit ABI. 767 * 768 * Simulate the same behaviour by unconditionally splitting the 769 * return value in the same way. 770 */ 771 movq %rax, %r13 772 shrq $32, %r13 /* upper 32-bits into %edx */ 773 movl %eax, %r12d /* lower 32-bits into %eax */ 774 775 /* 776 * Optimistically assume that there's no post-syscall 777 * work to do. (This is to avoid having to call syscall_mstate() 778 * with interrupts disabled) 779 */ 780 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 781 782 /* 783 * We must protect ourselves from being descheduled here; 784 * If we were, and we ended up on another cpu, or another 785 * lwp got in ahead of us, it could change the segment 786 * registers without us noticing before we return to userland. 787 */ 788 CLI(%r14) 789 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 790 jne _full_syscall_postsys32 791 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 792 793 /* 794 * To get back to userland, we need to put the return %rip in %rcx and 795 * the return %rfl in %r11d. The sysret instruction also arranges 796 * to fix up %cs and %ss; everything else is our responsibility. 797 */ 798 799 movl %r12d, %eax /* %eax: rval1 */ 800 movl REGOFF_RBX(%rsp), %ebx 801 /* %ecx used for return pointer */ 802 movl %r13d, %edx /* %edx: rval2 */ 803 movl REGOFF_RBP(%rsp), %ebp 804 movl REGOFF_RSI(%rsp), %esi 805 movl REGOFF_RDI(%rsp), %edi 806 807 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */ 808 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */ 809 movl REGOFF_RSP(%rsp), %esp 810 811 ASSERT_UPCALL_MASK_IS_SET 812 SWAPGS /* user gsbase */ 813 ALTENTRY(nopop_sys_syscall32_sysretl) 814 SYSRETL 815 SET_SIZE(nopop_sys_syscall32_sysretl) 816 /*NOTREACHED*/ 817 818_full_syscall_postsys32: 819 STI 820 /* 821 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 822 * so that we can account for the extra work it takes us to finish. 823 */ 824 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 825 movq %r15, %rdi 826 movq %r12, %rsi /* rval1 - %eax */ 827 movq %r13, %rdx /* rval2 - %edx */ 828 call syscall_exit 829 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 830 jmp _sys_rtt 831 SET_SIZE(sys_syscall32) 832 SET_SIZE(brand_sys_syscall32) 833 834#endif /* __lint */ 835 836/* 837 * System call handler via the sysenter instruction 838 * Used only for 32-bit system calls on the 64-bit kernel. 839 * 840 * The caller in userland has arranged that: 841 * 842 * - %eax contains the syscall number 843 * - %ecx contains the user %esp 844 * - %edx contains the return %eip 845 * - the user stack contains the args to the syscall 846 * 847 * Hardware and (privileged) initialization code have arranged that by 848 * the time the sysenter instructions completes: 849 * 850 * - %rip is pointing to sys_sysenter (below). 851 * - %cs and %ss are set to kernel text and stack (data) selectors. 852 * - %rsp is pointing at the lwp's stack 853 * - interrupts have been disabled. 854 * 855 * Note that we are unable to return both "rvals" to userland with 856 * this call, as %edx is used by the sysexit instruction. 857 * 858 * One final complication in this routine is its interaction with 859 * single-stepping in a debugger. For most of the system call mechanisms, 860 * the CPU automatically clears the single-step flag before we enter the 861 * kernel. The sysenter mechanism does not clear the flag, so a user 862 * single-stepping through a libc routine may suddenly find him/herself 863 * single-stepping through the kernel. To detect this, kmdb compares the 864 * trap %pc to the [brand_]sys_enter addresses on each single-step trap. 865 * If it finds that we have single-stepped to a sysenter entry point, it 866 * explicitly clears the flag and executes the sys_sysenter routine. 867 * 868 * One final complication in this final complication is the fact that we 869 * have two different entry points for sysenter: brand_sys_sysenter and 870 * sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping 871 * through the kernel with kmdb, we will eventually hit the instruction at 872 * sys_sysenter. kmdb cannot distinguish between that valid single-step 873 * and the undesirable one mentioned above. To avoid this situation, we 874 * simply add a jump over the instruction at sys_sysenter to make it 875 * impossible to single-step to it. 876 */ 877#if defined(__lint) 878 879void 880sys_sysenter() 881{} 882 883#else /* __lint */ 884 885 ENTRY_NP(brand_sys_sysenter) 886 SWAPGS /* kernel gsbase */ 887 ALTENTRY(_brand_sys_sysenter_post_swapgs) 888 BRAND_CALLBACK(BRAND_CB_SYSENTER) 889 /* 890 * Jump over sys_sysenter to allow single-stepping as described 891 * above. 892 */ 893 jmp _sys_sysenter_post_swapgs 894 895 ALTENTRY(sys_sysenter) 896 SWAPGS /* kernel gsbase */ 897 898 ALTENTRY(_sys_sysenter_post_swapgs) 899 movq %gs:CPU_THREAD, %r15 900 901 movl $U32CS_SEL, REGOFF_CS(%rsp) 902 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */ 903 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */ 904 pushfq 905 popq %r10 906 movl $UDS_SEL, REGOFF_SS(%rsp) 907 908 /* 909 * Set the interrupt flag before storing the flags to the 910 * flags image on the stack so we can return to user with 911 * interrupts enabled if we return via sys_rtt_syscall32 912 */ 913 orq $PS_IE, %r10 914 movq %r10, REGOFF_RFL(%rsp) 915 916 movl %edi, REGOFF_RDI(%rsp) 917 movl %esi, REGOFF_RSI(%rsp) 918 movl %ebp, REGOFF_RBP(%rsp) 919 movl %ebx, REGOFF_RBX(%rsp) 920 movl %edx, REGOFF_RDX(%rsp) 921 movl %ecx, REGOFF_RCX(%rsp) 922 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 923 movq $0, REGOFF_SAVFP(%rsp) 924 movq $0, REGOFF_SAVPC(%rsp) 925 926 /* 927 * Copy these registers here in case we end up stopped with 928 * someone (like, say, /proc) messing with our register state. 929 * We don't -restore- them unless we have to in update_sregs. 930 * 931 * Since userland -can't- change fsbase or gsbase directly, 932 * we don't bother to capture them here. 933 */ 934 xorl %ebx, %ebx 935 movw %ds, %bx 936 movq %rbx, REGOFF_DS(%rsp) 937 movw %es, %bx 938 movq %rbx, REGOFF_ES(%rsp) 939 movw %fs, %bx 940 movq %rbx, REGOFF_FS(%rsp) 941 movw %gs, %bx 942 movq %rbx, REGOFF_GS(%rsp) 943 944 /* 945 * Application state saved in the regs structure on the stack 946 * %eax is the syscall number 947 * %rsp is the thread's stack, %r15 is curthread 948 * REG_RSP(%rsp) is the user's stack 949 */ 950 951 SYSCALL_TRAPTRACE($TT_SYSENTER) 952 953 movq %rsp, %rbp 954 955 movq T_LWP(%r15), %r14 956 ASSERT_NO_RUPDATE_PENDING(%r14) 957 958 ENABLE_INTR_FLAGS 959 960 /* 961 * Catch 64-bit process trying to issue sysenter instruction 962 * on Nocona based systems. 963 */ 964 movq LWP_PROCP(%r14), %rax 965 cmpq $DATAMODEL_ILP32, P_MODEL(%rax) 966 je 7f 967 968 /* 969 * For a non-32-bit process, simulate a #ud, since that's what 970 * native hardware does. The traptrace entry (above) will 971 * let you know what really happened. 972 */ 973 movq $T_ILLINST, REGOFF_TRAPNO(%rsp) 974 movq REGOFF_CS(%rsp), %rdi 975 movq %rdi, REGOFF_ERR(%rsp) 976 movq %rsp, %rdi 977 movq REGOFF_RIP(%rsp), %rsi 978 movl %gs:CPU_ID, %edx 979 call trap 980 jmp _sys_rtt 9817: 982 983 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 984 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */ 985 986 ASSERT_LWPTOREGS(%r14, %rsp) 987 988 incq %gs:CPU_STATS_SYS_SYSCALL 989 990 /* 991 * Make some space for MAXSYSARGS (currently 8) 32-bit args 992 * placed into 64-bit (long) arg slots, plus one 64-bit 993 * (long) arg count, maintaining 16 byte alignment. 994 */ 995 subq $SYS_DROP, %rsp 996 movb $LWP_SYS, LWP_STATE(%r14) 997 movq %r15, %rdi 998 movq %rsp, %rsi 999 call syscall_entry 1000 1001 /* 1002 * Fetch the arguments copied onto the kernel stack and put 1003 * them in the right registers to invoke a C-style syscall handler. 1004 * %rax contains the handler address. 1005 */ 1006 movq %rax, %rbx 1007 movl 0(%rsp), %edi 1008 movl 8(%rsp), %esi 1009 movl 0x10(%rsp), %edx 1010 movl 0x18(%rsp), %ecx 1011 movl 0x20(%rsp), %r8d 1012 movl 0x28(%rsp), %r9d 1013 1014 call *SY_CALLC(%rbx) 1015 1016 movq %rbp, %rsp /* pop the args */ 1017 1018 /* 1019 * amd64 syscall handlers -always- return a 64-bit value in %rax. 1020 * On the 32-bit kernel, the always return that value in %eax:%edx 1021 * as required by the 32-bit ABI. 1022 * 1023 * Simulate the same behaviour by unconditionally splitting the 1024 * return value in the same way. 1025 */ 1026 movq %rax, %r13 1027 shrq $32, %r13 /* upper 32-bits into %edx */ 1028 movl %eax, %r12d /* lower 32-bits into %eax */ 1029 1030 /* 1031 * Optimistically assume that there's no post-syscall 1032 * work to do. (This is to avoid having to call syscall_mstate() 1033 * with interrupts disabled) 1034 */ 1035 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 1036 1037 /* 1038 * We must protect ourselves from being descheduled here; 1039 * If we were, and we ended up on another cpu, or another 1040 * lwp got int ahead of us, it could change the segment 1041 * registers without us noticing before we return to userland. 1042 */ 1043 cli 1044 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 1045 jne _full_syscall_postsys32 1046 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 1047 1048 /* 1049 * To get back to userland, load up the 32-bit registers and 1050 * sysexit back where we came from. 1051 */ 1052 1053 /* 1054 * Interrupts will be turned on by the 'sti' executed just before 1055 * sysexit. The following ensures that restoring the user's rflags 1056 * doesn't enable interrupts too soon. 1057 */ 1058 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp) 1059 1060 /* 1061 * (There's no point in loading up %edx because the sysexit 1062 * mechanism smashes it.) 1063 */ 1064 movl %r12d, %eax 1065 movl REGOFF_RBX(%rsp), %ebx 1066 movl REGOFF_RBP(%rsp), %ebp 1067 movl REGOFF_RSI(%rsp), %esi 1068 movl REGOFF_RDI(%rsp), %edi 1069 1070 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */ 1071 pushq REGOFF_RFL(%rsp) 1072 popfq 1073 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */ 1074 swapgs 1075 sti 1076 sysexit 1077 SET_SIZE(sys_sysenter) 1078 SET_SIZE(_sys_sysenter_post_swapgs) 1079 SET_SIZE(brand_sys_sysenter) 1080 1081#endif /* __lint */ 1082 1083#if defined(__lint) 1084/* 1085 * System call via an int80. This entry point is only used by the Linux 1086 * application environment. Unlike the other entry points, there is no 1087 * default action to take if no callback is registered for this process. 1088 */ 1089void 1090sys_int80() 1091{} 1092 1093#else /* __lint */ 1094 1095 ENTRY_NP(brand_sys_int80) 1096 SWAPGS /* kernel gsbase */ 1097 XPV_TRAP_POP 1098 BRAND_CALLBACK(BRAND_CB_INT80) 1099 SWAPGS /* user gsbase */ 1100#if defined(__xpv) 1101 jmp nopop_int80 1102#endif 1103 1104 ENTRY_NP(sys_int80) 1105 /* 1106 * We hit an int80, but this process isn't of a brand with an int80 1107 * handler. Bad process! Make it look as if the INT failed. 1108 * Modify %rip to point before the INT, push the expected error 1109 * code and fake a GP fault. Note on 64-bit hypervisor we need 1110 * to undo the XPV_TRAP_POP and push rcx and r11 back on the stack 1111 * because gptrap will pop them again with its own XPV_TRAP_POP. 1112 */ 1113#if defined(__xpv) 1114 XPV_TRAP_POP 1115nopop_int80: 1116#endif 1117 subq $2, (%rsp) /* int insn 2-bytes */ 1118 pushq $_CONST(_MUL(T_INT80, GATE_DESC_SIZE) + 2) 1119#if defined(__xpv) 1120 push %r11 1121 push %rcx 1122#endif 1123 jmp gptrap / GP fault 1124 SET_SIZE(sys_int80) 1125 SET_SIZE(brand_sys_int80) 1126#endif /* __lint */ 1127 1128 1129/* 1130 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by 1131 * the generic i386 libc to do system calls. We do a small amount of setup 1132 * before jumping into the existing sys_syscall32 path. 1133 */ 1134#if defined(__lint) 1135 1136/*ARGSUSED*/ 1137void 1138sys_syscall_int() 1139{} 1140 1141#else /* __lint */ 1142 1143 ENTRY_NP(brand_sys_syscall_int) 1144 SWAPGS /* kernel gsbase */ 1145 XPV_TRAP_POP 1146 BRAND_CALLBACK(BRAND_CB_INT91) 1147 SWAPGS /* user gsbase */ 1148 1149#if defined(__xpv) 1150 jmp nopop_syscall_int 1151#endif 1152 1153 ALTENTRY(sys_syscall_int) 1154 SWAPGS /* kernel gsbase */ 1155 1156#if defined(__xpv) 1157 XPV_TRAP_POP 1158nopop_syscall_int: 1159#endif 1160 1161 movq %gs:CPU_THREAD, %r15 1162 movq T_STACK(%r15), %rsp 1163 movl %eax, %eax 1164 /* 1165 * Set t_post_sys on this thread to force ourselves out via the slow 1166 * path. It might be possible at some later date to optimize this out 1167 * and use a faster return mechanism. 1168 */ 1169 movb $1, T_POST_SYS(%r15) 1170 CLEAN_CS 1171 jmp _syscall32_save 1172 SET_SIZE(sys_syscall_int) 1173 SET_SIZE(brand_sys_syscall_int) 1174 1175#endif /* __lint */ 1176 1177/* 1178 * Legacy 32-bit applications and old libc implementations do lcalls; 1179 * we should never get here because the LDT entry containing the syscall 1180 * segment descriptor has the "segment present" bit cleared, which means 1181 * we end up processing those system calls in trap() via a not-present trap. 1182 * 1183 * We do it this way because a call gate unhelpfully does -nothing- to the 1184 * interrupt flag bit, so an interrupt can run us just after the lcall 1185 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and 1186 * INTR_POP paths would have to be slightly more complex to dance around 1187 * this problem, and end up depending explicitly on the first 1188 * instruction of this handler being either swapgs or cli. 1189 */ 1190 1191#if defined(__lint) 1192 1193/*ARGSUSED*/ 1194void 1195sys_lcall32() 1196{} 1197 1198#else /* __lint */ 1199 1200 ENTRY_NP(sys_lcall32) 1201 SWAPGS /* kernel gsbase */ 1202 pushq $0 1203 pushq %rbp 1204 movq %rsp, %rbp 1205 leaq __lcall_panic_str(%rip), %rdi 1206 xorl %eax, %eax 1207 call panic 1208 SET_SIZE(sys_lcall32) 1209 1210__lcall_panic_str: 1211 .string "sys_lcall32: shouldn't be here!" 1212 1213/* 1214 * Declare a uintptr_t which covers the entire pc range of syscall 1215 * handlers for the stack walkers that need this. 1216 */ 1217 .align CPTRSIZE 1218 .globl _allsyscalls_size 1219 .type _allsyscalls_size, @object 1220_allsyscalls_size: 1221 .NWORD . - _allsyscalls 1222 SET_SIZE(_allsyscalls_size) 1223 1224#endif /* __lint */ 1225 1226/* 1227 * These are the thread context handlers for lwps using sysenter/sysexit. 1228 */ 1229 1230#if defined(__lint) 1231 1232/*ARGSUSED*/ 1233void 1234sep_save(void *ksp) 1235{} 1236 1237/*ARGSUSED*/ 1238void 1239sep_restore(void *ksp) 1240{} 1241 1242#else /* __lint */ 1243 1244 /* 1245 * setting this value to zero as we switch away causes the 1246 * stack-pointer-on-sysenter to be NULL, ensuring that we 1247 * don't silently corrupt another (preempted) thread stack 1248 * when running an lwp that (somehow) didn't get sep_restore'd 1249 */ 1250 ENTRY_NP(sep_save) 1251 xorl %edx, %edx 1252 xorl %eax, %eax 1253 movl $MSR_INTC_SEP_ESP, %ecx 1254 wrmsr 1255 ret 1256 SET_SIZE(sep_save) 1257 1258 /* 1259 * Update the kernel stack pointer as we resume onto this cpu. 1260 */ 1261 ENTRY_NP(sep_restore) 1262 movq %rdi, %rdx 1263 shrq $32, %rdx 1264 movl %edi, %eax 1265 movl $MSR_INTC_SEP_ESP, %ecx 1266 wrmsr 1267 ret 1268 SET_SIZE(sep_restore) 1269 1270#endif /* __lint */ 1271