1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
  24  */
  25 
  26 #include <sys/asm_linkage.h>
  27 #include <sys/asm_misc.h>
  28 #include <sys/regset.h>
  29 #include <sys/privregs.h>
  30 #include <sys/psw.h>
  31 #include <sys/machbrand.h>
  32 
  33 #if defined(__lint)
  34 
  35 #include <sys/types.h>
  36 #include <sys/thread.h>
  37 #include <sys/systm.h>
  38 
  39 #else   /* __lint */
  40 
  41 #include <sys/segments.h>
  42 #include <sys/pcb.h>
  43 #include <sys/trap.h>
  44 #include <sys/ftrace.h>
  45 #include <sys/traptrace.h>
  46 #include <sys/clock.h>
  47 #include <sys/model.h>
  48 #include <sys/panic.h>
  49 
  50 #if defined(__xpv)
  51 #include <sys/hypervisor.h>
  52 #endif
  53 
  54 #include "assym.h"
  55 
  56 #endif  /* __lint */
  57 
  58 /*
  59  * We implement five flavours of system call entry points
  60  *
  61  * -    syscall/sysretq         (amd64 generic)
  62  * -    syscall/sysretl         (i386 plus SYSC bit)
  63  * -    sysenter/sysexit        (i386 plus SEP bit)
  64  * -    int/iret                (i386 generic)
  65  * -    lcall/iret              (i386 generic)
  66  *
  67  * The current libc included in Solaris uses int/iret as the base unoptimized
  68  * kernel entry method. Older libc implementations and legacy binaries may use
  69  * the lcall call gate, so it must continue to be supported.
  70  *
  71  * System calls that use an lcall call gate are processed in trap() via a
  72  * segment-not-present trap, i.e. lcalls are extremely slow(!).
  73  *
  74  * The basic pattern used in the 32-bit SYSC handler at this point in time is
  75  * to have the bare minimum of assembler, and get to the C handlers as
  76  * quickly as possible.
  77  *
  78  * The 64-bit handler is much closer to the sparcv9 handler; that's
  79  * because of passing arguments in registers.  The 32-bit world still
  80  * passes arguments on the stack -- that makes that handler substantially
  81  * more complex.
  82  *
  83  * The two handlers share a few code fragments which are broken
  84  * out into preprocessor macros below.
  85  *
  86  * XX64 come back and speed all this up later.  The 32-bit stuff looks
  87  * especially easy to speed up the argument copying part ..
  88  *
  89  *
  90  * Notes about segment register usage (c.f. the 32-bit kernel)
  91  *
  92  * In the 32-bit kernel, segment registers are dutifully saved and
  93  * restored on all mode transitions because the kernel uses them directly.
  94  * When the processor is running in 64-bit mode, segment registers are
  95  * largely ignored.
  96  *
  97  * %cs and %ss
  98  *      controlled by the hardware mechanisms that make mode transitions
  99  *
 100  * The remaining segment registers have to either be pointing at a valid
 101  * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
 102  *
 103  * %ds and %es
 104  *      always ignored
 105  *
 106  * %fs and %gs
 107  *      fsbase and gsbase are used to control the place they really point at.
 108  *      The kernel only depends on %gs, and controls its own gsbase via swapgs
 109  *
 110  * Note that loading segment registers is still costly because the GDT
 111  * lookup still happens (this is because the hardware can't know that we're
 112  * not setting up these segment registers for a 32-bit program).  Thus we
 113  * avoid doing this in the syscall path, and defer them to lwp context switch
 114  * handlers, so the register values remain virtualized to the lwp.
 115  */
 116 
 117 #if defined(SYSCALLTRACE)
 118 #define ORL_SYSCALLTRACE(r32)           \
 119         orl     syscalltrace(%rip), r32
 120 #else
 121 #define ORL_SYSCALLTRACE(r32)
 122 #endif
 123 
 124 /*
 125  * In the 32-bit kernel, we do absolutely nothing before getting into the
 126  * brand callback checks.  In 64-bit land, we do swapgs and then come here.
 127  * We assume that the %rsp- and %r15-stashing fields in the CPU structure
 128  * are still unused.
 129  *
 130  * Check if a brand_mach_ops callback is defined for the specified callback_id
 131  * type.  If so invoke it with the kernel's %gs value loaded and the following
 132  * data on the stack:
 133  *
 134  * stack:  --------------------------------------
 135  *      32 | callback pointer                   |
 136  *    | 24 | user (or interrupt) stack pointer  |
 137  *    | 16 | lwp pointer                        |
 138  *    v  8 | userland return address            |
 139  *       0 | callback wrapper return addr       |
 140  *         --------------------------------------
 141  *
 142  * Since we're pushing the userland return address onto the kernel stack
 143  * we need to get that address without accessing the user's stack (since we
 144  * can't trust that data).  There are different ways to get the userland
 145  * return address depending on how the syscall trap was made:
 146  *
 147  * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
 148  * b) For sys_sysenter the return address is in %rdx.
 149  * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
 150  *    the stack pointer points at the state saved when we took the interrupt:
 151  *       ------------------------
 152  *    |  | user's %ss           |
 153  *    |  | user's %esp          |
 154  *    |  | EFLAGS register      |
 155  *    v  | user's %cs           |
 156  *       | user's %eip          |
 157  *       ------------------------
 158  *
 159  * The 2nd parameter to the BRAND_CALLBACK macro is either the
 160  * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro.  These macros are
 161  * used to generate the proper code to get the userland return address for
 162  * each syscall entry point.
 163  *
 164  * The interface to the brand callbacks on the 64-bit kernel assumes %r15
 165  * is available as a scratch register within the callback.  If the callback
 166  * returns within the kernel then this macro will restore %r15.  If the
 167  * callback is going to return directly to userland then it should restore
 168  * %r15 before returning to userland.
 169  */
 170 #define BRAND_URET_FROM_REG(rip_reg)                                    \
 171         pushq   rip_reg                 /* push the return address      */
 172 
 173 /*
 174  * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
 175  * is currently pointing at the user return address (%eip).
 176  */
 177 #define BRAND_URET_FROM_INTR_STACK()                                    \
 178         movq    %gs:CPU_RTMP_RSP, %r15  /* grab the intr. stack pointer */ ;\
 179         pushq   (%r15)                  /* push the return address      */
 180 
 181 #define BRAND_CALLBACK(callback_id, push_userland_ret)                      \
 182         movq    %rsp, %gs:CPU_RTMP_RSP  /* save the stack pointer       */ ;\
 183         movq    %r15, %gs:CPU_RTMP_R15  /* save %r15                    */ ;\
 184         movq    %gs:CPU_THREAD, %r15    /* load the thread pointer      */ ;\
 185         movq    T_STACK(%r15), %rsp     /* switch to the kernel stack   */ ;\
 186         subq    $16, %rsp               /* save space for 2 pointers    */ ;\
 187         pushq   %r14                    /* save %r14                    */ ;\
 188         movq    %gs:CPU_RTMP_RSP, %r14                                     ;\
 189         movq    %r14, 8(%rsp)           /* stash the user stack pointer */ ;\
 190         popq    %r14                    /* restore %r14                 */ ;\
 191         movq    T_LWP(%r15), %r15       /* load the lwp pointer         */ ;\
 192         pushq   %r15                    /* push the lwp pointer         */ ;\
 193         movq    LWP_PROCP(%r15), %r15   /* load the proc pointer        */ ;\
 194         movq    P_BRAND(%r15), %r15     /* load the brand pointer       */ ;\
 195         movq    B_MACHOPS(%r15), %r15   /* load the machops pointer     */ ;\
 196         movq    _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15            ;\
 197         cmpq    $0, %r15                                                   ;\
 198         je      1f                                                         ;\
 199         movq    %r15, 16(%rsp)          /* save the callback pointer    */ ;\
 200         push_userland_ret               /* push the return address      */ ;\
 201         call    *24(%rsp)               /* call callback                */ ;\
 202 1:      movq    %gs:CPU_RTMP_R15, %r15  /* restore %r15                 */ ;\
 203         movq    %gs:CPU_RTMP_RSP, %rsp  /* restore the stack pointer    */
 204 
 205 #define MSTATE_TRANSITION(from, to)             \
 206         movl    $from, %edi;                    \
 207         movl    $to, %esi;                      \
 208         call    syscall_mstate
 209 
 210 /*
 211  * Check to see if a simple (direct) return is possible i.e.
 212  *
 213  *      if (t->t_post_sys_ast | syscalltrace |
 214  *          lwp->lwp_pcb.pcb_rupdate == 1)
 215  *              do full version ;
 216  *
 217  * Preconditions:
 218  * -    t is curthread
 219  * Postconditions:
 220  * -    condition code NE is set if post-sys is too complex
 221  * -    rtmp is zeroed if it isn't (we rely on this!)
 222  * -    ltmp is smashed
 223  */
 224 #define CHECK_POSTSYS_NE(t, ltmp, rtmp)                 \
 225         movq    T_LWP(t), ltmp;                         \
 226         movzbl  PCB_RUPDATE(ltmp), rtmp;                \
 227         ORL_SYSCALLTRACE(rtmp);                         \
 228         orl     T_POST_SYS_AST(t), rtmp;                \
 229         cmpl    $0, rtmp
 230         
 231 /*
 232  * Fix up the lwp, thread, and eflags for a successful return
 233  *
 234  * Preconditions:
 235  * -    zwreg contains zero
 236  */
 237 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg)           \
 238         movb    $LWP_USER, LWP_STATE(lwp);              \
 239         movw    zwreg, T_SYSNUM(t);                     \
 240         andb    $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
 241 
 242 /*
 243  * ASSERT(lwptoregs(lwp) == rp);
 244  *
 245  * This may seem obvious, but very odd things happen if this
 246  * assertion is false
 247  *
 248  * Preconditions:
 249  *      (%rsp is ready for normal call sequence)
 250  * Postconditions (if assertion is true):
 251  *      %r11 is smashed
 252  *
 253  * ASSERT(rp->r_cs == descnum)
 254  *
 255  * The code selector is written into the regs structure when the
 256  * lwp stack is created.  We use this ASSERT to validate that
 257  * the regs structure really matches how we came in.
 258  *
 259  * Preconditions:
 260  *      (%rsp is ready for normal call sequence)
 261  * Postconditions (if assertion is true):
 262  *      -none-
 263  *
 264  * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
 265  *
 266  * If this is false, it meant that we returned to userland without
 267  * updating the segment registers as we were supposed to.
 268  *
 269  * Note that we must ensure no interrupts or other traps intervene
 270  * between entering privileged mode and performing the assertion,
 271  * otherwise we may perform a context switch on the thread, which
 272  * will end up setting pcb_rupdate to 1 again.
 273  */
 274 #if defined(DEBUG)
 275 
 276 #if !defined(__lint)
 277 
 278 __lwptoregs_msg:
 279         .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
 280 
 281 __codesel_msg:
 282         .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
 283 
 284 __no_rupdate_msg:
 285         .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
 286 
 287 #endif  /* !__lint */
 288 
 289 #define ASSERT_LWPTOREGS(lwp, rp)                       \
 290         movq    LWP_REGS(lwp), %r11;                    \
 291         cmpq    rp, %r11;                               \
 292         je      7f;                                     \
 293         leaq    __lwptoregs_msg(%rip), %rdi;            \
 294         movl    $__LINE__, %esi;                        \
 295         movq    lwp, %rdx;                              \
 296         movq    %r11, %rcx;                             \
 297         movq    rp, %r8;                                \
 298         xorl    %eax, %eax;                             \
 299         call    panic;                                  \
 300 7:
 301 
 302 #define ASSERT_NO_RUPDATE_PENDING(lwp)                  \
 303         testb   $0x1, PCB_RUPDATE(lwp);                 \
 304         je      8f;                                     \
 305         movq    lwp, %rdx;                              \
 306         leaq    __no_rupdate_msg(%rip), %rdi;           \
 307         movl    $__LINE__, %esi;                        \
 308         xorl    %eax, %eax;                             \
 309         call    panic;                                  \
 310 8:
 311 
 312 #else
 313 #define ASSERT_LWPTOREGS(lwp, rp)
 314 #define ASSERT_NO_RUPDATE_PENDING(lwp)
 315 #endif
 316 
 317 /*
 318  * Do the traptrace thing and restore any registers we used
 319  * in situ.  Assumes that %rsp is pointing at the base of
 320  * the struct regs, obviously ..
 321  */     
 322 #ifdef TRAPTRACE        
 323 #define SYSCALL_TRAPTRACE(ttype)                                \
 324         TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype);               \
 325         TRACE_REGS(%rdi, %rsp, %rbx, %rcx);                     \
 326         TRACE_STAMP(%rdi);      /* rdtsc clobbers %eax, %edx */ \
 327         movq    REGOFF_RAX(%rsp), %rax;                         \
 328         movq    REGOFF_RBX(%rsp), %rbx;                         \
 329         movq    REGOFF_RCX(%rsp), %rcx;                         \
 330         movq    REGOFF_RDX(%rsp), %rdx;                         \
 331         movl    %eax, TTR_SYSNUM(%rdi);                         \
 332         movq    REGOFF_RDI(%rsp), %rdi
 333 
 334 #define SYSCALL_TRAPTRACE32(ttype)                              \
 335         SYSCALL_TRAPTRACE(ttype);                               \
 336         /* paranoia: clean the top 32-bits of the registers */  \
 337         orl     %eax, %eax;                                     \
 338         orl     %ebx, %ebx;                                     \
 339         orl     %ecx, %ecx;                                     \
 340         orl     %edx, %edx;                                     \
 341         orl     %edi, %edi      
 342 #else   /* TRAPTRACE */
 343 #define SYSCALL_TRAPTRACE(ttype)
 344 #define SYSCALL_TRAPTRACE32(ttype)      
 345 #endif  /* TRAPTRACE */
 346 
 347 /*
 348  * The 64-bit libc syscall wrapper does this:
 349  *
 350  * fn(<args>)
 351  * {
 352  *      movq    %rcx, %r10      -- because syscall smashes %rcx
 353  *      movl    $CODE, %eax
 354  *      syscall
 355  *      <error processing>
 356  * }
 357  *
 358  * Thus when we come into the kernel:
 359  *
 360  *      %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
 361  *      %rax is the syscall number
 362  *      %r12-%r15 contain caller state
 363  *
 364  * The syscall instruction arranges that:
 365  *      
 366  *      %rcx contains the return %rip
 367  *      %r11d contains bottom 32-bits of %rflags
 368  *      %rflags is masked (as determined by the SFMASK msr)
 369  *      %cs is set to UCS_SEL (as determined by the STAR msr)
 370  *      %ss is set to UDS_SEL (as determined by the STAR msr)
 371  *      %rip is set to sys_syscall (as determined by the LSTAR msr)
 372  *
 373  * Or in other words, we have no registers available at all.
 374  * Only swapgs can save us!
 375  *
 376  * Under the hypervisor, the swapgs has happened already.  However, the
 377  * state of the world is very different from that we're familiar with.
 378  *
 379  * In particular, we have a stack structure like that for interrupt
 380  * gates, except that the %cs and %ss registers are modified for reasons
 381  * that are not entirely clear.  Critically, the %rcx/%r11 values do
 382  * *not* reflect the usage of those registers under a 'real' syscall[1];
 383  * the stack, therefore, looks like this:
 384  *
 385  *      0x0(rsp)        potentially junk %rcx
 386  *      0x8(rsp)        potentially junk %r11
 387  *      0x10(rsp)       user %rip
 388  *      0x18(rsp)       modified %cs
 389  *      0x20(rsp)       user %rflags
 390  *      0x28(rsp)       user %rsp
 391  *      0x30(rsp)       modified %ss
 392  *
 393  *
 394  * and before continuing on, we must load the %rip into %rcx and the
 395  * %rflags into %r11.
 396  *
 397  * [1] They used to, and we relied on it, but this was broken in 3.1.1.
 398  * Sigh.
 399  */
 400 #if defined(__xpv)
 401 #define XPV_SYSCALL_PROD                                                \
 402         movq    0x10(%rsp), %rcx;                                       \
 403         movq    0x20(%rsp), %r11;                                       \
 404         movq    0x28(%rsp), %rsp
 405 #else
 406 #define XPV_SYSCALL_PROD /* nothing */
 407 #endif
 408 
 409 #if defined(__lint)
 410 
 411 /*ARGSUSED*/
 412 void
 413 sys_syscall()
 414 {}
 415 
 416 void
 417 _allsyscalls()
 418 {}
 419 
 420 size_t _allsyscalls_size;
 421 
 422 #else   /* __lint */
 423 
 424         ENTRY_NP2(brand_sys_syscall,_allsyscalls)
 425         SWAPGS                          /* kernel gsbase */
 426         XPV_SYSCALL_PROD
 427         BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
 428         jmp     noprod_sys_syscall
 429 
 430         ALTENTRY(sys_syscall)
 431         SWAPGS                          /* kernel gsbase */
 432         XPV_SYSCALL_PROD
 433 
 434 noprod_sys_syscall:
 435         movq    %r15, %gs:CPU_RTMP_R15
 436         movq    %rsp, %gs:CPU_RTMP_RSP
 437 
 438         movq    %gs:CPU_THREAD, %r15
 439         movq    T_STACK(%r15), %rsp     /* switch from user to kernel stack */
 440 
 441         ASSERT_UPCALL_MASK_IS_SET
 442 
 443         movl    $UCS_SEL, REGOFF_CS(%rsp)
 444         movq    %rcx, REGOFF_RIP(%rsp)          /* syscall: %rip -> %rcx */
 445         movq    %r11, REGOFF_RFL(%rsp)          /* syscall: %rfl -> %r11d */
 446         movl    $UDS_SEL, REGOFF_SS(%rsp)
 447 
 448         movl    %eax, %eax                      /* wrapper: sysc# -> %eax */
 449         movq    %rdi, REGOFF_RDI(%rsp)
 450         movq    %rsi, REGOFF_RSI(%rsp)
 451         movq    %rdx, REGOFF_RDX(%rsp)
 452         movq    %r10, REGOFF_RCX(%rsp)          /* wrapper: %rcx -> %r10 */
 453         movq    %r10, %rcx                      /* arg[3] for direct calls */
 454 
 455         movq    %r8, REGOFF_R8(%rsp)
 456         movq    %r9, REGOFF_R9(%rsp)
 457         movq    %rax, REGOFF_RAX(%rsp)
 458         movq    %rbx, REGOFF_RBX(%rsp)
 459 
 460         movq    %rbp, REGOFF_RBP(%rsp)
 461         movq    %r10, REGOFF_R10(%rsp)
 462         movq    %gs:CPU_RTMP_RSP, %r11
 463         movq    %r11, REGOFF_RSP(%rsp)
 464         movq    %r12, REGOFF_R12(%rsp)
 465 
 466         movq    %r13, REGOFF_R13(%rsp)
 467         movq    %r14, REGOFF_R14(%rsp)
 468         movq    %gs:CPU_RTMP_R15, %r10
 469         movq    %r10, REGOFF_R15(%rsp)
 470         movq    $0, REGOFF_SAVFP(%rsp)
 471         movq    $0, REGOFF_SAVPC(%rsp)
 472 
 473         /*
 474          * Copy these registers here in case we end up stopped with
 475          * someone (like, say, /proc) messing with our register state.
 476          * We don't -restore- them unless we have to in update_sregs.
 477          *
 478          * Since userland -can't- change fsbase or gsbase directly,
 479          * and capturing them involves two serializing instructions,
 480          * we don't bother to capture them here.
 481          */
 482         xorl    %ebx, %ebx
 483         movw    %ds, %bx
 484         movq    %rbx, REGOFF_DS(%rsp)
 485         movw    %es, %bx
 486         movq    %rbx, REGOFF_ES(%rsp)
 487         movw    %fs, %bx
 488         movq    %rbx, REGOFF_FS(%rsp)
 489         movw    %gs, %bx
 490         movq    %rbx, REGOFF_GS(%rsp)
 491 
 492         /*
 493          * Machine state saved in the regs structure on the stack
 494          * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
 495          * %eax is the syscall number
 496          * %rsp is the thread's stack, %r15 is curthread
 497          * REG_RSP(%rsp) is the user's stack
 498          */
 499 
 500         SYSCALL_TRAPTRACE($TT_SYSC64)
 501 
 502         movq    %rsp, %rbp
 503         
 504         movq    T_LWP(%r15), %r14
 505         ASSERT_NO_RUPDATE_PENDING(%r14)
 506         ENABLE_INTR_FLAGS
 507 
 508         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 509         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate call) */
 510 
 511         ASSERT_LWPTOREGS(%r14, %rsp)
 512 
 513         movb    $LWP_SYS, LWP_STATE(%r14)
 514         incq    LWP_RU_SYSC(%r14)
 515         movb    $NORMALRETURN, LWP_EOSYS(%r14)
 516 
 517         incq    %gs:CPU_STATS_SYS_SYSCALL
 518 
 519         movw    %ax, T_SYSNUM(%r15)
 520         movzbl  T_PRE_SYS(%r15), %ebx
 521         ORL_SYSCALLTRACE(%ebx)
 522         testl   %ebx, %ebx
 523         jne     _syscall_pre
 524 
 525 _syscall_invoke:
 526         movq    REGOFF_RDI(%rbp), %rdi
 527         movq    REGOFF_RSI(%rbp), %rsi
 528         movq    REGOFF_RDX(%rbp), %rdx
 529         movq    REGOFF_RCX(%rbp), %rcx
 530         movq    REGOFF_R8(%rbp), %r8
 531         movq    REGOFF_R9(%rbp), %r9
 532 
 533         cmpl    $NSYSCALL, %eax
 534         jae     _syscall_ill    
 535         shll    $SYSENT_SIZE_SHIFT, %eax
 536         leaq    sysent(%rax), %rbx
 537 
 538         call    *SY_CALLC(%rbx)
 539 
 540         movq    %rax, %r12
 541         movq    %rdx, %r13
 542 
 543         /*
 544          * If the handler returns two ints, then we need to split the
 545          * 64-bit return value into two 32-bit values.
 546          */
 547         testw   $SE_32RVAL2, SY_FLAGS(%rbx)
 548         je      5f
 549         movq    %r12, %r13
 550         shrq    $32, %r13       /* upper 32-bits into %edx */
 551         movl    %r12d, %r12d    /* lower 32-bits into %eax */
 552 5:
 553         /*
 554          * Optimistically assume that there's no post-syscall
 555          * work to do.  (This is to avoid having to call syscall_mstate()
 556          * with interrupts disabled)
 557          */
 558         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 559 
 560         /*
 561          * We must protect ourselves from being descheduled here;
 562          * If we were, and we ended up on another cpu, or another
 563          * lwp got in ahead of us, it could change the segment
 564          * registers without us noticing before we return to userland.
 565          */
 566         CLI(%r14)
 567         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
 568         jne     _syscall_post
 569 
 570         /*
 571          * We need to protect ourselves against non-canonical return values
 572          * because Intel doesn't check for them on sysret (AMD does).  Canonical
 573          * addresses on current amd64 processors only use 48-bits for VAs; an
 574          * address is canonical if all upper bits (47-63) are identical. If we
 575          * find a non-canonical %rip, we opt to go through the full
 576          * _syscall_post path which takes us into an iretq which is not
 577          * susceptible to the same problems sysret is.
 578          * 
 579          * We're checking for a canonical address by first doing an arithmetic
 580          * shift. This will fill in the remaining bits with the value of bit 63.
 581          * If the address were canonical, the register would now have either all
 582          * zeroes or all ones in it. Therefore we add one (inducing overflow)
 583          * and compare against 1. A canonical address will either be zero or one
 584          * at this point, hence the use of ja.
 585          *
 586          * At this point, r12 and r13 have the return value so we can't use
 587          * those registers.
 588          */
 589         movq    REGOFF_RIP(%rsp), %rcx
 590         sarq    $47, %rcx
 591         incq    %rcx
 592         cmpq    $1, %rcx
 593         ja      _syscall_post
 594 
 595 
 596         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 597 
 598         movq    %r12, REGOFF_RAX(%rsp)
 599         movq    %r13, REGOFF_RDX(%rsp)
 600 
 601         /*
 602          * To get back to userland, we need the return %rip in %rcx and
 603          * the return %rfl in %r11d.  The sysretq instruction also arranges
 604          * to fix up %cs and %ss; everything else is our responsibility.
 605          */
 606         movq    REGOFF_RDI(%rsp), %rdi
 607         movq    REGOFF_RSI(%rsp), %rsi
 608         movq    REGOFF_RDX(%rsp), %rdx
 609         /* %rcx used to restore %rip value */
 610 
 611         movq    REGOFF_R8(%rsp), %r8
 612         movq    REGOFF_R9(%rsp), %r9
 613         movq    REGOFF_RAX(%rsp), %rax
 614         movq    REGOFF_RBX(%rsp), %rbx
 615 
 616         movq    REGOFF_RBP(%rsp), %rbp  
 617         movq    REGOFF_R10(%rsp), %r10
 618         /* %r11 used to restore %rfl value */
 619         movq    REGOFF_R12(%rsp), %r12
 620 
 621         movq    REGOFF_R13(%rsp), %r13
 622         movq    REGOFF_R14(%rsp), %r14
 623         movq    REGOFF_R15(%rsp), %r15
 624 
 625         movq    REGOFF_RIP(%rsp), %rcx  
 626         movl    REGOFF_RFL(%rsp), %r11d
 627 
 628 #if defined(__xpv)
 629         addq    $REGOFF_RIP, %rsp
 630 #else
 631         movq    REGOFF_RSP(%rsp), %rsp
 632 #endif
 633 
 634         /*
 635          * There can be no instructions between the ALTENTRY below and
 636          * SYSRET or we could end up breaking brand support. See label usage
 637          * in sn1_brand_syscall_callback for an example.
 638          */
 639         ASSERT_UPCALL_MASK_IS_SET
 640 #if defined(__xpv)
 641         SYSRETQ
 642         ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
 643 
 644         /*
 645          * We can only get here after executing a brand syscall
 646          * interposition callback handler and simply need to
 647          * "sysretq" back to userland. On the hypervisor this
 648          * involves the iret hypercall which requires us to construct
 649          * just enough of the stack needed for the hypercall.
 650          * (rip, cs, rflags, rsp, ss).
 651          */
 652         movq    %rsp, %gs:CPU_RTMP_RSP          /* save user's rsp */
 653         movq    %gs:CPU_THREAD, %r11
 654         movq    T_STACK(%r11), %rsp
 655 
 656         movq    %rcx, REGOFF_RIP(%rsp)
 657         movl    $UCS_SEL, REGOFF_CS(%rsp)
 658         movq    %gs:CPU_RTMP_RSP, %r11
 659         movq    %r11, REGOFF_RSP(%rsp)
 660         pushfq
 661         popq    %r11                            /* hypercall enables ints */
 662         movq    %r11, REGOFF_RFL(%rsp)
 663         movl    $UDS_SEL, REGOFF_SS(%rsp)
 664         addq    $REGOFF_RIP, %rsp
 665         /*
 666          * XXPV: see comment in SYSRETQ definition for future optimization
 667          *       we could take.
 668          */
 669         ASSERT_UPCALL_MASK_IS_SET
 670         SYSRETQ
 671 #else
 672         ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
 673         SWAPGS                          /* user gsbase */
 674         SYSRETQ
 675 #endif
 676         /*NOTREACHED*/
 677         SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
 678 
 679 _syscall_pre:
 680         call    pre_syscall
 681         movl    %eax, %r12d
 682         testl   %eax, %eax
 683         jne     _syscall_post_call
 684         /*
 685          * Didn't abort, so reload the syscall args and invoke the handler.
 686          */
 687         movzwl  T_SYSNUM(%r15), %eax    
 688         jmp     _syscall_invoke
 689 
 690 _syscall_ill:
 691         call    nosys
 692         movq    %rax, %r12
 693         movq    %rdx, %r13
 694         jmp     _syscall_post_call
 695 
 696 _syscall_post:
 697         STI
 698         /*
 699          * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
 700          * so that we can account for the extra work it takes us to finish.
 701          */
 702         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 703 _syscall_post_call:
 704         movq    %r12, %rdi
 705         movq    %r13, %rsi
 706         call    post_syscall
 707         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 708         jmp     _sys_rtt
 709         SET_SIZE(sys_syscall)
 710         SET_SIZE(brand_sys_syscall)
 711 
 712 #endif  /* __lint */
 713 
 714 #if defined(__lint)
 715 
 716 /*ARGSUSED*/
 717 void
 718 sys_syscall32()
 719 {}
 720 
 721 #else   /* __lint */
 722 
 723         ENTRY_NP(brand_sys_syscall32)
 724         SWAPGS                          /* kernel gsbase */
 725         XPV_TRAP_POP
 726         BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
 727         jmp     nopop_sys_syscall32
 728 
 729         ALTENTRY(sys_syscall32)
 730         SWAPGS                          /* kernel gsbase */
 731         XPV_TRAP_POP
 732 
 733 nopop_sys_syscall32:
 734         movl    %esp, %r10d
 735         movq    %gs:CPU_THREAD, %r15
 736         movq    T_STACK(%r15), %rsp
 737         movl    %eax, %eax
 738 
 739         movl    $U32CS_SEL, REGOFF_CS(%rsp)
 740         movl    %ecx, REGOFF_RIP(%rsp)          /* syscall: %rip -> %rcx */
 741         movq    %r11, REGOFF_RFL(%rsp)          /* syscall: %rfl -> %r11d */
 742         movq    %r10, REGOFF_RSP(%rsp)
 743         movl    $UDS_SEL, REGOFF_SS(%rsp)
 744 
 745 _syscall32_save:
 746         movl    %edi, REGOFF_RDI(%rsp)
 747         movl    %esi, REGOFF_RSI(%rsp)
 748         movl    %ebp, REGOFF_RBP(%rsp)
 749         movl    %ebx, REGOFF_RBX(%rsp)
 750         movl    %edx, REGOFF_RDX(%rsp)
 751         movl    %ecx, REGOFF_RCX(%rsp)
 752         movl    %eax, REGOFF_RAX(%rsp)          /* wrapper: sysc# -> %eax */
 753         movq    $0, REGOFF_SAVFP(%rsp)
 754         movq    $0, REGOFF_SAVPC(%rsp)
 755 
 756         /*
 757          * Copy these registers here in case we end up stopped with
 758          * someone (like, say, /proc) messing with our register state.
 759          * We don't -restore- them unless we have to in update_sregs.
 760          *
 761          * Since userland -can't- change fsbase or gsbase directly,
 762          * we don't bother to capture them here.
 763          */
 764         xorl    %ebx, %ebx
 765         movw    %ds, %bx
 766         movq    %rbx, REGOFF_DS(%rsp)
 767         movw    %es, %bx
 768         movq    %rbx, REGOFF_ES(%rsp)
 769         movw    %fs, %bx
 770         movq    %rbx, REGOFF_FS(%rsp)
 771         movw    %gs, %bx
 772         movq    %rbx, REGOFF_GS(%rsp)
 773 
 774         /*
 775          * Application state saved in the regs structure on the stack
 776          * %eax is the syscall number
 777          * %rsp is the thread's stack, %r15 is curthread
 778          * REG_RSP(%rsp) is the user's stack
 779          */
 780 
 781         SYSCALL_TRAPTRACE32($TT_SYSC)
 782 
 783         movq    %rsp, %rbp
 784 
 785         movq    T_LWP(%r15), %r14
 786         ASSERT_NO_RUPDATE_PENDING(%r14)
 787 
 788         ENABLE_INTR_FLAGS
 789 
 790         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 791         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate call) */
 792 
 793         ASSERT_LWPTOREGS(%r14, %rsp)
 794 
 795         incq     %gs:CPU_STATS_SYS_SYSCALL
 796 
 797         /*
 798          * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
 799          * into 64-bit (long) arg slots, maintaining 16 byte alignment.  Or
 800          * more succinctly:
 801          *
 802          *      SA(MAXSYSARGS * sizeof (long)) == 64
 803          */
 804 #define SYS_DROP        64                      /* drop for args */
 805         subq    $SYS_DROP, %rsp
 806         movb    $LWP_SYS, LWP_STATE(%r14)
 807         movq    %r15, %rdi
 808         movq    %rsp, %rsi
 809         call    syscall_entry
 810 
 811         /*
 812          * Fetch the arguments copied onto the kernel stack and put
 813          * them in the right registers to invoke a C-style syscall handler.
 814          * %rax contains the handler address.
 815          *
 816          * Ideas for making all this go faster of course include simply
 817          * forcibly fetching 6 arguments from the user stack under lofault
 818          * protection, reverting to copyin_args only when watchpoints
 819          * are in effect.
 820          *
 821          * (If we do this, make sure that exec and libthread leave
 822          * enough space at the top of the stack to ensure that we'll
 823          * never do a fetch from an invalid page.)
 824          *
 825          * Lots of ideas here, but they won't really help with bringup B-)
 826          * Correctness can't wait, performance can wait a little longer ..
 827          */
 828 
 829         movq    %rax, %rbx
 830         movl    0(%rsp), %edi
 831         movl    8(%rsp), %esi
 832         movl    0x10(%rsp), %edx
 833         movl    0x18(%rsp), %ecx
 834         movl    0x20(%rsp), %r8d
 835         movl    0x28(%rsp), %r9d
 836 
 837         call    *SY_CALLC(%rbx)
 838 
 839         movq    %rbp, %rsp      /* pop the args */
 840 
 841         /*
 842          * amd64 syscall handlers -always- return a 64-bit value in %rax.
 843          * On the 32-bit kernel, they always return that value in %eax:%edx
 844          * as required by the 32-bit ABI.
 845          *
 846          * Simulate the same behaviour by unconditionally splitting the
 847          * return value in the same way.
 848          */
 849         movq    %rax, %r13
 850         shrq    $32, %r13       /* upper 32-bits into %edx */
 851         movl    %eax, %r12d     /* lower 32-bits into %eax */
 852 
 853         /*
 854          * Optimistically assume that there's no post-syscall
 855          * work to do.  (This is to avoid having to call syscall_mstate()
 856          * with interrupts disabled)
 857          */
 858         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 859 
 860         /*
 861          * We must protect ourselves from being descheduled here;
 862          * If we were, and we ended up on another cpu, or another
 863          * lwp got in ahead of us, it could change the segment
 864          * registers without us noticing before we return to userland.
 865          */
 866         CLI(%r14)
 867         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
 868         jne     _full_syscall_postsys32
 869         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 870 
 871         /*
 872          * To get back to userland, we need to put the return %rip in %rcx and
 873          * the return %rfl in %r11d.  The sysret instruction also arranges
 874          * to fix up %cs and %ss; everything else is our responsibility.
 875          */
 876 
 877         movl    %r12d, %eax                     /* %eax: rval1 */
 878         movl    REGOFF_RBX(%rsp), %ebx
 879         /* %ecx used for return pointer */
 880         movl    %r13d, %edx                     /* %edx: rval2 */
 881         movl    REGOFF_RBP(%rsp), %ebp
 882         movl    REGOFF_RSI(%rsp), %esi
 883         movl    REGOFF_RDI(%rsp), %edi
 884 
 885         movl    REGOFF_RFL(%rsp), %r11d         /* %r11 -> eflags */
 886         movl    REGOFF_RIP(%rsp), %ecx          /* %ecx -> %eip */
 887         movl    REGOFF_RSP(%rsp), %esp
 888 
 889         ASSERT_UPCALL_MASK_IS_SET
 890         ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
 891         SWAPGS                          /* user gsbase */
 892         SYSRETL
 893         SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
 894         /*NOTREACHED*/
 895 
 896 _full_syscall_postsys32:
 897         STI
 898         /*
 899          * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
 900          * so that we can account for the extra work it takes us to finish.
 901          */
 902         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 903         movq    %r15, %rdi
 904         movq    %r12, %rsi                      /* rval1 - %eax */
 905         movq    %r13, %rdx                      /* rval2 - %edx */
 906         call    syscall_exit
 907         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 908         jmp     _sys_rtt
 909         SET_SIZE(sys_syscall32)
 910         SET_SIZE(brand_sys_syscall32)
 911 
 912 #endif  /* __lint */
 913 
 914 /*
 915  * System call handler via the sysenter instruction
 916  * Used only for 32-bit system calls on the 64-bit kernel.
 917  *
 918  * The caller in userland has arranged that:
 919  *
 920  * -    %eax contains the syscall number
 921  * -    %ecx contains the user %esp
 922  * -    %edx contains the return %eip
 923  * -    the user stack contains the args to the syscall
 924  *
 925  * Hardware and (privileged) initialization code have arranged that by
 926  * the time the sysenter instructions completes:
 927  *
 928  * - %rip is pointing to sys_sysenter (below).
 929  * - %cs and %ss are set to kernel text and stack (data) selectors.
 930  * - %rsp is pointing at the lwp's stack
 931  * - interrupts have been disabled.
 932  *
 933  * Note that we are unable to return both "rvals" to userland with
 934  * this call, as %edx is used by the sysexit instruction.
 935  *
 936  * One final complication in this routine is its interaction with
 937  * single-stepping in a debugger.  For most of the system call mechanisms,
 938  * the CPU automatically clears the single-step flag before we enter the
 939  * kernel.  The sysenter mechanism does not clear the flag, so a user
 940  * single-stepping through a libc routine may suddenly find him/herself
 941  * single-stepping through the kernel.  To detect this, kmdb compares the
 942  * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
 943  * If it finds that we have single-stepped to a sysenter entry point, it
 944  * explicitly clears the flag and executes the sys_sysenter routine.
 945  *
 946  * One final complication in this final complication is the fact that we
 947  * have two different entry points for sysenter: brand_sys_sysenter and
 948  * sys_sysenter.  If we enter at brand_sys_sysenter and start single-stepping
 949  * through the kernel with kmdb, we will eventually hit the instruction at
 950  * sys_sysenter.  kmdb cannot distinguish between that valid single-step
 951  * and the undesirable one mentioned above.  To avoid this situation, we
 952  * simply add a jump over the instruction at sys_sysenter to make it
 953  * impossible to single-step to it.
 954  */
 955 #if defined(__lint)
 956 
 957 void
 958 sys_sysenter()
 959 {}
 960 
 961 #else   /* __lint */
 962 
 963         ENTRY_NP(brand_sys_sysenter)
 964         SWAPGS                          /* kernel gsbase */
 965         ALTENTRY(_brand_sys_sysenter_post_swapgs)
 966         BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
 967         /*
 968          * Jump over sys_sysenter to allow single-stepping as described
 969          * above.
 970          */
 971         jmp     _sys_sysenter_post_swapgs
 972 
 973         ALTENTRY(sys_sysenter)
 974         SWAPGS                          /* kernel gsbase */
 975 
 976         ALTENTRY(_sys_sysenter_post_swapgs)
 977         movq    %gs:CPU_THREAD, %r15
 978 
 979         movl    $U32CS_SEL, REGOFF_CS(%rsp)
 980         movl    %ecx, REGOFF_RSP(%rsp)          /* wrapper: %esp -> %ecx */
 981         movl    %edx, REGOFF_RIP(%rsp)          /* wrapper: %eip -> %edx */
 982         pushfq
 983         popq    %r10
 984         movl    $UDS_SEL, REGOFF_SS(%rsp)
 985 
 986         /*
 987          * Set the interrupt flag before storing the flags to the
 988          * flags image on the stack so we can return to user with
 989          * interrupts enabled if we return via sys_rtt_syscall32
 990          */
 991         orq     $PS_IE, %r10
 992         movq    %r10, REGOFF_RFL(%rsp)
 993 
 994         movl    %edi, REGOFF_RDI(%rsp)
 995         movl    %esi, REGOFF_RSI(%rsp)
 996         movl    %ebp, REGOFF_RBP(%rsp)
 997         movl    %ebx, REGOFF_RBX(%rsp)
 998         movl    %edx, REGOFF_RDX(%rsp)
 999         movl    %ecx, REGOFF_RCX(%rsp)
1000         movl    %eax, REGOFF_RAX(%rsp)          /* wrapper: sysc# -> %eax */
1001         movq    $0, REGOFF_SAVFP(%rsp)
1002         movq    $0, REGOFF_SAVPC(%rsp)
1003 
1004         /*
1005          * Copy these registers here in case we end up stopped with
1006          * someone (like, say, /proc) messing with our register state.
1007          * We don't -restore- them unless we have to in update_sregs.
1008          *
1009          * Since userland -can't- change fsbase or gsbase directly,
1010          * we don't bother to capture them here.
1011          */
1012         xorl    %ebx, %ebx
1013         movw    %ds, %bx
1014         movq    %rbx, REGOFF_DS(%rsp)
1015         movw    %es, %bx
1016         movq    %rbx, REGOFF_ES(%rsp)
1017         movw    %fs, %bx
1018         movq    %rbx, REGOFF_FS(%rsp)
1019         movw    %gs, %bx
1020         movq    %rbx, REGOFF_GS(%rsp)
1021 
1022         /*
1023          * Application state saved in the regs structure on the stack
1024          * %eax is the syscall number
1025          * %rsp is the thread's stack, %r15 is curthread
1026          * REG_RSP(%rsp) is the user's stack
1027          */
1028 
1029         SYSCALL_TRAPTRACE($TT_SYSENTER)
1030 
1031         movq    %rsp, %rbp
1032 
1033         movq    T_LWP(%r15), %r14
1034         ASSERT_NO_RUPDATE_PENDING(%r14)
1035 
1036         ENABLE_INTR_FLAGS
1037 
1038         /*
1039          * Catch 64-bit process trying to issue sysenter instruction
1040          * on Nocona based systems.
1041          */
1042         movq    LWP_PROCP(%r14), %rax
1043         cmpq    $DATAMODEL_ILP32, P_MODEL(%rax)
1044         je      7f
1045 
1046         /*
1047          * For a non-32-bit process, simulate a #ud, since that's what
1048          * native hardware does.  The traptrace entry (above) will
1049          * let you know what really happened.
1050          */
1051         movq    $T_ILLINST, REGOFF_TRAPNO(%rsp)
1052         movq    REGOFF_CS(%rsp), %rdi
1053         movq    %rdi, REGOFF_ERR(%rsp)
1054         movq    %rsp, %rdi
1055         movq    REGOFF_RIP(%rsp), %rsi
1056         movl    %gs:CPU_ID, %edx
1057         call    trap
1058         jmp     _sys_rtt
1059 7:
1060 
1061         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1062         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate calls) */
1063 
1064         ASSERT_LWPTOREGS(%r14, %rsp)
1065 
1066         incq    %gs:CPU_STATS_SYS_SYSCALL
1067 
1068         /*
1069          * Make some space for MAXSYSARGS (currently 8) 32-bit args
1070          * placed into 64-bit (long) arg slots, plus one 64-bit
1071          * (long) arg count, maintaining 16 byte alignment.
1072          */
1073         subq    $SYS_DROP, %rsp
1074         movb    $LWP_SYS, LWP_STATE(%r14)
1075         movq    %r15, %rdi
1076         movq    %rsp, %rsi
1077         call    syscall_entry
1078 
1079         /*
1080          * Fetch the arguments copied onto the kernel stack and put
1081          * them in the right registers to invoke a C-style syscall handler.
1082          * %rax contains the handler address.
1083          */
1084         movq    %rax, %rbx
1085         movl    0(%rsp), %edi
1086         movl    8(%rsp), %esi
1087         movl    0x10(%rsp), %edx
1088         movl    0x18(%rsp), %ecx
1089         movl    0x20(%rsp), %r8d
1090         movl    0x28(%rsp), %r9d
1091 
1092         call    *SY_CALLC(%rbx)
1093 
1094         movq    %rbp, %rsp      /* pop the args */
1095 
1096         /*
1097          * amd64 syscall handlers -always- return a 64-bit value in %rax.
1098          * On the 32-bit kernel, the always return that value in %eax:%edx
1099          * as required by the 32-bit ABI.
1100          *
1101          * Simulate the same behaviour by unconditionally splitting the
1102          * return value in the same way.
1103          */
1104         movq    %rax, %r13
1105         shrq    $32, %r13       /* upper 32-bits into %edx */
1106         movl    %eax, %r12d     /* lower 32-bits into %eax */
1107 
1108         /*
1109          * Optimistically assume that there's no post-syscall
1110          * work to do.  (This is to avoid having to call syscall_mstate()
1111          * with interrupts disabled)
1112          */
1113         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1114 
1115         /*
1116          * We must protect ourselves from being descheduled here;
1117          * If we were, and we ended up on another cpu, or another
1118          * lwp got int ahead of us, it could change the segment
1119          * registers without us noticing before we return to userland.
1120          */
1121         cli
1122         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1123         jne     _full_syscall_postsys32
1124         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1125 
1126         /*
1127          * To get back to userland, load up the 32-bit registers and
1128          * sysexit back where we came from.
1129          */
1130 
1131         /*
1132          * Interrupts will be turned on by the 'sti' executed just before
1133          * sysexit.  The following ensures that restoring the user's rflags
1134          * doesn't enable interrupts too soon.
1135          */
1136         andq    $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1137 
1138         /*
1139          * (There's no point in loading up %edx because the sysexit
1140          * mechanism smashes it.)
1141          */
1142         movl    %r12d, %eax
1143         movl    REGOFF_RBX(%rsp), %ebx
1144         movl    REGOFF_RBP(%rsp), %ebp
1145         movl    REGOFF_RSI(%rsp), %esi
1146         movl    REGOFF_RDI(%rsp), %edi
1147 
1148         movl    REGOFF_RIP(%rsp), %edx  /* sysexit: %edx -> %eip */
1149         pushq   REGOFF_RFL(%rsp)
1150         popfq
1151         movl    REGOFF_RSP(%rsp), %ecx  /* sysexit: %ecx -> %esp */
1152         ALTENTRY(sys_sysenter_swapgs_sysexit)
1153         swapgs
1154         sti
1155         sysexit
1156         SET_SIZE(sys_sysenter_swapgs_sysexit)
1157         SET_SIZE(sys_sysenter)
1158         SET_SIZE(_sys_sysenter_post_swapgs)
1159         SET_SIZE(brand_sys_sysenter)
1160 
1161 #endif  /* __lint */
1162 
1163 /*
1164  * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1165  * the generic i386 libc to do system calls. We do a small amount of setup
1166  * before jumping into the existing sys_syscall32 path.
1167  */
1168 #if defined(__lint)
1169 
1170 /*ARGSUSED*/
1171 void
1172 sys_syscall_int()
1173 {}
1174 
1175 #else   /* __lint */
1176 
1177         ENTRY_NP(brand_sys_syscall_int)
1178         SWAPGS                          /* kernel gsbase */
1179         XPV_TRAP_POP
1180         BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1181         jmp     nopop_syscall_int
1182 
1183         ALTENTRY(sys_syscall_int)
1184         SWAPGS                          /* kernel gsbase */
1185         XPV_TRAP_POP
1186 
1187 nopop_syscall_int:
1188         movq    %gs:CPU_THREAD, %r15
1189         movq    T_STACK(%r15), %rsp
1190         movl    %eax, %eax
1191         /*
1192          * Set t_post_sys on this thread to force ourselves out via the slow
1193          * path. It might be possible at some later date to optimize this out
1194          * and use a faster return mechanism.
1195          */
1196         movb    $1, T_POST_SYS(%r15)
1197         CLEAN_CS
1198         jmp     _syscall32_save
1199         /*
1200          * There should be no instructions between this label and SWAPGS/IRET
1201          * or we could end up breaking branded zone support. See the usage of
1202          * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1203          * for examples.
1204          */
1205         ALTENTRY(sys_sysint_swapgs_iret)
1206         SWAPGS                          /* user gsbase */
1207         IRET
1208         /*NOTREACHED*/
1209         SET_SIZE(sys_sysint_swapgs_iret)
1210         SET_SIZE(sys_syscall_int)
1211         SET_SIZE(brand_sys_syscall_int)
1212 
1213 #endif  /* __lint */
1214         
1215 /*
1216  * Legacy 32-bit applications and old libc implementations do lcalls;
1217  * we should never get here because the LDT entry containing the syscall
1218  * segment descriptor has the "segment present" bit cleared, which means
1219  * we end up processing those system calls in trap() via a not-present trap.
1220  *
1221  * We do it this way because a call gate unhelpfully does -nothing- to the
1222  * interrupt flag bit, so an interrupt can run us just after the lcall
1223  * completes, but just before the swapgs takes effect.   Thus the INTR_PUSH and
1224  * INTR_POP paths would have to be slightly more complex to dance around
1225  * this problem, and end up depending explicitly on the first
1226  * instruction of this handler being either swapgs or cli.
1227  */
1228 
1229 #if defined(__lint)
1230 
1231 /*ARGSUSED*/
1232 void
1233 sys_lcall32()
1234 {}
1235 
1236 #else   /* __lint */
1237 
1238         ENTRY_NP(sys_lcall32)
1239         SWAPGS                          /* kernel gsbase */
1240         pushq   $0
1241         pushq   %rbp
1242         movq    %rsp, %rbp
1243         leaq    __lcall_panic_str(%rip), %rdi
1244         xorl    %eax, %eax
1245         call    panic
1246         SET_SIZE(sys_lcall32)
1247 
1248 __lcall_panic_str:      
1249         .string "sys_lcall32: shouldn't be here!"
1250 
1251 /*
1252  * Declare a uintptr_t which covers the entire pc range of syscall
1253  * handlers for the stack walkers that need this.
1254  */
1255         .align  CPTRSIZE
1256         .globl  _allsyscalls_size
1257         .type   _allsyscalls_size, @object
1258 _allsyscalls_size:
1259         .NWORD  . - _allsyscalls
1260         SET_SIZE(_allsyscalls_size)
1261 
1262 #endif  /* __lint */
1263 
1264 /*
1265  * These are the thread context handlers for lwps using sysenter/sysexit.
1266  */
1267 
1268 #if defined(__lint)
1269 
1270 /*ARGSUSED*/
1271 void
1272 sep_save(void *ksp)
1273 {}
1274 
1275 /*ARGSUSED*/
1276 void
1277 sep_restore(void *ksp)
1278 {}
1279 
1280 #else   /* __lint */
1281 
1282         /*
1283          * setting this value to zero as we switch away causes the
1284          * stack-pointer-on-sysenter to be NULL, ensuring that we
1285          * don't silently corrupt another (preempted) thread stack
1286          * when running an lwp that (somehow) didn't get sep_restore'd
1287          */
1288         ENTRY_NP(sep_save)
1289         xorl    %edx, %edx
1290         xorl    %eax, %eax
1291         movl    $MSR_INTC_SEP_ESP, %ecx
1292         wrmsr
1293         ret
1294         SET_SIZE(sep_save)
1295 
1296         /*
1297          * Update the kernel stack pointer as we resume onto this cpu.
1298          */
1299         ENTRY_NP(sep_restore)
1300         movq    %rdi, %rdx
1301         shrq    $32, %rdx
1302         movl    %edi, %eax
1303         movl    $MSR_INTC_SEP_ESP, %ecx
1304         wrmsr
1305         ret
1306         SET_SIZE(sep_restore)
1307 
1308 #endif  /* __lint */