1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2019 Joyent, Inc.
  24  * Copyright (c) 2016 by Delphix. All rights reserved.
  25  */
  26 
  27 #include <sys/asm_linkage.h>
  28 #include <sys/asm_misc.h>
  29 #include <sys/regset.h>
  30 #include <sys/privregs.h>
  31 #include <sys/psw.h>
  32 #include <sys/machbrand.h>
  33 
  34 #if defined(__lint)
  35 
  36 #include <sys/types.h>
  37 #include <sys/thread.h>
  38 #include <sys/systm.h>
  39 
  40 #else   /* __lint */
  41 
  42 #include <sys/segments.h>
  43 #include <sys/pcb.h>
  44 #include <sys/trap.h>
  45 #include <sys/ftrace.h>
  46 #include <sys/traptrace.h>
  47 #include <sys/clock.h>
  48 #include <sys/model.h>
  49 #include <sys/panic.h>
  50 
  51 #if defined(__xpv)
  52 #include <sys/hypervisor.h>
  53 #endif
  54 
  55 #include "assym.h"
  56 
  57 #endif  /* __lint */
  58 
  59 /*
  60  * We implement five flavours of system call entry points
  61  *
  62  * -    syscall/sysretq         (amd64 generic)
  63  * -    syscall/sysretl         (i386 plus SYSC bit)
  64  * -    sysenter/sysexit        (i386 plus SEP bit)
  65  * -    int/iret                (i386 generic)
  66  * -    lcall/iret              (i386 generic)
  67  *
  68  * The current libc included in Solaris uses int/iret as the base unoptimized
  69  * kernel entry method. Older libc implementations and legacy binaries may use
  70  * the lcall call gate, so it must continue to be supported.
  71  *
  72  * System calls that use an lcall call gate are processed in trap() via a
  73  * segment-not-present trap, i.e. lcalls are extremely slow(!).
  74  *
  75  * The basic pattern used in the 32-bit SYSC handler at this point in time is
  76  * to have the bare minimum of assembler, and get to the C handlers as
  77  * quickly as possible.
  78  *
  79  * The 64-bit handler is much closer to the sparcv9 handler; that's
  80  * because of passing arguments in registers.  The 32-bit world still
  81  * passes arguments on the stack -- that makes that handler substantially
  82  * more complex.
  83  *
  84  * The two handlers share a few code fragments which are broken
  85  * out into preprocessor macros below.
  86  *
  87  * XX64 come back and speed all this up later.  The 32-bit stuff looks
  88  * especially easy to speed up the argument copying part ..
  89  *
  90  *
  91  * Notes about segment register usage (c.f. the 32-bit kernel)
  92  *
  93  * In the 32-bit kernel, segment registers are dutifully saved and
  94  * restored on all mode transitions because the kernel uses them directly.
  95  * When the processor is running in 64-bit mode, segment registers are
  96  * largely ignored.
  97  *
  98  * %cs and %ss
  99  *      controlled by the hardware mechanisms that make mode transitions
 100  *
 101  * The remaining segment registers have to either be pointing at a valid
 102  * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
 103  *
 104  * %ds and %es
 105  *      always ignored
 106  *
 107  * %fs and %gs
 108  *      fsbase and gsbase are used to control the place they really point at.
 109  *      The kernel only depends on %gs, and controls its own gsbase via swapgs
 110  *
 111  * Note that loading segment registers is still costly because the GDT
 112  * lookup still happens (this is because the hardware can't know that we're
 113  * not setting up these segment registers for a 32-bit program).  Thus we
 114  * avoid doing this in the syscall path, and defer them to lwp context switch
 115  * handlers, so the register values remain virtualized to the lwp.
 116  */
 117 
 118 #if defined(SYSCALLTRACE)
 119 #define ORL_SYSCALLTRACE(r32)           \
 120         orl     syscalltrace(%rip), r32
 121 #else
 122 #define ORL_SYSCALLTRACE(r32)
 123 #endif
 124 
 125 /*
 126  * In the 32-bit kernel, we do absolutely nothing before getting into the
 127  * brand callback checks.  In 64-bit land, we do swapgs and then come here.
 128  * We assume that the %rsp- and %r15-stashing fields in the CPU structure
 129  * are still unused.
 130  *
 131  * Check if a brand_mach_ops callback is defined for the specified callback_id
 132  * type.  If so invoke it with the kernel's %gs value loaded and the following
 133  * data on the stack:
 134  *
 135  * stack:  --------------------------------------
 136  *      32 | callback pointer                   |
 137  *    | 24 | user (or interrupt) stack pointer  |
 138  *    | 16 | lwp pointer                        |
 139  *    v  8 | userland return address            |
 140  *       0 | callback wrapper return addr       |
 141  *         --------------------------------------
 142  *
 143  * Since we're pushing the userland return address onto the kernel stack
 144  * we need to get that address without accessing the user's stack (since we
 145  * can't trust that data).  There are different ways to get the userland
 146  * return address depending on how the syscall trap was made:
 147  *
 148  * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
 149  * b) For sys_sysenter the return address is in %rdx.
 150  * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
 151  *    the stack pointer points at the state saved when we took the interrupt:
 152  *       ------------------------
 153  *    |  | user's %ss           |
 154  *    |  | user's %esp          |
 155  *    |  | EFLAGS register      |
 156  *    v  | user's %cs           |
 157  *       | user's %eip          |
 158  *       ------------------------
 159  *
 160  * The 2nd parameter to the BRAND_CALLBACK macro is either the
 161  * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro.  These macros are
 162  * used to generate the proper code to get the userland return address for
 163  * each syscall entry point.
 164  *
 165  * The interface to the brand callbacks on the 64-bit kernel assumes %r15
 166  * is available as a scratch register within the callback.  If the callback
 167  * returns within the kernel then this macro will restore %r15.  If the
 168  * callback is going to return directly to userland then it should restore
 169  * %r15 before returning to userland.
 170  */
 171 #define BRAND_URET_FROM_REG(rip_reg)                                    \
 172         pushq   rip_reg                 /* push the return address      */
 173 
 174 /*
 175  * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
 176  * is currently pointing at the user return address (%eip).
 177  */
 178 #define BRAND_URET_FROM_INTR_STACK()                                    \
 179         movq    %gs:CPU_RTMP_RSP, %r15  /* grab the intr. stack pointer */ ;\
 180         pushq   (%r15)                  /* push the return address      */
 181 
 182 #define BRAND_CALLBACK(callback_id, push_userland_ret)                      \
 183         movq    %rsp, %gs:CPU_RTMP_RSP  /* save the stack pointer       */ ;\
 184         movq    %r15, %gs:CPU_RTMP_R15  /* save %r15                    */ ;\
 185         movq    %gs:CPU_THREAD, %r15    /* load the thread pointer      */ ;\
 186         movq    T_STACK(%r15), %rsp     /* switch to the kernel stack   */ ;\
 187         subq    $16, %rsp               /* save space for 2 pointers    */ ;\
 188         pushq   %r14                    /* save %r14                    */ ;\
 189         movq    %gs:CPU_RTMP_RSP, %r14                                     ;\
 190         movq    %r14, 8(%rsp)           /* stash the user stack pointer */ ;\
 191         popq    %r14                    /* restore %r14                 */ ;\
 192         movq    T_LWP(%r15), %r15       /* load the lwp pointer         */ ;\
 193         pushq   %r15                    /* push the lwp pointer         */ ;\
 194         movq    LWP_PROCP(%r15), %r15   /* load the proc pointer        */ ;\
 195         movq    P_BRAND(%r15), %r15     /* load the brand pointer       */ ;\
 196         movq    B_MACHOPS(%r15), %r15   /* load the machops pointer     */ ;\
 197         movq    _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15            ;\
 198         cmpq    $0, %r15                                                   ;\
 199         je      1f                                                         ;\
 200         movq    %r15, 16(%rsp)          /* save the callback pointer    */ ;\
 201         push_userland_ret               /* push the return address      */ ;\
 202         movq    24(%rsp), %r15          /* load callback pointer        */ ;\
 203         INDIRECT_CALL_REG(r15)          /* call callback                */ ;\
 204 1:      movq    %gs:CPU_RTMP_R15, %r15  /* restore %r15                 */ ;\
 205         movq    %gs:CPU_RTMP_RSP, %rsp  /* restore the stack pointer    */
 206 
 207 #define MSTATE_TRANSITION(from, to)             \
 208         movl    $from, %edi;                    \
 209         movl    $to, %esi;                      \
 210         call    syscall_mstate
 211 
 212 /*
 213  * Check to see if a simple (direct) return is possible i.e.
 214  *
 215  *      if (t->t_post_sys_ast | syscalltrace |
 216  *          lwp->lwp_pcb.pcb_rupdate == 1)
 217  *              do full version ;
 218  *
 219  * Preconditions:
 220  * -    t is curthread
 221  * Postconditions:
 222  * -    condition code NE is set if post-sys is too complex
 223  * -    rtmp is zeroed if it isn't (we rely on this!)
 224  * -    ltmp is smashed
 225  */
 226 #define CHECK_POSTSYS_NE(t, ltmp, rtmp)                 \
 227         movq    T_LWP(t), ltmp;                         \
 228         movzbl  PCB_RUPDATE(ltmp), rtmp;                \
 229         ORL_SYSCALLTRACE(rtmp);                         \
 230         orl     T_POST_SYS_AST(t), rtmp;                \
 231         cmpl    $0, rtmp
 232 
 233 /*
 234  * Fix up the lwp, thread, and eflags for a successful return
 235  *
 236  * Preconditions:
 237  * -    zwreg contains zero
 238  */
 239 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg)           \
 240         movb    $LWP_USER, LWP_STATE(lwp);              \
 241         movw    zwreg, T_SYSNUM(t);                     \
 242         andb    $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
 243 
 244 /*
 245  * ASSERT(lwptoregs(lwp) == rp);
 246  *
 247  * This may seem obvious, but very odd things happen if this
 248  * assertion is false
 249  *
 250  * Preconditions:
 251  *      (%rsp is ready for normal call sequence)
 252  * Postconditions (if assertion is true):
 253  *      %r11 is smashed
 254  *
 255  * ASSERT(rp->r_cs == descnum)
 256  *
 257  * The code selector is written into the regs structure when the
 258  * lwp stack is created.  We use this ASSERT to validate that
 259  * the regs structure really matches how we came in.
 260  *
 261  * Preconditions:
 262  *      (%rsp is ready for normal call sequence)
 263  * Postconditions (if assertion is true):
 264  *      -none-
 265  *
 266  * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
 267  *
 268  * If this is false, it meant that we returned to userland without
 269  * updating the segment registers as we were supposed to.
 270  *
 271  * Note that we must ensure no interrupts or other traps intervene
 272  * between entering privileged mode and performing the assertion,
 273  * otherwise we may perform a context switch on the thread, which
 274  * will end up setting pcb_rupdate to 1 again.
 275  *
 276  * ASSERT(%cr0 & CR0_TS == 0);
 277  * Preconditions:
 278  *      (%rsp is ready for normal call sequence)
 279  * Postconditions (if assertion is true):
 280  *      (specified register is clobbered)
 281  *
 282  * Check to make sure that we are returning to user land and that CR0.TS
 283  * is not set. This is required as part of the eager FPU (see
 284  * uts/intel/ia32/os/fpu.c for more information).
 285  */
 286 
 287 #if defined(DEBUG)
 288 
 289 #if !defined(__lint)
 290 
 291 __lwptoregs_msg:
 292         .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
 293 
 294 __codesel_msg:
 295         .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
 296 
 297 __no_rupdate_msg:
 298         .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
 299 
 300 __bad_ts_msg:
 301         .string "sysscall_asm_amd64.s:%d CR0.TS set on user return"
 302 
 303 #endif  /* !__lint */
 304 
 305 #define ASSERT_LWPTOREGS(lwp, rp)                       \
 306         movq    LWP_REGS(lwp), %r11;                    \
 307         cmpq    rp, %r11;                               \
 308         je      7f;                                     \
 309         leaq    __lwptoregs_msg(%rip), %rdi;            \
 310         movl    $__LINE__, %esi;                        \
 311         movq    lwp, %rdx;                              \
 312         movq    %r11, %rcx;                             \
 313         movq    rp, %r8;                                \
 314         xorl    %eax, %eax;                             \
 315         call    panic;                                  \
 316 7:
 317 
 318 #define ASSERT_NO_RUPDATE_PENDING(lwp)                  \
 319         testb   $0x1, PCB_RUPDATE(lwp);                 \
 320         je      8f;                                     \
 321         movq    lwp, %rdx;                              \
 322         leaq    __no_rupdate_msg(%rip), %rdi;           \
 323         movl    $__LINE__, %esi;                        \
 324         xorl    %eax, %eax;                             \
 325         call    panic;                                  \
 326 8:
 327 
 328 #define ASSERT_CR0TS_ZERO(reg)                          \
 329         movq    %cr0, reg;                              \
 330         testq   $CR0_TS, reg;                           \
 331         jz      9f;                                     \
 332         leaq    __bad_ts_msg(%rip), %rdi;               \
 333         movl    $__LINE__, %esi;                        \
 334         xorl    %eax, %eax;                             \
 335         call    panic;                                  \
 336 9:
 337 
 338 #else
 339 #define ASSERT_LWPTOREGS(lwp, rp)
 340 #define ASSERT_NO_RUPDATE_PENDING(lwp)
 341 #define ASSERT_CR0TS_ZERO(reg)
 342 #endif
 343 
 344 /*
 345  * Do the traptrace thing and restore any registers we used
 346  * in situ.  Assumes that %rsp is pointing at the base of
 347  * the struct regs, obviously ..
 348  */
 349 #ifdef TRAPTRACE
 350 #define SYSCALL_TRAPTRACE(ttype)                                \
 351         TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype);               \
 352         TRACE_REGS(%rdi, %rsp, %rbx, %rcx);                     \
 353         TRACE_STAMP(%rdi);      /* rdtsc clobbers %eax, %edx */ \
 354         movq    REGOFF_RAX(%rsp), %rax;                         \
 355         movq    REGOFF_RBX(%rsp), %rbx;                         \
 356         movq    REGOFF_RCX(%rsp), %rcx;                         \
 357         movq    REGOFF_RDX(%rsp), %rdx;                         \
 358         movl    %eax, TTR_SYSNUM(%rdi);                         \
 359         movq    REGOFF_RDI(%rsp), %rdi
 360 
 361 #define SYSCALL_TRAPTRACE32(ttype)                              \
 362         SYSCALL_TRAPTRACE(ttype);                               \
 363         /* paranoia: clean the top 32-bits of the registers */  \
 364         orl     %eax, %eax;                                     \
 365         orl     %ebx, %ebx;                                     \
 366         orl     %ecx, %ecx;                                     \
 367         orl     %edx, %edx;                                     \
 368         orl     %edi, %edi
 369 #else   /* TRAPTRACE */
 370 #define SYSCALL_TRAPTRACE(ttype)
 371 #define SYSCALL_TRAPTRACE32(ttype)
 372 #endif  /* TRAPTRACE */
 373 
 374 /*
 375  * The 64-bit libc syscall wrapper does this:
 376  *
 377  * fn(<args>)
 378  * {
 379  *      movq    %rcx, %r10      -- because syscall smashes %rcx
 380  *      movl    $CODE, %eax
 381  *      syscall
 382  *      <error processing>
 383  * }
 384  *
 385  * Thus when we come into the kernel:
 386  *
 387  *      %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
 388  *      %rax is the syscall number
 389  *      %r12-%r15 contain caller state
 390  *
 391  * The syscall instruction arranges that:
 392  *
 393  *      %rcx contains the return %rip
 394  *      %r11d contains bottom 32-bits of %rflags
 395  *      %rflags is masked (as determined by the SFMASK msr)
 396  *      %cs is set to UCS_SEL (as determined by the STAR msr)
 397  *      %ss is set to UDS_SEL (as determined by the STAR msr)
 398  *      %rip is set to sys_syscall (as determined by the LSTAR msr)
 399  *
 400  * Or in other words, we have no registers available at all.
 401  * Only swapgs can save us!
 402  *
 403  * Under the hypervisor, the swapgs has happened already.  However, the
 404  * state of the world is very different from that we're familiar with.
 405  *
 406  * In particular, we have a stack structure like that for interrupt
 407  * gates, except that the %cs and %ss registers are modified for reasons
 408  * that are not entirely clear.  Critically, the %rcx/%r11 values do
 409  * *not* reflect the usage of those registers under a 'real' syscall[1];
 410  * the stack, therefore, looks like this:
 411  *
 412  *      0x0(rsp)        potentially junk %rcx
 413  *      0x8(rsp)        potentially junk %r11
 414  *      0x10(rsp)       user %rip
 415  *      0x18(rsp)       modified %cs
 416  *      0x20(rsp)       user %rflags
 417  *      0x28(rsp)       user %rsp
 418  *      0x30(rsp)       modified %ss
 419  *
 420  *
 421  * and before continuing on, we must load the %rip into %rcx and the
 422  * %rflags into %r11.
 423  *
 424  * [1] They used to, and we relied on it, but this was broken in 3.1.1.
 425  * Sigh.
 426  */
 427 #if defined(__xpv)
 428 #define XPV_SYSCALL_PROD                                                \
 429         movq    0x10(%rsp), %rcx;                                       \
 430         movq    0x20(%rsp), %r11;                                       \
 431         movq    0x28(%rsp), %rsp
 432 #else
 433 #define XPV_SYSCALL_PROD /* nothing */
 434 #endif
 435 
 436 #if defined(__lint)
 437 
 438 /*ARGSUSED*/
 439 void
 440 sys_syscall()
 441 {}
 442 
 443 void
 444 _allsyscalls()
 445 {}
 446 
 447 size_t _allsyscalls_size;
 448 
 449 #else   /* __lint */
 450 
 451         ENTRY_NP2(brand_sys_syscall,_allsyscalls)
 452         SWAPGS                          /* kernel gsbase */
 453         XPV_SYSCALL_PROD
 454         BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
 455         jmp     noprod_sys_syscall
 456 
 457         ALTENTRY(sys_syscall)
 458         SWAPGS                          /* kernel gsbase */
 459         XPV_SYSCALL_PROD
 460 
 461 noprod_sys_syscall:
 462         movq    %r15, %gs:CPU_RTMP_R15
 463         movq    %rsp, %gs:CPU_RTMP_RSP
 464 
 465         movq    %gs:CPU_THREAD, %r15
 466         movq    T_STACK(%r15), %rsp     /* switch from user to kernel stack */
 467 
 468         ASSERT_UPCALL_MASK_IS_SET
 469 
 470         movl    $UCS_SEL, REGOFF_CS(%rsp)
 471         movq    %rcx, REGOFF_RIP(%rsp)          /* syscall: %rip -> %rcx */
 472         movq    %r11, REGOFF_RFL(%rsp)          /* syscall: %rfl -> %r11d */
 473         movl    $UDS_SEL, REGOFF_SS(%rsp)
 474 
 475         movl    %eax, %eax                      /* wrapper: sysc# -> %eax */
 476         movq    %rdi, REGOFF_RDI(%rsp)
 477         movq    %rsi, REGOFF_RSI(%rsp)
 478         movq    %rdx, REGOFF_RDX(%rsp)
 479         movq    %r10, REGOFF_RCX(%rsp)          /* wrapper: %rcx -> %r10 */
 480         movq    %r10, %rcx                      /* arg[3] for direct calls */
 481 
 482         movq    %r8, REGOFF_R8(%rsp)
 483         movq    %r9, REGOFF_R9(%rsp)
 484         movq    %rax, REGOFF_RAX(%rsp)
 485         movq    %rbx, REGOFF_RBX(%rsp)
 486 
 487         movq    %rbp, REGOFF_RBP(%rsp)
 488         movq    %r10, REGOFF_R10(%rsp)
 489         movq    %gs:CPU_RTMP_RSP, %r11
 490         movq    %r11, REGOFF_RSP(%rsp)
 491         movq    %r12, REGOFF_R12(%rsp)
 492 
 493         movq    %r13, REGOFF_R13(%rsp)
 494         movq    %r14, REGOFF_R14(%rsp)
 495         movq    %gs:CPU_RTMP_R15, %r10
 496         movq    %r10, REGOFF_R15(%rsp)
 497         movq    $0, REGOFF_SAVFP(%rsp)
 498         movq    $0, REGOFF_SAVPC(%rsp)
 499 
 500         /*
 501          * Copy these registers here in case we end up stopped with
 502          * someone (like, say, /proc) messing with our register state.
 503          * We don't -restore- them unless we have to in update_sregs.
 504          *
 505          * Since userland -can't- change fsbase or gsbase directly,
 506          * and capturing them involves two serializing instructions,
 507          * we don't bother to capture them here.
 508          */
 509         xorl    %ebx, %ebx
 510         movw    %ds, %bx
 511         movq    %rbx, REGOFF_DS(%rsp)
 512         movw    %es, %bx
 513         movq    %rbx, REGOFF_ES(%rsp)
 514         movw    %fs, %bx
 515         movq    %rbx, REGOFF_FS(%rsp)
 516         movw    %gs, %bx
 517         movq    %rbx, REGOFF_GS(%rsp)
 518 
 519         /*
 520          * If we're trying to use TRAPTRACE though, I take that back: we're
 521          * probably debugging some problem in the SWAPGS logic and want to know
 522          * what the incoming gsbase was.
 523          *
 524          * Since we already did SWAPGS, record the KGSBASE.
 525          */
 526 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
 527         movl    $MSR_AMD_KGSBASE, %ecx
 528         rdmsr
 529         movl    %eax, REGOFF_GSBASE(%rsp)
 530         movl    %edx, REGOFF_GSBASE+4(%rsp)
 531 #endif
 532 
 533         /*
 534          * Machine state saved in the regs structure on the stack
 535          * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
 536          * %eax is the syscall number
 537          * %rsp is the thread's stack, %r15 is curthread
 538          * REG_RSP(%rsp) is the user's stack
 539          */
 540 
 541         SYSCALL_TRAPTRACE($TT_SYSC64)
 542 
 543         movq    %rsp, %rbp
 544 
 545         movq    T_LWP(%r15), %r14
 546         ASSERT_NO_RUPDATE_PENDING(%r14)
 547         ENABLE_INTR_FLAGS
 548 
 549         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 550         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate call) */
 551 
 552         ASSERT_LWPTOREGS(%r14, %rsp)
 553 
 554         movb    $LWP_SYS, LWP_STATE(%r14)
 555         incq    LWP_RU_SYSC(%r14)
 556         movb    $NORMALRETURN, LWP_EOSYS(%r14)
 557 
 558         incq    %gs:CPU_STATS_SYS_SYSCALL
 559 
 560         movw    %ax, T_SYSNUM(%r15)
 561         movzbl  T_PRE_SYS(%r15), %ebx
 562         ORL_SYSCALLTRACE(%ebx)
 563         testl   %ebx, %ebx
 564         jne     _syscall_pre
 565 
 566 _syscall_invoke:
 567         movq    REGOFF_RDI(%rbp), %rdi
 568         movq    REGOFF_RSI(%rbp), %rsi
 569         movq    REGOFF_RDX(%rbp), %rdx
 570         movq    REGOFF_RCX(%rbp), %rcx
 571         movq    REGOFF_R8(%rbp), %r8
 572         movq    REGOFF_R9(%rbp), %r9
 573 
 574         cmpl    $NSYSCALL, %eax
 575         jae     _syscall_ill
 576         shll    $SYSENT_SIZE_SHIFT, %eax
 577         leaq    sysent(%rax), %rbx
 578 
 579         movq    SY_CALLC(%rbx), %rax
 580         INDIRECT_CALL_REG(rax)
 581 
 582         movq    %rax, %r12
 583         movq    %rdx, %r13
 584 
 585         /*
 586          * If the handler returns two ints, then we need to split the
 587          * 64-bit return value into two 32-bit values.
 588          */
 589         testw   $SE_32RVAL2, SY_FLAGS(%rbx)
 590         je      5f
 591         movq    %r12, %r13
 592         shrq    $32, %r13       /* upper 32-bits into %edx */
 593         movl    %r12d, %r12d    /* lower 32-bits into %eax */
 594 5:
 595         /*
 596          * Optimistically assume that there's no post-syscall
 597          * work to do.  (This is to avoid having to call syscall_mstate()
 598          * with interrupts disabled)
 599          */
 600         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 601 
 602         /*
 603          * We must protect ourselves from being descheduled here;
 604          * If we were, and we ended up on another cpu, or another
 605          * lwp got in ahead of us, it could change the segment
 606          * registers without us noticing before we return to userland.
 607          */
 608         CLI(%r14)
 609         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
 610         jne     _syscall_post
 611 
 612         /*
 613          * We need to protect ourselves against non-canonical return values
 614          * because Intel doesn't check for them on sysret (AMD does).  Canonical
 615          * addresses on current amd64 processors only use 48-bits for VAs; an
 616          * address is canonical if all upper bits (47-63) are identical. If we
 617          * find a non-canonical %rip, we opt to go through the full
 618          * _syscall_post path which takes us into an iretq which is not
 619          * susceptible to the same problems sysret is.
 620          *
 621          * We're checking for a canonical address by first doing an arithmetic
 622          * shift. This will fill in the remaining bits with the value of bit 63.
 623          * If the address were canonical, the register would now have either all
 624          * zeroes or all ones in it. Therefore we add one (inducing overflow)
 625          * and compare against 1. A canonical address will either be zero or one
 626          * at this point, hence the use of ja.
 627          *
 628          * At this point, r12 and r13 have the return value so we can't use
 629          * those registers.
 630          */
 631         movq    REGOFF_RIP(%rsp), %rcx
 632         sarq    $47, %rcx
 633         incq    %rcx
 634         cmpq    $1, %rcx
 635         ja      _syscall_post
 636 
 637 
 638         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 639 
 640         movq    %r12, REGOFF_RAX(%rsp)
 641         movq    %r13, REGOFF_RDX(%rsp)
 642 
 643         /*
 644          * Clobber %r11 as we check CR0.TS.
 645          */
 646         ASSERT_CR0TS_ZERO(%r11)
 647 
 648         /*
 649          * Unlike other cases, because we need to restore the user stack pointer
 650          * before exiting the kernel we must clear the microarch state before
 651          * getting here. This should be safe because it means that the only
 652          * values on the bus after this are based on the user's registers and
 653          * potentially the addresses where we stored them. Given the constraints
 654          * of sysret, that's how it has to be.
 655          */
 656         call    x86_md_clear
 657 
 658         /*
 659          * To get back to userland, we need the return %rip in %rcx and
 660          * the return %rfl in %r11d.  The sysretq instruction also arranges
 661          * to fix up %cs and %ss; everything else is our responsibility.
 662          */
 663         movq    REGOFF_RDI(%rsp), %rdi
 664         movq    REGOFF_RSI(%rsp), %rsi
 665         movq    REGOFF_RDX(%rsp), %rdx
 666         /* %rcx used to restore %rip value */
 667 
 668         movq    REGOFF_R8(%rsp), %r8
 669         movq    REGOFF_R9(%rsp), %r9
 670         movq    REGOFF_RAX(%rsp), %rax
 671         movq    REGOFF_RBX(%rsp), %rbx
 672 
 673         movq    REGOFF_RBP(%rsp), %rbp
 674         movq    REGOFF_R10(%rsp), %r10
 675         /* %r11 used to restore %rfl value */
 676         movq    REGOFF_R12(%rsp), %r12
 677 
 678         movq    REGOFF_R13(%rsp), %r13
 679         movq    REGOFF_R14(%rsp), %r14
 680         movq    REGOFF_R15(%rsp), %r15
 681 
 682         movq    REGOFF_RIP(%rsp), %rcx
 683         movl    REGOFF_RFL(%rsp), %r11d
 684 
 685 #if defined(__xpv)
 686         addq    $REGOFF_RIP, %rsp
 687 #else
 688         movq    REGOFF_RSP(%rsp), %rsp
 689 #endif
 690 
 691         /*
 692          * There can be no instructions between the ALTENTRY below and
 693          * SYSRET or we could end up breaking brand support. See label usage
 694          * in sn1_brand_syscall_callback for an example.
 695          */
 696         ASSERT_UPCALL_MASK_IS_SET
 697 #if defined(__xpv)
 698         SYSRETQ
 699         ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
 700 
 701         /*
 702          * We can only get here after executing a brand syscall
 703          * interposition callback handler and simply need to
 704          * "sysretq" back to userland. On the hypervisor this
 705          * involves the iret hypercall which requires us to construct
 706          * just enough of the stack needed for the hypercall.
 707          * (rip, cs, rflags, rsp, ss).
 708          */
 709         movq    %rsp, %gs:CPU_RTMP_RSP          /* save user's rsp */
 710         movq    %gs:CPU_THREAD, %r11
 711         movq    T_STACK(%r11), %rsp
 712 
 713         movq    %rcx, REGOFF_RIP(%rsp)
 714         movl    $UCS_SEL, REGOFF_CS(%rsp)
 715         movq    %gs:CPU_RTMP_RSP, %r11
 716         movq    %r11, REGOFF_RSP(%rsp)
 717         pushfq
 718         popq    %r11                            /* hypercall enables ints */
 719         movq    %r11, REGOFF_RFL(%rsp)
 720         movl    $UDS_SEL, REGOFF_SS(%rsp)
 721         addq    $REGOFF_RIP, %rsp
 722         /*
 723          * XXPV: see comment in SYSRETQ definition for future optimization
 724          *       we could take.
 725          */
 726         ASSERT_UPCALL_MASK_IS_SET
 727         SYSRETQ
 728 #else
 729         ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
 730         jmp     tr_sysretq
 731 #endif
 732         /*NOTREACHED*/
 733         SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
 734 
 735 _syscall_pre:
 736         call    pre_syscall
 737         movl    %eax, %r12d
 738         testl   %eax, %eax
 739         jne     _syscall_post_call
 740         /*
 741          * Didn't abort, so reload the syscall args and invoke the handler.
 742          */
 743         movzwl  T_SYSNUM(%r15), %eax
 744         jmp     _syscall_invoke
 745 
 746 _syscall_ill:
 747         call    nosys
 748         movq    %rax, %r12
 749         movq    %rdx, %r13
 750         jmp     _syscall_post_call
 751 
 752 _syscall_post:
 753         STI
 754         /*
 755          * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
 756          * so that we can account for the extra work it takes us to finish.
 757          */
 758         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 759 _syscall_post_call:
 760         movq    %r12, %rdi
 761         movq    %r13, %rsi
 762         call    post_syscall
 763         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 764         jmp     _sys_rtt
 765         SET_SIZE(sys_syscall)
 766         SET_SIZE(brand_sys_syscall)
 767 
 768 #endif  /* __lint */
 769 
 770 #if defined(__lint)
 771 
 772 /*ARGSUSED*/
 773 void
 774 sys_syscall32()
 775 {}
 776 
 777 #else   /* __lint */
 778 
 779         ENTRY_NP(brand_sys_syscall32)
 780         SWAPGS                          /* kernel gsbase */
 781         XPV_TRAP_POP
 782         BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
 783         jmp     nopop_sys_syscall32
 784 
 785         ALTENTRY(sys_syscall32)
 786         SWAPGS                          /* kernel gsbase */
 787         XPV_TRAP_POP
 788 
 789 nopop_sys_syscall32:
 790         movl    %esp, %r10d
 791         movq    %gs:CPU_THREAD, %r15
 792         movq    T_STACK(%r15), %rsp
 793         movl    %eax, %eax
 794 
 795         movl    $U32CS_SEL, REGOFF_CS(%rsp)
 796         movl    %ecx, REGOFF_RIP(%rsp)          /* syscall: %rip -> %rcx */
 797         movq    %r11, REGOFF_RFL(%rsp)          /* syscall: %rfl -> %r11d */
 798         movq    %r10, REGOFF_RSP(%rsp)
 799         movl    $UDS_SEL, REGOFF_SS(%rsp)
 800 
 801 _syscall32_save:
 802         movl    %edi, REGOFF_RDI(%rsp)
 803         movl    %esi, REGOFF_RSI(%rsp)
 804         movl    %ebp, REGOFF_RBP(%rsp)
 805         movl    %ebx, REGOFF_RBX(%rsp)
 806         movl    %edx, REGOFF_RDX(%rsp)
 807         movl    %ecx, REGOFF_RCX(%rsp)
 808         movl    %eax, REGOFF_RAX(%rsp)          /* wrapper: sysc# -> %eax */
 809         movq    $0, REGOFF_SAVFP(%rsp)
 810         movq    $0, REGOFF_SAVPC(%rsp)
 811 
 812         /*
 813          * Copy these registers here in case we end up stopped with
 814          * someone (like, say, /proc) messing with our register state.
 815          * We don't -restore- them unless we have to in update_sregs.
 816          *
 817          * Since userland -can't- change fsbase or gsbase directly,
 818          * we don't bother to capture them here.
 819          */
 820         xorl    %ebx, %ebx
 821         movw    %ds, %bx
 822         movq    %rbx, REGOFF_DS(%rsp)
 823         movw    %es, %bx
 824         movq    %rbx, REGOFF_ES(%rsp)
 825         movw    %fs, %bx
 826         movq    %rbx, REGOFF_FS(%rsp)
 827         movw    %gs, %bx
 828         movq    %rbx, REGOFF_GS(%rsp)
 829 
 830         /*
 831          * If we're trying to use TRAPTRACE though, I take that back: we're
 832          * probably debugging some problem in the SWAPGS logic and want to know
 833          * what the incoming gsbase was.
 834          *
 835          * Since we already did SWAPGS, record the KGSBASE.
 836          */
 837 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
 838         movl    $MSR_AMD_KGSBASE, %ecx
 839         rdmsr
 840         movl    %eax, REGOFF_GSBASE(%rsp)
 841         movl    %edx, REGOFF_GSBASE+4(%rsp)
 842 #endif
 843 
 844         /*
 845          * Application state saved in the regs structure on the stack
 846          * %eax is the syscall number
 847          * %rsp is the thread's stack, %r15 is curthread
 848          * REG_RSP(%rsp) is the user's stack
 849          */
 850 
 851         SYSCALL_TRAPTRACE32($TT_SYSC)
 852 
 853         movq    %rsp, %rbp
 854 
 855         movq    T_LWP(%r15), %r14
 856         ASSERT_NO_RUPDATE_PENDING(%r14)
 857 
 858         ENABLE_INTR_FLAGS
 859 
 860         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 861         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate call) */
 862 
 863         ASSERT_LWPTOREGS(%r14, %rsp)
 864 
 865         incq     %gs:CPU_STATS_SYS_SYSCALL
 866 
 867         /*
 868          * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
 869          * into 64-bit (long) arg slots, maintaining 16 byte alignment.  Or
 870          * more succinctly:
 871          *
 872          *      SA(MAXSYSARGS * sizeof (long)) == 64
 873          */
 874 #define SYS_DROP        64                      /* drop for args */
 875         subq    $SYS_DROP, %rsp
 876         movb    $LWP_SYS, LWP_STATE(%r14)
 877         movq    %r15, %rdi
 878         movq    %rsp, %rsi
 879         call    syscall_entry
 880 
 881         /*
 882          * Fetch the arguments copied onto the kernel stack and put
 883          * them in the right registers to invoke a C-style syscall handler.
 884          * %rax contains the handler address.
 885          *
 886          * Ideas for making all this go faster of course include simply
 887          * forcibly fetching 6 arguments from the user stack under lofault
 888          * protection, reverting to copyin_args only when watchpoints
 889          * are in effect.
 890          *
 891          * (If we do this, make sure that exec and libthread leave
 892          * enough space at the top of the stack to ensure that we'll
 893          * never do a fetch from an invalid page.)
 894          *
 895          * Lots of ideas here, but they won't really help with bringup B-)
 896          * Correctness can't wait, performance can wait a little longer ..
 897          */
 898 
 899         movq    %rax, %rbx
 900         movl    0(%rsp), %edi
 901         movl    8(%rsp), %esi
 902         movl    0x10(%rsp), %edx
 903         movl    0x18(%rsp), %ecx
 904         movl    0x20(%rsp), %r8d
 905         movl    0x28(%rsp), %r9d
 906 
 907         movq    SY_CALLC(%rbx), %rax
 908         INDIRECT_CALL_REG(rax)
 909 
 910         movq    %rbp, %rsp      /* pop the args */
 911 
 912         /*
 913          * amd64 syscall handlers -always- return a 64-bit value in %rax.
 914          * On the 32-bit kernel, they always return that value in %eax:%edx
 915          * as required by the 32-bit ABI.
 916          *
 917          * Simulate the same behaviour by unconditionally splitting the
 918          * return value in the same way.
 919          */
 920         movq    %rax, %r13
 921         shrq    $32, %r13       /* upper 32-bits into %edx */
 922         movl    %eax, %r12d     /* lower 32-bits into %eax */
 923 
 924         /*
 925          * Optimistically assume that there's no post-syscall
 926          * work to do.  (This is to avoid having to call syscall_mstate()
 927          * with interrupts disabled)
 928          */
 929         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 930 
 931         /*
 932          * We must protect ourselves from being descheduled here;
 933          * If we were, and we ended up on another cpu, or another
 934          * lwp got in ahead of us, it could change the segment
 935          * registers without us noticing before we return to userland.
 936          */
 937         CLI(%r14)
 938         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
 939         jne     _full_syscall_postsys32
 940         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 941 
 942         /*
 943          * Clobber %r11 as we check CR0.TS.
 944          */
 945         ASSERT_CR0TS_ZERO(%r11)
 946 
 947         /*
 948          * Unlike other cases, because we need to restore the user stack pointer
 949          * before exiting the kernel we must clear the microarch state before
 950          * getting here. This should be safe because it means that the only
 951          * values on the bus after this are based on the user's registers and
 952          * potentially the addresses where we stored them. Given the constraints
 953          * of sysret, that's how it has to be.
 954          */
 955         call    x86_md_clear
 956 
 957         /*
 958          * To get back to userland, we need to put the return %rip in %rcx and
 959          * the return %rfl in %r11d.  The sysret instruction also arranges
 960          * to fix up %cs and %ss; everything else is our responsibility.
 961          */
 962 
 963         movl    %r12d, %eax                     /* %eax: rval1 */
 964         movl    REGOFF_RBX(%rsp), %ebx
 965         /* %ecx used for return pointer */
 966         movl    %r13d, %edx                     /* %edx: rval2 */
 967         movl    REGOFF_RBP(%rsp), %ebp
 968         movl    REGOFF_RSI(%rsp), %esi
 969         movl    REGOFF_RDI(%rsp), %edi
 970 
 971         movl    REGOFF_RFL(%rsp), %r11d         /* %r11 -> eflags */
 972         movl    REGOFF_RIP(%rsp), %ecx          /* %ecx -> %eip */
 973         movl    REGOFF_RSP(%rsp), %esp
 974 
 975         ASSERT_UPCALL_MASK_IS_SET
 976         ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
 977         jmp     tr_sysretl
 978         SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
 979         /*NOTREACHED*/
 980 
 981 _full_syscall_postsys32:
 982         STI
 983         /*
 984          * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
 985          * so that we can account for the extra work it takes us to finish.
 986          */
 987         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 988         movq    %r15, %rdi
 989         movq    %r12, %rsi                      /* rval1 - %eax */
 990         movq    %r13, %rdx                      /* rval2 - %edx */
 991         call    syscall_exit
 992         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 993         jmp     _sys_rtt
 994         SET_SIZE(sys_syscall32)
 995         SET_SIZE(brand_sys_syscall32)
 996 
 997 #endif  /* __lint */
 998 
 999 /*
1000  * System call handler via the sysenter instruction
1001  * Used only for 32-bit system calls on the 64-bit kernel.
1002  *
1003  * The caller in userland has arranged that:
1004  *
1005  * -    %eax contains the syscall number
1006  * -    %ecx contains the user %esp
1007  * -    %edx contains the return %eip
1008  * -    the user stack contains the args to the syscall
1009  *
1010  * Hardware and (privileged) initialization code have arranged that by
1011  * the time the sysenter instructions completes:
1012  *
1013  * - %rip is pointing to sys_sysenter (below).
1014  * - %cs and %ss are set to kernel text and stack (data) selectors.
1015  * - %rsp is pointing at the lwp's stack
1016  * - interrupts have been disabled.
1017  *
1018  * Note that we are unable to return both "rvals" to userland with
1019  * this call, as %edx is used by the sysexit instruction.
1020  *
1021  * One final complication in this routine is its interaction with
1022  * single-stepping in a debugger.  For most of the system call mechanisms, the
1023  * CPU automatically clears the single-step flag before we enter the kernel.
1024  * The sysenter mechanism does not clear the flag, so a user single-stepping
1025  * through a libc routine may suddenly find themself single-stepping through the
1026  * kernel.  To detect this, kmdb and trap() both compare the trap %pc to the
1027  * [brand_]sys_enter addresses on each single-step trap.  If it finds that we
1028  * have single-stepped to a sysenter entry point, it explicitly clears the flag
1029  * and executes the sys_sysenter routine.
1030  *
1031  * One final complication in this final complication is the fact that we have
1032  * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
1033  * If we enter at brand_sys_sysenter and start single-stepping through the
1034  * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
1035  * kmdb cannot distinguish between that valid single-step and the undesirable
1036  * one mentioned above.  To avoid this situation, we simply add a jump over the
1037  * instruction at sys_sysenter to make it impossible to single-step to it.
1038  */
1039 #if defined(__lint)
1040 
1041 void
1042 sys_sysenter()
1043 {}
1044 
1045 #else   /* __lint */
1046 
1047         ENTRY_NP(brand_sys_sysenter)
1048         SWAPGS                          /* kernel gsbase */
1049         ALTENTRY(_brand_sys_sysenter_post_swapgs)
1050 
1051         BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
1052         /*
1053          * Jump over sys_sysenter to allow single-stepping as described
1054          * above.
1055          */
1056         jmp     _sys_sysenter_post_swapgs
1057 
1058         ALTENTRY(sys_sysenter)
1059         SWAPGS                          /* kernel gsbase */
1060         ALTENTRY(_sys_sysenter_post_swapgs)
1061 
1062         movq    %gs:CPU_THREAD, %r15
1063 
1064         movl    $U32CS_SEL, REGOFF_CS(%rsp)
1065         movl    %ecx, REGOFF_RSP(%rsp)          /* wrapper: %esp -> %ecx */
1066         movl    %edx, REGOFF_RIP(%rsp)          /* wrapper: %eip -> %edx */
1067         /*
1068          * NOTE: none of the instructions that run before we get here should
1069          * clobber bits in (R)FLAGS! This includes the kpti trampoline.
1070          */
1071         pushfq
1072         popq    %r10
1073         movl    $UDS_SEL, REGOFF_SS(%rsp)
1074 
1075         /*
1076          * Set the interrupt flag before storing the flags to the
1077          * flags image on the stack so we can return to user with
1078          * interrupts enabled if we return via sys_rtt_syscall32
1079          */
1080         orq     $PS_IE, %r10
1081         movq    %r10, REGOFF_RFL(%rsp)
1082 
1083         movl    %edi, REGOFF_RDI(%rsp)
1084         movl    %esi, REGOFF_RSI(%rsp)
1085         movl    %ebp, REGOFF_RBP(%rsp)
1086         movl    %ebx, REGOFF_RBX(%rsp)
1087         movl    %edx, REGOFF_RDX(%rsp)
1088         movl    %ecx, REGOFF_RCX(%rsp)
1089         movl    %eax, REGOFF_RAX(%rsp)          /* wrapper: sysc# -> %eax */
1090         movq    $0, REGOFF_SAVFP(%rsp)
1091         movq    $0, REGOFF_SAVPC(%rsp)
1092 
1093         /*
1094          * Copy these registers here in case we end up stopped with
1095          * someone (like, say, /proc) messing with our register state.
1096          * We don't -restore- them unless we have to in update_sregs.
1097          *
1098          * Since userland -can't- change fsbase or gsbase directly,
1099          * we don't bother to capture them here.
1100          */
1101         xorl    %ebx, %ebx
1102         movw    %ds, %bx
1103         movq    %rbx, REGOFF_DS(%rsp)
1104         movw    %es, %bx
1105         movq    %rbx, REGOFF_ES(%rsp)
1106         movw    %fs, %bx
1107         movq    %rbx, REGOFF_FS(%rsp)
1108         movw    %gs, %bx
1109         movq    %rbx, REGOFF_GS(%rsp)
1110 
1111         /*
1112          * If we're trying to use TRAPTRACE though, I take that back: we're
1113          * probably debugging some problem in the SWAPGS logic and want to know
1114          * what the incoming gsbase was.
1115          *
1116          * Since we already did SWAPGS, record the KGSBASE.
1117          */
1118 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
1119         movl    $MSR_AMD_KGSBASE, %ecx
1120         rdmsr
1121         movl    %eax, REGOFF_GSBASE(%rsp)
1122         movl    %edx, REGOFF_GSBASE+4(%rsp)
1123 #endif
1124 
1125         /*
1126          * Application state saved in the regs structure on the stack
1127          * %eax is the syscall number
1128          * %rsp is the thread's stack, %r15 is curthread
1129          * REG_RSP(%rsp) is the user's stack
1130          */
1131 
1132         SYSCALL_TRAPTRACE($TT_SYSENTER)
1133 
1134         movq    %rsp, %rbp
1135 
1136         movq    T_LWP(%r15), %r14
1137         ASSERT_NO_RUPDATE_PENDING(%r14)
1138 
1139         ENABLE_INTR_FLAGS
1140 
1141         /*
1142          * Catch 64-bit process trying to issue sysenter instruction
1143          * on Nocona based systems.
1144          */
1145         movq    LWP_PROCP(%r14), %rax
1146         cmpq    $DATAMODEL_ILP32, P_MODEL(%rax)
1147         je      7f
1148 
1149         /*
1150          * For a non-32-bit process, simulate a #ud, since that's what
1151          * native hardware does.  The traptrace entry (above) will
1152          * let you know what really happened.
1153          */
1154         movq    $T_ILLINST, REGOFF_TRAPNO(%rsp)
1155         movq    REGOFF_CS(%rsp), %rdi
1156         movq    %rdi, REGOFF_ERR(%rsp)
1157         movq    %rsp, %rdi
1158         movq    REGOFF_RIP(%rsp), %rsi
1159         movl    %gs:CPU_ID, %edx
1160         call    trap
1161         jmp     _sys_rtt
1162 7:
1163 
1164         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1165         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate calls) */
1166 
1167         ASSERT_LWPTOREGS(%r14, %rsp)
1168 
1169         incq    %gs:CPU_STATS_SYS_SYSCALL
1170 
1171         /*
1172          * Make some space for MAXSYSARGS (currently 8) 32-bit args
1173          * placed into 64-bit (long) arg slots, plus one 64-bit
1174          * (long) arg count, maintaining 16 byte alignment.
1175          */
1176         subq    $SYS_DROP, %rsp
1177         movb    $LWP_SYS, LWP_STATE(%r14)
1178         movq    %r15, %rdi
1179         movq    %rsp, %rsi
1180         call    syscall_entry
1181 
1182         /*
1183          * Fetch the arguments copied onto the kernel stack and put
1184          * them in the right registers to invoke a C-style syscall handler.
1185          * %rax contains the handler address.
1186          */
1187         movq    %rax, %rbx
1188         movl    0(%rsp), %edi
1189         movl    8(%rsp), %esi
1190         movl    0x10(%rsp), %edx
1191         movl    0x18(%rsp), %ecx
1192         movl    0x20(%rsp), %r8d
1193         movl    0x28(%rsp), %r9d
1194 
1195         movq    SY_CALLC(%rbx), %rax
1196         INDIRECT_CALL_REG(rax)
1197 
1198         movq    %rbp, %rsp      /* pop the args */
1199 
1200         /*
1201          * amd64 syscall handlers -always- return a 64-bit value in %rax.
1202          * On the 32-bit kernel, the always return that value in %eax:%edx
1203          * as required by the 32-bit ABI.
1204          *
1205          * Simulate the same behaviour by unconditionally splitting the
1206          * return value in the same way.
1207          */
1208         movq    %rax, %r13
1209         shrq    $32, %r13       /* upper 32-bits into %edx */
1210         movl    %eax, %r12d     /* lower 32-bits into %eax */
1211 
1212         /*
1213          * Optimistically assume that there's no post-syscall
1214          * work to do.  (This is to avoid having to call syscall_mstate()
1215          * with interrupts disabled)
1216          */
1217         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1218 
1219         /*
1220          * We must protect ourselves from being descheduled here;
1221          * If we were, and we ended up on another cpu, or another
1222          * lwp got int ahead of us, it could change the segment
1223          * registers without us noticing before we return to userland.
1224          *
1225          * This cli is undone in the tr_sysexit trampoline code.
1226          */
1227         cli
1228         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1229         jne     _full_syscall_postsys32
1230         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1231 
1232         /*
1233          * To get back to userland, load up the 32-bit registers and
1234          * sysexit back where we came from.
1235          */
1236 
1237         /*
1238          * Interrupts will be turned on by the 'sti' executed just before
1239          * sysexit.  The following ensures that restoring the user's rflags
1240          * doesn't enable interrupts too soon.
1241          */
1242         andq    $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1243 
1244         /*
1245          * Clobber %r11 as we check CR0.TS.
1246          */
1247         ASSERT_CR0TS_ZERO(%r11)
1248 
1249         /*
1250          * (There's no point in loading up %edx because the sysexit
1251          * mechanism smashes it.)
1252          */
1253         movl    %r12d, %eax
1254         movl    REGOFF_RBX(%rsp), %ebx
1255         movl    REGOFF_RBP(%rsp), %ebp
1256         movl    REGOFF_RSI(%rsp), %esi
1257         movl    REGOFF_RDI(%rsp), %edi
1258 
1259         movl    REGOFF_RIP(%rsp), %edx  /* sysexit: %edx -> %eip */
1260         pushq   REGOFF_RFL(%rsp)
1261         popfq
1262         movl    REGOFF_RSP(%rsp), %ecx  /* sysexit: %ecx -> %esp */
1263         ALTENTRY(sys_sysenter_swapgs_sysexit)
1264         call    x86_md_clear
1265         jmp     tr_sysexit
1266         SET_SIZE(sys_sysenter_swapgs_sysexit)
1267         SET_SIZE(sys_sysenter)
1268         SET_SIZE(_sys_sysenter_post_swapgs)
1269         SET_SIZE(brand_sys_sysenter)
1270 
1271 #endif  /* __lint */
1272 
1273 /*
1274  * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1275  * the generic i386 libc to do system calls. We do a small amount of setup
1276  * before jumping into the existing sys_syscall32 path.
1277  */
1278 #if defined(__lint)
1279 
1280 /*ARGSUSED*/
1281 void
1282 sys_syscall_int()
1283 {}
1284 
1285 #else   /* __lint */
1286 
1287         ENTRY_NP(brand_sys_syscall_int)
1288         SWAPGS                          /* kernel gsbase */
1289         XPV_TRAP_POP
1290         call    smap_enable
1291         BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1292         jmp     nopop_syscall_int
1293 
1294         ALTENTRY(sys_syscall_int)
1295         SWAPGS                          /* kernel gsbase */
1296         XPV_TRAP_POP
1297         call    smap_enable
1298 
1299 nopop_syscall_int:
1300         movq    %gs:CPU_THREAD, %r15
1301         movq    T_STACK(%r15), %rsp
1302         movl    %eax, %eax
1303         /*
1304          * Set t_post_sys on this thread to force ourselves out via the slow
1305          * path. It might be possible at some later date to optimize this out
1306          * and use a faster return mechanism.
1307          */
1308         movb    $1, T_POST_SYS(%r15)
1309         CLEAN_CS
1310         jmp     _syscall32_save
1311         /*
1312          * There should be no instructions between this label and SWAPGS/IRET
1313          * or we could end up breaking branded zone support. See the usage of
1314          * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1315          * for examples.
1316          *
1317          * We want to swapgs to maintain the invariant that all entries into
1318          * tr_iret_user are done on the user gsbase.
1319          */
1320         ALTENTRY(sys_sysint_swapgs_iret)
1321         call    x86_md_clear
1322         SWAPGS
1323         jmp     tr_iret_user
1324         /*NOTREACHED*/
1325         SET_SIZE(sys_sysint_swapgs_iret)
1326         SET_SIZE(sys_syscall_int)
1327         SET_SIZE(brand_sys_syscall_int)
1328 
1329 #endif  /* __lint */
1330 
1331 /*
1332  * Legacy 32-bit applications and old libc implementations do lcalls;
1333  * we should never get here because the LDT entry containing the syscall
1334  * segment descriptor has the "segment present" bit cleared, which means
1335  * we end up processing those system calls in trap() via a not-present trap.
1336  *
1337  * We do it this way because a call gate unhelpfully does -nothing- to the
1338  * interrupt flag bit, so an interrupt can run us just after the lcall
1339  * completes, but just before the swapgs takes effect.   Thus the INTR_PUSH and
1340  * INTR_POP paths would have to be slightly more complex to dance around
1341  * this problem, and end up depending explicitly on the first
1342  * instruction of this handler being either swapgs or cli.
1343  */
1344 
1345 #if defined(__lint)
1346 
1347 /*ARGSUSED*/
1348 void
1349 sys_lcall32()
1350 {}
1351 
1352 #else   /* __lint */
1353 
1354         ENTRY_NP(sys_lcall32)
1355         SWAPGS                          /* kernel gsbase */
1356         pushq   $0
1357         pushq   %rbp
1358         movq    %rsp, %rbp
1359         leaq    __lcall_panic_str(%rip), %rdi
1360         xorl    %eax, %eax
1361         call    panic
1362         SET_SIZE(sys_lcall32)
1363 
1364 __lcall_panic_str:
1365         .string "sys_lcall32: shouldn't be here!"
1366 
1367 /*
1368  * Declare a uintptr_t which covers the entire pc range of syscall
1369  * handlers for the stack walkers that need this.
1370  */
1371         .align  CPTRSIZE
1372         .globl  _allsyscalls_size
1373         .type   _allsyscalls_size, @object
1374 _allsyscalls_size:
1375         .NWORD  . - _allsyscalls
1376         SET_SIZE(_allsyscalls_size)
1377 
1378 #endif  /* __lint */
1379 
1380 /*
1381  * These are the thread context handlers for lwps using sysenter/sysexit.
1382  */
1383 
1384 #if defined(__lint)
1385 
1386 /*ARGSUSED*/
1387 void
1388 sep_save(void *ksp)
1389 {}
1390 
1391 /*ARGSUSED*/
1392 void
1393 sep_restore(void *ksp)
1394 {}
1395 
1396 #else   /* __lint */
1397 
1398         /*
1399          * setting this value to zero as we switch away causes the
1400          * stack-pointer-on-sysenter to be NULL, ensuring that we
1401          * don't silently corrupt another (preempted) thread stack
1402          * when running an lwp that (somehow) didn't get sep_restore'd
1403          */
1404         ENTRY_NP(sep_save)
1405         xorl    %edx, %edx
1406         xorl    %eax, %eax
1407         movl    $MSR_INTC_SEP_ESP, %ecx
1408         wrmsr
1409         ret
1410         SET_SIZE(sep_save)
1411 
1412         /*
1413          * Update the kernel stack pointer as we resume onto this cpu.
1414          */
1415         ENTRY_NP(sep_restore)
1416         movq    %rdi, %rdx
1417         shrq    $32, %rdx
1418         movl    %edi, %eax
1419         movl    $MSR_INTC_SEP_ESP, %ecx
1420         wrmsr
1421         ret
1422         SET_SIZE(sep_restore)
1423 
1424 #endif  /* __lint */