Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>

@@ -1,9 +1,9 @@
 /*
  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
- * Copyright (c) 2017 Joyent, Inc.
+ * Copyright (c) 2018 Joyent, Inc.
  */
 
 /*
  * Copyright (c) 1989, 1990 William F. Jolitz.
  * Copyright (c) 1990 The Regents of the University of California.

@@ -110,10 +110,24 @@
 #define TRAP_ERR(trapno)        \
         push    $trapno 
 
 #endif  /* __xpv && __amd64 */
 
+        /*
+         * These are the stacks used on cpu0 for taking double faults,
+         * NMIs and MCEs (the latter two only on amd64 where we have IST).
+         *
+         * We define them here instead of in a C file so that we can page-align
+         * them (gcc won't do that in a .c file).
+         */
+        .data
+        DGDEF3(dblfault_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
+        .fill   DEFAULTSTKSZ, 1, 0
+        DGDEF3(nmi_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
+        .fill   DEFAULTSTKSZ, 1, 0
+        DGDEF3(mce_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
+        .fill   DEFAULTSTKSZ, 1, 0
 
         /*
          * #DE
          */
         ENTRY_NP(div0trap)

@@ -161,10 +175,16 @@
         leaq    sys_sysenter(%rip), %r11
         cmpq    %r11, 24(%rsp)  /* Compare to saved r_rip on the stack */
         je      1f
         leaq    brand_sys_sysenter(%rip), %r11
         cmpq    %r11, 24(%rsp)  /* Compare to saved r_rip on the stack */
+        je      1f
+        leaq    tr_sys_sysenter(%rip), %r11
+        cmpq    %r11, 24(%rsp)
+        je      1f
+        leaq    tr_brand_sys_sysenter(%rip), %r11
+        cmpq    %r11, 24(%rsp)
         jne     2f
 1:      SWAPGS
 2:      popq    %r11
 #endif  /* !__xpv */
 

@@ -212,10 +232,14 @@
  * for this processor.  If we came from userland, set kgsbase else
  * set gsbase.  We find the proper cpu struct by looping through
  * the cpu structs for all processors till we find a match for the gdt
  * of the trapping processor.  The stack is expected to be pointing at
  * the standard regs pushed by hardware on a trap (plus error code and trapno).
+ *
+ * It's ok for us to clobber gsbase here (and possibly end up with both gsbase
+ * and kgsbase set to the same value) because we're not going back the normal
+ * way out of here (via IRET). Where we're going, we don't need no user %gs.
  */
 #define SET_CPU_GSBASE                                                  \
         subq    $REGOFF_TRAPNO, %rsp;   /* save regs */                 \
         movq    %rax, REGOFF_RAX(%rsp);                                 \
         movq    %rbx, REGOFF_RBX(%rsp);                                 \

@@ -292,11 +316,11 @@
 
         movq    %rbp, %rdi
         call    av_dispatch_nmivect
 
         INTR_POP
-        IRET
+        jmp     tr_iret_auto
         /*NOTREACHED*/
         SET_SIZE(nmiint)
 
 #elif defined(__i386)
 

@@ -431,11 +455,11 @@
         movq    56(%rsp), %rax          /* load calling SS */
         movq    %rax, 40(%rsp)          /* store calling SS */
         movq    32(%rsp), %rax          /* reload calling RSP */
         movq    %rbp, (%rax)            /* store %rbp there */
         popq    %rax                    /* pop off temp */
-        IRET                            /* return from interrupt */
+        jmp     tr_iret_kernel          /* return from interrupt */
         /*NOTREACHED*/
 
 ud_leave:
         /*
          * We must emulate a "leave", which is the same as a "movq %rbp, %rsp"

@@ -452,21 +476,21 @@
         movq    (%rbp), %rax            /* get new %rbp */
         addq    $8, %rbp                /* adjust new %rsp */
         movq    %rbp, 32(%rsp)          /* store new %rsp */
         movq    %rax, %rbp              /* set new %rbp */
         popq    %rax                    /* pop off temp */
-        IRET                            /* return from interrupt */
+        jmp     tr_iret_kernel          /* return from interrupt */
         /*NOTREACHED*/
 
 ud_nop:
         /*
          * We must emulate a "nop".  This is obviously not hard:  we need only
          * advance the %rip by one.
          */
         INTR_POP
         incq    (%rsp)
-        IRET
+        jmp     tr_iret_kernel
         /*NOTREACHED*/
 
 ud_ret:
         INTR_POP
         pushq   %rax                    /* push temp */

@@ -473,11 +497,11 @@
         movq    32(%rsp), %rax          /* load %rsp */
         movq    (%rax), %rax            /* load calling RIP */
         movq    %rax, 8(%rsp)           /* store calling RIP */
         addq    $8, 32(%rsp)            /* adjust new %rsp */
         popq    %rax                    /* pop off temp */
-        IRET                            /* return from interrupt */
+        jmp     tr_iret_kernel          /* return from interrupt */
         /*NOTREACHED*/
 
 ud_trap:
         /*
          * We're going to let the kernel handle this as a normal #UD.  If,

@@ -747,11 +771,11 @@
 _patch_xrstorq_rbx:
         fxrstorq (%rbx)
         popq    %rdx
         popq    %rbx
         popq    %rax
-        IRET
+        jmp     tr_iret_auto
         /*NOTREACHED*/
 
 .handle_in_trap:
         popq    %rdx
         popq    %rbx

@@ -1125,11 +1149,11 @@
         SET_SIZE(pentium_pftrap)
 
 #endif  /* !__amd64 */
 
         ENTRY_NP(resvtrap)
-        TRAP_NOERR(15)          /* (reserved)  */
+        TRAP_NOERR(T_RESVTRAP)  /* (reserved)  */
         jmp     cmntrap
         SET_SIZE(resvtrap)
 
         /*
          * #MF

@@ -1205,19 +1229,14 @@
         TRAP_NOERR(T_SIMDFPE)   /* $19 */
         jmp     cmninttrap
         SET_SIZE(xmtrap)
 
         ENTRY_NP(invaltrap)
-        TRAP_NOERR(30)          /* very invalid */
+        TRAP_NOERR(T_INVALTRAP) /* very invalid */
         jmp     cmntrap
         SET_SIZE(invaltrap)
 
-        ENTRY_NP(invalint)
-        TRAP_NOERR(31)          /* even more so */
-        jmp     cmnint
-        SET_SIZE(invalint)
-
         .globl  fasttable
 
 #if defined(__amd64)
 
         ENTRY_NP(fasttrap)

@@ -1284,11 +1303,11 @@
          * XXX a constant would be nicer.
          */
         ENTRY_NP(fast_null)
         XPV_TRAP_POP
         orq     $PS_C, 24(%rsp) /* set carry bit in user flags */
-        IRET
+        jmp     tr_iret_auto
         /*NOTREACHED*/
         SET_SIZE(fast_null)
 
 #elif defined(__i386)