Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
9210 remove KMDB branch debugging support
9211 ::crregs could do with cr2/cr3 support
9209 ::ttrace should be able to filter by thread
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Yuri Pankov <yuripv@yuripv.net>
@@ -19,128 +19,215 @@
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
+ *
+ * Copyright 2018 Joyent, Inc.
*/
-#pragma ident "%Z%%M% %I% %E% SMI"
-
/*
- * Companion to kdi_idt.c - the implementation of the trap and interrupt
+ * Companion to kdi_asm.s - the implementation of the trap and interrupt
* handlers. For the most part, these handlers do the same thing - they
* push a trap number onto the stack, followed by a jump to kdi_cmnint.
* Each trap and interrupt has its own handler because each one pushes a
* different number.
*/
+#if defined(__lint)
+#include <sys/types.h>
+#else
+
#include <sys/asm_linkage.h>
+#include <sys/asm_misc.h>
+#include <sys/machprivregs.h>
+#include <sys/privregs.h>
#include <sys/kdi_regs.h>
+#include <sys/trap.h>
+#include <sys/param.h>
-/* Nothing in this file is of interest to lint. */
-#if !defined(__lint)
+#include <kdi_assym.h>
+#include <assym.h>
/*
- * The default ASM_ENTRY_ALIGN (16) wastes far too much space. Pay no
- * attention to the fleet of nop's we're adding to each handler.
+ * The default ASM_ENTRY_ALIGN (16) wastes far too much space.
*/
#undef ASM_ENTRY_ALIGN
#define ASM_ENTRY_ALIGN 8
/*
- * We need the .align in ENTRY_NP (defined to be ASM_ENTRY_ALIGN) to match our
- * manual .align (KDI_MSR_PATCHOFF) in order to ensure that the space reserved
- * at the beginning of the handler for code is exactly KDI_MSR_PATCHOFF bytes
- * long. Note that the #error below isn't supported by the preprocessor invoked
- * by as(1), and won't stop the build, but it'll emit a noticeable error message
- * which won't escape the filters.
+ * Generic trap and interrupt handlers.
*/
-#if ASM_ENTRY_ALIGN != KDI_MSR_PATCHOFF
-#error "ASM_ENTRY_ALIGN != KDI_MSR_PATCHOFF"
-this won't assemble
-#endif
-/*
- * kdi_idt_patch will, on certain processors, replace the patch points below
- * with MSR-clearing code. kdi_id_patch has intimate knowledge of the size of
- * the nop hole, as well as the structure of the handlers. Do not change
- * anything here without also changing kdi_idt_patch.
- */
+#if defined(__xpv)
-/*
- * Generic trap and interrupt handlers.
- */
+#define INTERRUPT_TRAMPOLINE
-#if defined(__xpv) && defined(__amd64)
+#else
/*
- * The hypervisor places r11 and rcx on the stack.
+ * If we're !xpv, then we will need to support KPTI (kernel page table
+ * isolation), where we have separate page tables for user and kernel modes.
+ * There's more detail about this in kpti_trampolines.s and hat_i86.c
*/
-#define TRAP_NOERR(trapno) \
- popq %rcx; \
- popq %r11; \
- pushq $trapno
+#define INTERRUPT_TRAMPOLINE \
+ pushq %r13; \
+ pushq %r14; \
+ subq $KPTI_R14, %rsp; \
+ /* Check for clobbering */ \
+ cmp $0, KPTI_FLAG(%rsp); \
+ je 1f; \
+ /* Don't worry, this totally works */ \
+ int $8; \
+1: \
+ movq $1, KPTI_FLAG(%rsp); \
+ /* Save current %cr3. */ \
+ mov %cr3, %r14; \
+ mov %r14, KPTI_TR_CR3(%rsp); \
+ /* Switch to paranoid %cr3. */ \
+ mov kpti_safe_cr3, %r14; \
+ mov %r14, %cr3; \
+ \
+ cmpw $KCS_SEL, KPTI_CS(%rsp); \
+ je 3f; \
+2: \
+ /* Get our cpu_t in %r13 */ \
+ mov %rsp, %r13; \
+ and $(~(MMU_PAGESIZE - 1)), %r13; \
+ subq $CPU_KPTI_START, %r13; \
+ /* Use top of the kthread stk */ \
+ mov CPU_THREAD(%r13), %r14; \
+ mov T_STACK(%r14), %r14; \
+ addq $REGSIZE+MINFRAME, %r14; \
+ jmp 5f; \
+3: \
+ /* Check the %rsp in the frame. */ \
+ /* Is it above kernel base? */ \
+ mov kpti_kbase, %r14; \
+ cmp %r14, KPTI_RSP(%rsp); \
+ jb 2b; \
+ /* Is it within the kpti_frame page? */ \
+ mov %rsp, %r13; \
+ and $(~(MMU_PAGESIZE - 1)), %r13; \
+ mov KPTI_RSP(%rsp), %r14; \
+ and $(~(MMU_PAGESIZE - 1)), %r14; \
+ cmp %r13, %r14; \
+ je 2b; \
+ /* Use the %rsp from the trap frame. */ \
+ /* We already did %cr3. */ \
+ mov KPTI_RSP(%rsp), %r14; \
+ and $(~0xf), %r14; \
+5: \
+ mov %rsp, %r13; \
+ /* %r14 contains our destination stk */ \
+ mov %r14, %rsp; \
+ pushq KPTI_SS(%r13); \
+ pushq KPTI_RSP(%r13); \
+ pushq KPTI_RFLAGS(%r13); \
+ pushq KPTI_CS(%r13); \
+ pushq KPTI_RIP(%r13); \
+ pushq KPTI_ERR(%r13); \
+ mov KPTI_R14(%r13), %r14; \
+ movq $0, KPTI_FLAG(%r13); \
+ mov KPTI_R13(%r13), %r13
-#define TRAP_ERR(trapno) \
- popq %rcx; \
- popq %r11; \
- pushq $0; \
- pushq $trapno
+#endif /* !__xpv */
-#else
-#define TRAP_NOERR(trapno) \
- push $trapno
-
-#define TRAP_ERR(trapno) \
- push $0; \
- push $trapno
-
-#endif /* __xpv && __amd64 */
-
-
#define MKIVCT(n) \
ENTRY_NP(kdi_ivct/**/n/**/); \
- TRAP_ERR(n); \
- .align KDI_MSR_PATCHOFF; \
- KDI_MSR_PATCH; \
+ XPV_TRAP_POP; \
+ push $0; /* err */ \
+ INTERRUPT_TRAMPOLINE; \
+ push $n; \
jmp kdi_cmnint; \
SET_SIZE(kdi_ivct/**/n/**/)
#define MKTRAPHDLR(n) \
ENTRY_NP(kdi_trap/**/n); \
- TRAP_ERR(n); \
- .align KDI_MSR_PATCHOFF; \
- KDI_MSR_PATCH; \
+ XPV_TRAP_POP; \
+ push $0; /* err */ \
+ INTERRUPT_TRAMPOLINE; \
+ push $n; \
jmp kdi_cmnint; \
SET_SIZE(kdi_trap/**/n/**/)
#define MKTRAPERRHDLR(n) \
ENTRY_NP(kdi_traperr/**/n); \
- TRAP_NOERR(n); \
- .align KDI_MSR_PATCHOFF; \
- KDI_MSR_PATCH; \
+ XPV_TRAP_POP; \
+ INTERRUPT_TRAMPOLINE; \
+ push $n; \
jmp kdi_cmnint; \
SET_SIZE(kdi_traperr/**/n)
+#if !defined(__xpv)
#define MKNMIHDLR \
ENTRY_NP(kdi_int2); \
- TRAP_NOERR(2); \
- .align KDI_MSR_PATCHOFF; \
- KDI_MSR_PATCH; \
+ push $0; \
+ push $2; \
+ pushq %r13; \
+ mov kpti_safe_cr3, %r13; \
+ mov %r13, %cr3; \
+ popq %r13; \
jmp kdi_nmiint; \
SET_SIZE(kdi_int2)
+#define MKMCEHDLR \
+ ENTRY_NP(kdi_trap18); \
+ push $0; \
+ push $18; \
+ pushq %r13; \
+ mov kpti_safe_cr3, %r13; \
+ mov %r13, %cr3; \
+ popq %r13; \
+ jmp kdi_cmnint; \
+ SET_SIZE(kdi_trap18)
+#else
+#define MKNMIHDLR \
+ ENTRY_NP(kdi_int2); \
+ push $0; \
+ push $2; \
+ jmp kdi_nmiint; \
+ SET_SIZE(kdi_int2)
+
+#define MKMCEHDLR \
+ ENTRY_NP(kdi_trap18); \
+ push $0; \
+ push $18; \
+ jmp kdi_cmnint; \
+ SET_SIZE(kdi_trap18)
+#endif
+
+/*
+ * The only way we should reach here is by an explicit "int 0x.." which is
+ * defined not to push an error code.
+ */
#define MKINVALHDLR \
ENTRY_NP(kdi_invaltrap); \
- TRAP_NOERR(255); \
- .align KDI_MSR_PATCHOFF; \
- KDI_MSR_PATCH; \
+ XPV_TRAP_POP; \
+ push $0; /* err */ \
+ INTERRUPT_TRAMPOLINE; \
+ push $255; \
jmp kdi_cmnint; \
SET_SIZE(kdi_invaltrap)
+ .data
+ DGDEF3(kdi_idt, 16 * NIDT, MMU_PAGESIZE)
+ .fill MMU_PAGESIZE, 1, 0
+
+#if !defined(__xpv)
+.section ".text"
+.align MMU_PAGESIZE
+.global kdi_isr_start
+kdi_isr_start:
+ nop
+
+.global kpti_safe_cr3
+.global kpti_kbase
+#endif
+
/*
* The handlers themselves
*/
MKINVALHDLR
@@ -153,21 +240,21 @@
MKTRAPHDLR(6)
MKTRAPHDLR(7)
MKTRAPHDLR(9)
MKTRAPHDLR(15)
MKTRAPHDLR(16)
- MKTRAPHDLR(17)
- MKTRAPHDLR(18)
+ MKMCEHDLR/*18*/
MKTRAPHDLR(19)
MKTRAPHDLR(20)
MKTRAPERRHDLR(8)
MKTRAPERRHDLR(10)
MKTRAPERRHDLR(11)
MKTRAPERRHDLR(12)
MKTRAPERRHDLR(13)
MKTRAPERRHDLR(14)
+ MKTRAPERRHDLR(17)
.globl kdi_ivct_size
kdi_ivct_size:
.NWORD [kdi_ivct33-kdi_ivct32]
@@ -228,6 +315,14 @@
MKIVCT(240); MKIVCT(241); MKIVCT(242); MKIVCT(243);
MKIVCT(244); MKIVCT(245); MKIVCT(246); MKIVCT(247);
MKIVCT(248); MKIVCT(249); MKIVCT(250); MKIVCT(251);
MKIVCT(252); MKIVCT(253); MKIVCT(254); MKIVCT(255);
+#if !defined(__xpv)
+.section ".text"
+.align MMU_PAGESIZE
+.global kdi_isr_end
+kdi_isr_end:
+ nop
#endif
+
+#endif /* !__lint */