Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
9210 remove KMDB branch debugging support
9211 ::crregs could do with cr2/cr3 support
9209 ::ttrace should be able to filter by thread
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Yuri Pankov <yuripv@yuripv.net>
*** 19,146 ****
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
- #pragma ident "%Z%%M% %I% %E% SMI"
-
/*
! * Companion to kdi_idt.c - the implementation of the trap and interrupt
* handlers. For the most part, these handlers do the same thing - they
* push a trap number onto the stack, followed by a jump to kdi_cmnint.
* Each trap and interrupt has its own handler because each one pushes a
* different number.
*/
#include <sys/asm_linkage.h>
#include <sys/kdi_regs.h>
! /* Nothing in this file is of interest to lint. */
! #if !defined(__lint)
/*
! * The default ASM_ENTRY_ALIGN (16) wastes far too much space. Pay no
! * attention to the fleet of nop's we're adding to each handler.
*/
#undef ASM_ENTRY_ALIGN
#define ASM_ENTRY_ALIGN 8
/*
! * We need the .align in ENTRY_NP (defined to be ASM_ENTRY_ALIGN) to match our
! * manual .align (KDI_MSR_PATCHOFF) in order to ensure that the space reserved
! * at the beginning of the handler for code is exactly KDI_MSR_PATCHOFF bytes
! * long. Note that the #error below isn't supported by the preprocessor invoked
! * by as(1), and won't stop the build, but it'll emit a noticeable error message
! * which won't escape the filters.
*/
- #if ASM_ENTRY_ALIGN != KDI_MSR_PATCHOFF
- #error "ASM_ENTRY_ALIGN != KDI_MSR_PATCHOFF"
- this won't assemble
- #endif
! /*
! * kdi_idt_patch will, on certain processors, replace the patch points below
! * with MSR-clearing code. kdi_id_patch has intimate knowledge of the size of
! * the nop hole, as well as the structure of the handlers. Do not change
! * anything here without also changing kdi_idt_patch.
! */
! /*
! * Generic trap and interrupt handlers.
! */
! #if defined(__xpv) && defined(__amd64)
/*
! * The hypervisor places r11 and rcx on the stack.
*/
! #define TRAP_NOERR(trapno) \
! popq %rcx; \
! popq %r11; \
! pushq $trapno
! #define TRAP_ERR(trapno) \
! popq %rcx; \
! popq %r11; \
! pushq $0; \
! pushq $trapno
- #else
- #define TRAP_NOERR(trapno) \
- push $trapno
-
- #define TRAP_ERR(trapno) \
- push $0; \
- push $trapno
-
- #endif /* __xpv && __amd64 */
-
-
#define MKIVCT(n) \
ENTRY_NP(kdi_ivct/**/n/**/); \
! TRAP_ERR(n); \
! .align KDI_MSR_PATCHOFF; \
! KDI_MSR_PATCH; \
jmp kdi_cmnint; \
SET_SIZE(kdi_ivct/**/n/**/)
#define MKTRAPHDLR(n) \
ENTRY_NP(kdi_trap/**/n); \
! TRAP_ERR(n); \
! .align KDI_MSR_PATCHOFF; \
! KDI_MSR_PATCH; \
jmp kdi_cmnint; \
SET_SIZE(kdi_trap/**/n/**/)
#define MKTRAPERRHDLR(n) \
ENTRY_NP(kdi_traperr/**/n); \
! TRAP_NOERR(n); \
! .align KDI_MSR_PATCHOFF; \
! KDI_MSR_PATCH; \
jmp kdi_cmnint; \
SET_SIZE(kdi_traperr/**/n)
#define MKNMIHDLR \
ENTRY_NP(kdi_int2); \
! TRAP_NOERR(2); \
! .align KDI_MSR_PATCHOFF; \
! KDI_MSR_PATCH; \
jmp kdi_nmiint; \
SET_SIZE(kdi_int2)
#define MKINVALHDLR \
ENTRY_NP(kdi_invaltrap); \
! TRAP_NOERR(255); \
! .align KDI_MSR_PATCHOFF; \
! KDI_MSR_PATCH; \
jmp kdi_cmnint; \
SET_SIZE(kdi_invaltrap)
/*
* The handlers themselves
*/
MKINVALHDLR
--- 19,233 ----
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
+ *
+ * Copyright 2018 Joyent, Inc.
*/
/*
! * Companion to kdi_asm.s - the implementation of the trap and interrupt
* handlers. For the most part, these handlers do the same thing - they
* push a trap number onto the stack, followed by a jump to kdi_cmnint.
* Each trap and interrupt has its own handler because each one pushes a
* different number.
*/
+ #if defined(__lint)
+ #include <sys/types.h>
+ #else
+
#include <sys/asm_linkage.h>
+ #include <sys/asm_misc.h>
+ #include <sys/machprivregs.h>
+ #include <sys/privregs.h>
#include <sys/kdi_regs.h>
+ #include <sys/trap.h>
+ #include <sys/param.h>
! #include <kdi_assym.h>
! #include <assym.h>
/*
! * The default ASM_ENTRY_ALIGN (16) wastes far too much space.
*/
#undef ASM_ENTRY_ALIGN
#define ASM_ENTRY_ALIGN 8
/*
! * Generic trap and interrupt handlers.
*/
! #if defined(__xpv)
! #define INTERRUPT_TRAMPOLINE
! #else
/*
! * If we're !xpv, then we will need to support KPTI (kernel page table
! * isolation), where we have separate page tables for user and kernel modes.
! * There's more detail about this in kpti_trampolines.s and hat_i86.c
*/
! #define INTERRUPT_TRAMPOLINE \
! pushq %r13; \
! pushq %r14; \
! subq $KPTI_R14, %rsp; \
! /* Check for clobbering */ \
! cmp $0, KPTI_FLAG(%rsp); \
! je 1f; \
! /* Don't worry, this totally works */ \
! int $8; \
! 1: \
! movq $1, KPTI_FLAG(%rsp); \
! /* Save current %cr3. */ \
! mov %cr3, %r14; \
! mov %r14, KPTI_TR_CR3(%rsp); \
! /* Switch to paranoid %cr3. */ \
! mov kpti_safe_cr3, %r14; \
! mov %r14, %cr3; \
! \
! cmpw $KCS_SEL, KPTI_CS(%rsp); \
! je 3f; \
! 2: \
! /* Get our cpu_t in %r13 */ \
! mov %rsp, %r13; \
! and $(~(MMU_PAGESIZE - 1)), %r13; \
! subq $CPU_KPTI_START, %r13; \
! /* Use top of the kthread stk */ \
! mov CPU_THREAD(%r13), %r14; \
! mov T_STACK(%r14), %r14; \
! addq $REGSIZE+MINFRAME, %r14; \
! jmp 5f; \
! 3: \
! /* Check the %rsp in the frame. */ \
! /* Is it above kernel base? */ \
! mov kpti_kbase, %r14; \
! cmp %r14, KPTI_RSP(%rsp); \
! jb 2b; \
! /* Is it within the kpti_frame page? */ \
! mov %rsp, %r13; \
! and $(~(MMU_PAGESIZE - 1)), %r13; \
! mov KPTI_RSP(%rsp), %r14; \
! and $(~(MMU_PAGESIZE - 1)), %r14; \
! cmp %r13, %r14; \
! je 2b; \
! /* Use the %rsp from the trap frame. */ \
! /* We already did %cr3. */ \
! mov KPTI_RSP(%rsp), %r14; \
! and $(~0xf), %r14; \
! 5: \
! mov %rsp, %r13; \
! /* %r14 contains our destination stk */ \
! mov %r14, %rsp; \
! pushq KPTI_SS(%r13); \
! pushq KPTI_RSP(%r13); \
! pushq KPTI_RFLAGS(%r13); \
! pushq KPTI_CS(%r13); \
! pushq KPTI_RIP(%r13); \
! pushq KPTI_ERR(%r13); \
! mov KPTI_R14(%r13), %r14; \
! movq $0, KPTI_FLAG(%r13); \
! mov KPTI_R13(%r13), %r13
! #endif /* !__xpv */
#define MKIVCT(n) \
ENTRY_NP(kdi_ivct/**/n/**/); \
! XPV_TRAP_POP; \
! push $0; /* err */ \
! INTERRUPT_TRAMPOLINE; \
! push $n; \
jmp kdi_cmnint; \
SET_SIZE(kdi_ivct/**/n/**/)
#define MKTRAPHDLR(n) \
ENTRY_NP(kdi_trap/**/n); \
! XPV_TRAP_POP; \
! push $0; /* err */ \
! INTERRUPT_TRAMPOLINE; \
! push $n; \
jmp kdi_cmnint; \
SET_SIZE(kdi_trap/**/n/**/)
#define MKTRAPERRHDLR(n) \
ENTRY_NP(kdi_traperr/**/n); \
! XPV_TRAP_POP; \
! INTERRUPT_TRAMPOLINE; \
! push $n; \
jmp kdi_cmnint; \
SET_SIZE(kdi_traperr/**/n)
+ #if !defined(__xpv)
#define MKNMIHDLR \
ENTRY_NP(kdi_int2); \
! push $0; \
! push $2; \
! pushq %r13; \
! mov kpti_safe_cr3, %r13; \
! mov %r13, %cr3; \
! popq %r13; \
jmp kdi_nmiint; \
SET_SIZE(kdi_int2)
+ #define MKMCEHDLR \
+ ENTRY_NP(kdi_trap18); \
+ push $0; \
+ push $18; \
+ pushq %r13; \
+ mov kpti_safe_cr3, %r13; \
+ mov %r13, %cr3; \
+ popq %r13; \
+ jmp kdi_cmnint; \
+ SET_SIZE(kdi_trap18)
+ #else
+ #define MKNMIHDLR \
+ ENTRY_NP(kdi_int2); \
+ push $0; \
+ push $2; \
+ jmp kdi_nmiint; \
+ SET_SIZE(kdi_int2)
+
+ #define MKMCEHDLR \
+ ENTRY_NP(kdi_trap18); \
+ push $0; \
+ push $18; \
+ jmp kdi_cmnint; \
+ SET_SIZE(kdi_trap18)
+ #endif
+
+ /*
+ * The only way we should reach here is by an explicit "int 0x.." which is
+ * defined not to push an error code.
+ */
#define MKINVALHDLR \
ENTRY_NP(kdi_invaltrap); \
! XPV_TRAP_POP; \
! push $0; /* err */ \
! INTERRUPT_TRAMPOLINE; \
! push $255; \
jmp kdi_cmnint; \
SET_SIZE(kdi_invaltrap)
+ .data
+ DGDEF3(kdi_idt, 16 * NIDT, MMU_PAGESIZE)
+ .fill MMU_PAGESIZE, 1, 0
+
+ #if !defined(__xpv)
+ .section ".text"
+ .align MMU_PAGESIZE
+ .global kdi_isr_start
+ kdi_isr_start:
+ nop
+
+ .global kpti_safe_cr3
+ .global kpti_kbase
+ #endif
+
/*
* The handlers themselves
*/
MKINVALHDLR
*** 153,173 ****
MKTRAPHDLR(6)
MKTRAPHDLR(7)
MKTRAPHDLR(9)
MKTRAPHDLR(15)
MKTRAPHDLR(16)
! MKTRAPHDLR(17)
! MKTRAPHDLR(18)
MKTRAPHDLR(19)
MKTRAPHDLR(20)
MKTRAPERRHDLR(8)
MKTRAPERRHDLR(10)
MKTRAPERRHDLR(11)
MKTRAPERRHDLR(12)
MKTRAPERRHDLR(13)
MKTRAPERRHDLR(14)
.globl kdi_ivct_size
kdi_ivct_size:
.NWORD [kdi_ivct33-kdi_ivct32]
--- 240,260 ----
MKTRAPHDLR(6)
MKTRAPHDLR(7)
MKTRAPHDLR(9)
MKTRAPHDLR(15)
MKTRAPHDLR(16)
! MKMCEHDLR/*18*/
MKTRAPHDLR(19)
MKTRAPHDLR(20)
MKTRAPERRHDLR(8)
MKTRAPERRHDLR(10)
MKTRAPERRHDLR(11)
MKTRAPERRHDLR(12)
MKTRAPERRHDLR(13)
MKTRAPERRHDLR(14)
+ MKTRAPERRHDLR(17)
.globl kdi_ivct_size
kdi_ivct_size:
.NWORD [kdi_ivct33-kdi_ivct32]
*** 228,233 ****
--- 315,328 ----
MKIVCT(240); MKIVCT(241); MKIVCT(242); MKIVCT(243);
MKIVCT(244); MKIVCT(245); MKIVCT(246); MKIVCT(247);
MKIVCT(248); MKIVCT(249); MKIVCT(250); MKIVCT(251);
MKIVCT(252); MKIVCT(253); MKIVCT(254); MKIVCT(255);
+ #if !defined(__xpv)
+ .section ".text"
+ .align MMU_PAGESIZE
+ .global kdi_isr_end
+ kdi_isr_end:
+ nop
#endif
+
+ #endif /* !__lint */