Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
*** 22,32 ****
/*
* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
! * Copyright 2011 Joyent, Inc. All rights reserved.
*/
/*
* Copyright (c) 1992 Terrence R. Lambert.
* Copyright (c) 1990 The Regents of the University of California.
--- 22,32 ----
/*
* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
! * Copyright 2018 Joyent, Inc. All rights reserved.
*/
/*
* Copyright (c) 1992 Terrence R. Lambert.
* Copyright (c) 1990 The Regents of the University of California.
*** 81,90 ****
--- 81,91 ----
#include <sys/cmn_err.h>
#include <sys/reboot.h>
#include <sys/kdi.h>
#include <sys/mach_mmu.h>
#include <sys/systm.h>
+ #include <sys/note.h>
#ifdef __xpv
#include <sys/hypervisor.h>
#include <vm/as.h>
#endif
*** 126,137 ****
user_desc_t ucs_off;
user_desc_t ucs32_on;
user_desc_t ucs32_off;
#endif /* __amd64 */
! #pragma align 16(dblfault_stack0)
! char dblfault_stack0[DEFAULTSTKSZ];
extern void fast_null(void);
extern hrtime_t get_hrtime(void);
extern hrtime_t gethrvtime(void);
extern hrtime_t get_hrestime(void);
--- 127,143 ----
user_desc_t ucs_off;
user_desc_t ucs32_on;
user_desc_t ucs32_off;
#endif /* __amd64 */
! /*
! * If the size of this is changed, you must update hat_pcp_setup() and the
! * definitions in exception.s
! */
! extern char dblfault_stack0[DEFAULTSTKSZ];
! extern char nmi_stack0[DEFAULTSTKSZ];
! extern char mce_stack0[DEFAULTSTKSZ];
extern void fast_null(void);
extern hrtime_t get_hrtime(void);
extern hrtime_t gethrvtime(void);
extern hrtime_t get_hrestime(void);
*** 308,368 ****
#endif /* __i386 */
/*
* Install gate segment descriptor for interrupt, trap, call and task gates.
*/
!
! #if defined(__amd64)
!
! /*ARGSUSED*/
! void
! set_gatesegd(gate_desc_t *dp, void (*func)(void), selector_t sel,
! uint_t type, uint_t dpl, uint_t vector)
{
! dp->sgd_looffset = (uintptr_t)func;
! dp->sgd_hioffset = (uintptr_t)func >> 16;
! dp->sgd_hi64offset = (uintptr_t)func >> (16 + 16);
! dp->sgd_selector = (uint16_t)sel;
!
! /*
! * For 64 bit native we use the IST stack mechanism
! * for double faults. All other traps use the CPL = 0
! * (tss_rsp0) stack.
! */
! #if !defined(__xpv)
! if (vector == T_DBLFLT)
! dp->sgd_ist = 1;
! else
#endif
- dp->sgd_ist = 0;
-
- dp->sgd_type = type;
- dp->sgd_dpl = dpl;
- dp->sgd_p = 1;
}
- #elif defined(__i386)
-
- /*ARGSUSED*/
void
set_gatesegd(gate_desc_t *dp, void (*func)(void), selector_t sel,
! uint_t type, uint_t dpl, uint_t unused)
{
dp->sgd_looffset = (uintptr_t)func;
dp->sgd_hioffset = (uintptr_t)func >> 16;
!
dp->sgd_selector = (uint16_t)sel;
! dp->sgd_stkcpy = 0; /* always zero bytes */
dp->sgd_type = type;
dp->sgd_dpl = dpl;
dp->sgd_p = 1;
}
- #endif /* __i386 */
-
/*
* Updates a single user descriptor in the the GDT of the current cpu.
* Caller is responsible for preventing cpu migration.
*/
--- 314,390 ----
#endif /* __i386 */
/*
* Install gate segment descriptor for interrupt, trap, call and task gates.
+ *
+ * For 64 bit native if we have KPTI enabled, we use the IST stack mechanism on
+ * all interrupts. We have different ISTs for each class of exceptions that are
+ * most likely to occur while handling an existing exception; while many of
+ * these are just going to panic, it's nice not to trample on the existing
+ * exception state for debugging purposes.
+ *
+ * Normal interrupts are all redirected unconditionally to the KPTI trampoline
+ * stack space. This unifies the trampoline handling between user and kernel
+ * space (and avoids the need to touch %gs).
+ *
+ * The KDI IDT *all* uses the DBG IST: consider single stepping tr_pftrap, when
+ * we do a read from KMDB that cause another #PF. Without its own IST, this
+ * would stomp on the kernel's mcpu_kpti_flt frame.
*/
! uint_t
! idt_vector_to_ist(uint_t vector)
{
! #if defined(__xpv)
! _NOTE(ARGUNUSED(vector));
! return (IST_NONE);
! #else
! switch (vector) {
! /* These should always use IST even without KPTI enabled. */
! case T_DBLFLT:
! return (IST_DF);
! case T_NMIFLT:
! return (IST_NMI);
! case T_MCE:
! return (IST_MCE);
! case T_BPTFLT:
! case T_SGLSTP:
! if (kpti_enable == 1) {
! return (IST_DBG);
! }
! return (IST_NONE);
! case T_STKFLT:
! case T_GPFLT:
! case T_PGFLT:
! if (kpti_enable == 1) {
! return (IST_NESTABLE);
! }
! return (IST_NONE);
! default:
! if (kpti_enable == 1) {
! return (IST_DEFAULT);
! }
! return (IST_NONE);
! }
#endif
}
void
set_gatesegd(gate_desc_t *dp, void (*func)(void), selector_t sel,
! uint_t type, uint_t dpl, uint_t ist)
{
dp->sgd_looffset = (uintptr_t)func;
dp->sgd_hioffset = (uintptr_t)func >> 16;
! dp->sgd_hi64offset = (uintptr_t)func >> (16 + 16);
dp->sgd_selector = (uint16_t)sel;
! dp->sgd_ist = ist;
dp->sgd_type = type;
dp->sgd_dpl = dpl;
dp->sgd_p = 1;
}
/*
* Updates a single user descriptor in the the GDT of the current cpu.
* Caller is responsible for preventing cpu migration.
*/
*** 915,1021 ****
* Perhaps they should have invented a trap gate that does an atomic swapgs?
*/
static void
init_idt_common(gate_desc_t *idt)
{
! set_gatesegd(&idt[T_ZERODIV], &div0trap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
! set_gatesegd(&idt[T_SGLSTP], &dbgtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
! set_gatesegd(&idt[T_NMIFLT], &nmiint, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
! set_gatesegd(&idt[T_BPTFLT], &brktrap, KCS_SEL, SDT_SYSIGT, TRP_UPL,
! 0);
! set_gatesegd(&idt[T_OVFLW], &ovflotrap, KCS_SEL, SDT_SYSIGT, TRP_UPL,
! 0);
! set_gatesegd(&idt[T_BOUNDFLT], &boundstrap, KCS_SEL, SDT_SYSIGT,
! TRP_KPL, 0);
! set_gatesegd(&idt[T_ILLINST], &invoptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
! set_gatesegd(&idt[T_NOEXTFLT], &ndptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
/*
* double fault handler.
*
* Note that on the hypervisor a guest does not receive #df faults.
* Instead a failsafe event is injected into the guest if its selectors
* and/or stack is in a broken state. See xen_failsafe_callback.
*/
#if !defined(__xpv)
! #if defined(__amd64)
!
! set_gatesegd(&idt[T_DBLFLT], &syserrtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! T_DBLFLT);
!
! #elif defined(__i386)
!
! /*
! * task gate required.
! */
! set_gatesegd(&idt[T_DBLFLT], NULL, DFTSS_SEL, SDT_SYSTASKGT, TRP_KPL,
! 0);
!
! #endif /* __i386 */
#endif /* !__xpv */
/*
* T_EXTOVRFLT coprocessor-segment-overrun not supported.
*/
- set_gatesegd(&idt[T_TSSFLT], &invtsstrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
- 0);
- set_gatesegd(&idt[T_SEGFLT], &segnptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
- 0);
- set_gatesegd(&idt[T_STKFLT], &stktrap, KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
- set_gatesegd(&idt[T_GPFLT], &gptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
- set_gatesegd(&idt[T_PGFLT], &pftrap, KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
- set_gatesegd(&idt[T_EXTERRFLT], &ndperr, KCS_SEL, SDT_SYSIGT, TRP_KPL,
- 0);
- set_gatesegd(&idt[T_ALIGNMENT], &achktrap, KCS_SEL, SDT_SYSIGT,
- TRP_KPL, 0);
- set_gatesegd(&idt[T_MCE], &mcetrap, KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
- set_gatesegd(&idt[T_SIMDFPE], &xmtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
-
/*
* install fast trap handler at 210.
*/
! set_gatesegd(&idt[T_FASTTRAP], &fasttrap, KCS_SEL, SDT_SYSIGT, TRP_UPL,
! 0);
/*
* System call handler.
*/
! #if defined(__amd64)
! set_gatesegd(&idt[T_SYSCALLINT], &sys_syscall_int, KCS_SEL, SDT_SYSIGT,
! TRP_UPL, 0);
- #elif defined(__i386)
- set_gatesegd(&idt[T_SYSCALLINT], &sys_call, KCS_SEL, SDT_SYSIGT,
- TRP_UPL, 0);
- #endif /* __i386 */
-
/*
* Install the DTrace interrupt handler for the pid provider.
*/
! set_gatesegd(&idt[T_DTRACE_RET], &dtrace_ret, KCS_SEL,
! SDT_SYSIGT, TRP_UPL, 0);
/*
* Prepare interposing descriptor for the syscall handler
* and cache copy of the default descriptor.
*/
brand_tbl[0].ih_inum = T_SYSCALLINT;
brand_tbl[0].ih_default_desc = idt0[T_SYSCALLINT];
! #if defined(__amd64)
! set_gatesegd(&(brand_tbl[0].ih_interp_desc), &brand_sys_syscall_int,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, 0);
! #elif defined(__i386)
! set_gatesegd(&(brand_tbl[0].ih_interp_desc), &brand_sys_call,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, 0);
! #endif /* __i386 */
brand_tbl[1].ih_inum = 0;
}
#if defined(__xpv)
--- 937,1047 ----
* Perhaps they should have invented a trap gate that does an atomic swapgs?
*/
static void
init_idt_common(gate_desc_t *idt)
{
! set_gatesegd(&idt[T_ZERODIV],
! (kpti_enable == 1) ? &tr_div0trap : &div0trap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_ZERODIV));
! set_gatesegd(&idt[T_SGLSTP],
! (kpti_enable == 1) ? &tr_dbgtrap : &dbgtrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_SGLSTP));
! set_gatesegd(&idt[T_NMIFLT],
! (kpti_enable == 1) ? &tr_nmiint : &nmiint,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_NMIFLT));
! set_gatesegd(&idt[T_BPTFLT],
! (kpti_enable == 1) ? &tr_brktrap : &brktrap,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_BPTFLT));
! set_gatesegd(&idt[T_OVFLW],
! (kpti_enable == 1) ? &tr_ovflotrap : &ovflotrap,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_OVFLW));
! set_gatesegd(&idt[T_BOUNDFLT],
! (kpti_enable == 1) ? &tr_boundstrap : &boundstrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_BOUNDFLT));
! set_gatesegd(&idt[T_ILLINST],
! (kpti_enable == 1) ? &tr_invoptrap : &invoptrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_ILLINST));
! set_gatesegd(&idt[T_NOEXTFLT],
! (kpti_enable == 1) ? &tr_ndptrap : &ndptrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_NOEXTFLT));
/*
* double fault handler.
*
* Note that on the hypervisor a guest does not receive #df faults.
* Instead a failsafe event is injected into the guest if its selectors
* and/or stack is in a broken state. See xen_failsafe_callback.
*/
#if !defined(__xpv)
! set_gatesegd(&idt[T_DBLFLT],
! (kpti_enable == 1) ? &tr_syserrtrap : &syserrtrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_DBLFLT));
#endif /* !__xpv */
/*
* T_EXTOVRFLT coprocessor-segment-overrun not supported.
*/
+ set_gatesegd(&idt[T_TSSFLT],
+ (kpti_enable == 1) ? &tr_invtsstrap : &invtsstrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_TSSFLT));
+ set_gatesegd(&idt[T_SEGFLT],
+ (kpti_enable == 1) ? &tr_segnptrap : &segnptrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_SEGFLT));
+ set_gatesegd(&idt[T_STKFLT],
+ (kpti_enable == 1) ? &tr_stktrap : &stktrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_STKFLT));
+ set_gatesegd(&idt[T_GPFLT],
+ (kpti_enable == 1) ? &tr_gptrap : &gptrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_GPFLT));
+ set_gatesegd(&idt[T_PGFLT],
+ (kpti_enable == 1) ? &tr_pftrap : &pftrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_PGFLT));
+ set_gatesegd(&idt[T_EXTERRFLT],
+ (kpti_enable == 1) ? &tr_ndperr : &ndperr,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_EXTERRFLT));
+ set_gatesegd(&idt[T_ALIGNMENT],
+ (kpti_enable == 1) ? &tr_achktrap : &achktrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_ALIGNMENT));
+ set_gatesegd(&idt[T_MCE],
+ (kpti_enable == 1) ? &tr_mcetrap : &mcetrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_MCE));
+ set_gatesegd(&idt[T_SIMDFPE],
+ (kpti_enable == 1) ? &tr_xmtrap : &xmtrap,
+ KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_SIMDFPE));
/*
* install fast trap handler at 210.
*/
! set_gatesegd(&idt[T_FASTTRAP],
! (kpti_enable == 1) ? &tr_fasttrap : &fasttrap,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_FASTTRAP));
/*
* System call handler.
*/
! set_gatesegd(&idt[T_SYSCALLINT],
! (kpti_enable == 1) ? &tr_sys_syscall_int : &sys_syscall_int,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_SYSCALLINT));
/*
* Install the DTrace interrupt handler for the pid provider.
*/
! set_gatesegd(&idt[T_DTRACE_RET],
! (kpti_enable == 1) ? &tr_dtrace_ret : &dtrace_ret,
! KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_DTRACE_RET));
/*
* Prepare interposing descriptor for the syscall handler
* and cache copy of the default descriptor.
*/
brand_tbl[0].ih_inum = T_SYSCALLINT;
brand_tbl[0].ih_default_desc = idt0[T_SYSCALLINT];
! set_gatesegd(&(brand_tbl[0].ih_interp_desc),
! (kpti_enable == 1) ? &tr_brand_sys_syscall_int :
! &brand_sys_syscall_int, KCS_SEL, SDT_SYSIGT, TRP_UPL,
! idt_vector_to_ist(T_SYSCALLINT));
brand_tbl[1].ih_inum = 0;
}
#if defined(__xpv)
*** 1039,1069 ****
* Initialize entire table with 'reserved' trap and then overwrite
* specific entries. T_EXTOVRFLT (9) is unsupported and reserved
* since it can only be generated on a 386 processor. 15 is also
* unsupported and reserved.
*/
! for (i = 0; i < NIDT; i++)
set_gatesegd(&idt[i], &resvtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
/*
* 20-31 reserved
*/
! for (i = 20; i < 32; i++)
set_gatesegd(&idt[i], &invaltrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! 0);
/*
* interrupts 32 - 255
*/
for (i = 32; i < 256; i++) {
(void) snprintf(ivctname, sizeof (ivctname), "ivct%d", i);
ivctptr = (void (*)(void))kobj_getsymvalue(ivctname, 0);
if (ivctptr == NULL)
panic("kobj_getsymvalue(%s) failed", ivctname);
! set_gatesegd(&idt[i], ivctptr, KCS_SEL, SDT_SYSIGT, TRP_KPL, 0);
}
/*
* Now install the common ones. Note that it will overlay some
* entries installed above like T_SYSCALLINT, T_FASTTRAP etc.
--- 1065,1121 ----
* Initialize entire table with 'reserved' trap and then overwrite
* specific entries. T_EXTOVRFLT (9) is unsupported and reserved
* since it can only be generated on a 386 processor. 15 is also
* unsupported and reserved.
*/
! #if !defined(__xpv)
! for (i = 0; i < NIDT; i++) {
! set_gatesegd(&idt[i],
! (kpti_enable == 1) ? &tr_resvtrap : &resvtrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL,
! idt_vector_to_ist(T_RESVTRAP));
! }
! #else
! for (i = 0; i < NIDT; i++) {
set_gatesegd(&idt[i], &resvtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! IST_NONE);
! }
! #endif
/*
* 20-31 reserved
*/
! #if !defined(__xpv)
! for (i = 20; i < 32; i++) {
! set_gatesegd(&idt[i],
! (kpti_enable == 1) ? &tr_invaltrap : &invaltrap,
! KCS_SEL, SDT_SYSIGT, TRP_KPL,
! idt_vector_to_ist(T_INVALTRAP));
! }
! #else
! for (i = 20; i < 32; i++) {
set_gatesegd(&idt[i], &invaltrap, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! IST_NONE);
! }
! #endif
/*
* interrupts 32 - 255
*/
for (i = 32; i < 256; i++) {
+ #if !defined(__xpv)
+ (void) snprintf(ivctname, sizeof (ivctname),
+ (kpti_enable == 1) ? "tr_ivct%d" : "ivct%d", i);
+ #else
(void) snprintf(ivctname, sizeof (ivctname), "ivct%d", i);
+ #endif
ivctptr = (void (*)(void))kobj_getsymvalue(ivctname, 0);
if (ivctptr == NULL)
panic("kobj_getsymvalue(%s) failed", ivctname);
! set_gatesegd(&idt[i], ivctptr, KCS_SEL, SDT_SYSIGT, TRP_KPL,
! idt_vector_to_ist(i));
}
/*
* Now install the common ones. Note that it will overlay some
* entries installed above like T_SYSCALLINT, T_FASTTRAP etc.
*** 1088,1158 ****
wr_ldtr(0);
#endif
}
#if !defined(__xpv)
- #if defined(__amd64)
static void
init_tss(void)
{
! /*
! * tss_rsp0 is dynamically filled in by resume() on each context switch.
! * All exceptions but #DF will run on the thread stack.
! * Set up the double fault stack here.
! */
! ktss0->tss_ist1 =
! (uint64_t)&dblfault_stack0[sizeof (dblfault_stack0)];
/*
! * Set I/O bit map offset equal to size of TSS segment limit
! * for no I/O permission map. This will force all user I/O
! * instructions to generate #gp fault.
*/
! ktss0->tss_bitmapbase = sizeof (*ktss0);
! /*
! * Point %tr to descriptor for ktss0 in gdt.
! */
! wr_tsr(KTSS_SEL);
! }
- #elif defined(__i386)
-
- static void
- init_tss(void)
- {
/*
! * ktss0->tss_esp dynamically filled in by resume() on each
! * context switch.
*/
! ktss0->tss_ss0 = KDS_SEL;
! ktss0->tss_eip = (uint32_t)_start;
! ktss0->tss_ds = ktss0->tss_es = ktss0->tss_ss = KDS_SEL;
! ktss0->tss_cs = KCS_SEL;
! ktss0->tss_fs = KFS_SEL;
! ktss0->tss_gs = KGS_SEL;
! ktss0->tss_ldt = ULDT_SEL;
! /*
! * Initialize double fault tss.
! */
! dftss0->tss_esp0 = (uint32_t)&dblfault_stack0[sizeof (dblfault_stack0)];
! dftss0->tss_ss0 = KDS_SEL;
! /*
! * tss_cr3 will get initialized in hat_kern_setup() once our page
! * tables have been setup.
! */
! dftss0->tss_eip = (uint32_t)syserrtrap;
! dftss0->tss_esp = (uint32_t)&dblfault_stack0[sizeof (dblfault_stack0)];
! dftss0->tss_cs = KCS_SEL;
! dftss0->tss_ds = KDS_SEL;
! dftss0->tss_es = KDS_SEL;
! dftss0->tss_ss = KDS_SEL;
! dftss0->tss_fs = KFS_SEL;
! dftss0->tss_gs = KGS_SEL;
/*
* Set I/O bit map offset equal to size of TSS segment limit
* for no I/O permission map. This will force all user I/O
* instructions to generate #gp fault.
--- 1140,1182 ----
wr_ldtr(0);
#endif
}
#if !defined(__xpv)
static void
init_tss(void)
{
! extern struct cpu cpus[];
/*
! * tss_rsp0 is dynamically filled in by resume() (in swtch.s) on each
! * context switch but it'll be overwritten with this same value anyway.
*/
! if (kpti_enable == 1) {
! ktss0->tss_rsp0 = (uint64_t)&cpus->cpu_m.mcpu_kpti.kf_tr_rsp;
! }
! /* Set up the IST stacks for double fault, NMI, MCE. */
! ktss0->tss_ist1 = (uintptr_t)&dblfault_stack0[sizeof (dblfault_stack0)];
! ktss0->tss_ist2 = (uintptr_t)&nmi_stack0[sizeof (nmi_stack0)];
! ktss0->tss_ist3 = (uintptr_t)&mce_stack0[sizeof (mce_stack0)];
/*
! * This IST stack is used for #DB,#BP (debug) interrupts (when KPTI is
! * enabled), and also for KDI (always).
*/
! ktss0->tss_ist4 = (uint64_t)&cpus->cpu_m.mcpu_kpti_dbg.kf_tr_rsp;
! if (kpti_enable == 1) {
! /* This IST stack is used for #GP,#PF,#SS (fault) interrupts. */
! ktss0->tss_ist5 =
! (uint64_t)&cpus->cpu_m.mcpu_kpti_flt.kf_tr_rsp;
! /* This IST stack is used for all other intrs (for KPTI). */
! ktss0->tss_ist6 = (uint64_t)&cpus->cpu_m.mcpu_kpti.kf_tr_rsp;
! }
/*
* Set I/O bit map offset equal to size of TSS segment limit
* for no I/O permission map. This will force all user I/O
* instructions to generate #gp fault.
*** 1163,1173 ****
* Point %tr to descriptor for ktss0 in gdt.
*/
wr_tsr(KTSS_SEL);
}
- #endif /* __i386 */
#endif /* !__xpv */
#if defined(__xpv)
void
--- 1187,1196 ----
*** 1255,1264 ****
--- 1278,1295 ----
gdt = init_gdt();
ASSERT(IS_P2ALIGNED((uintptr_t)gdt, PAGESIZE));
CPU->cpu_gdt = gdt;
/*
+ * Initialize this CPU's LDT.
+ */
+ CPU->cpu_m.mcpu_ldt = BOP_ALLOC(bootops, (caddr_t)LDT_VA,
+ LDT_CPU_SIZE, PAGESIZE);
+ bzero(CPU->cpu_m.mcpu_ldt, LDT_CPU_SIZE);
+ CPU->cpu_m.mcpu_ldt_len = 0;
+
+ /*
* Setup and install our IDT.
*/
init_idt(idt0);
idtr.dtr_base = (uintptr_t)idt0;
*** 1275,1284 ****
--- 1306,1318 ----
#endif /* __i386 */
init_tss();
CPU->cpu_tss = ktss0;
init_ldt();
+
+ /* Stash this so that the NMI,MCE,#DF and KDI handlers can use it. */
+ kpti_safe_cr3 = (uint64_t)getcr3();
}
#endif /* __xpv */
/*
*** 1335,1353 ****
CALLBACKF_mask_events);
#else
if (is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
wrmsr(MSR_AMD_LSTAR, (uintptr_t)brand_sys_syscall);
wrmsr(MSR_AMD_CSTAR, (uintptr_t)brand_sys_syscall32);
}
#endif
#endif /* __amd64 */
! if (is_x86_feature(x86_featureset, X86FSET_SEP))
wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)brand_sys_sysenter);
}
/*
* Disable interpositioning on the system call path by rewriting the
* sys{call|enter} MSRs and the syscall-related entries in the IDT to use
--- 1369,1398 ----
CALLBACKF_mask_events);
#else
if (is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
+ if (kpti_enable == 1) {
+ wrmsr(MSR_AMD_LSTAR, (uintptr_t)tr_brand_sys_syscall);
+ wrmsr(MSR_AMD_CSTAR, (uintptr_t)tr_brand_sys_syscall32);
+ } else {
wrmsr(MSR_AMD_LSTAR, (uintptr_t)brand_sys_syscall);
wrmsr(MSR_AMD_CSTAR, (uintptr_t)brand_sys_syscall32);
}
+ }
#endif
#endif /* __amd64 */
! if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
! if (kpti_enable == 1) {
! wrmsr(MSR_INTC_SEP_EIP,
! (uintptr_t)tr_brand_sys_sysenter);
! } else {
wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)brand_sys_sysenter);
+ }
+ }
}
/*
* Disable interpositioning on the system call path by rewriting the
* sys{call|enter} MSRs and the syscall-related entries in the IDT to use
*** 1379,1393 ****
CALLBACKF_mask_events);
#else
if (is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
wrmsr(MSR_AMD_LSTAR, (uintptr_t)sys_syscall);
wrmsr(MSR_AMD_CSTAR, (uintptr_t)sys_syscall32);
}
#endif
#endif /* __amd64 */
! if (is_x86_feature(x86_featureset, X86FSET_SEP))
wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)sys_sysenter);
}
--- 1424,1448 ----
CALLBACKF_mask_events);
#else
if (is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
+ if (kpti_enable == 1) {
+ wrmsr(MSR_AMD_LSTAR, (uintptr_t)tr_sys_syscall);
+ wrmsr(MSR_AMD_CSTAR, (uintptr_t)tr_sys_syscall32);
+ } else {
wrmsr(MSR_AMD_LSTAR, (uintptr_t)sys_syscall);
wrmsr(MSR_AMD_CSTAR, (uintptr_t)sys_syscall32);
}
+ }
#endif
#endif /* __amd64 */
! if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
! if (kpti_enable == 1) {
! wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)tr_sys_sysenter);
! } else {
wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)sys_sysenter);
+ }
+ }
}