Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>

*** 21,31 **** /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* ! * Copyright 2011 Joyent, Inc. All rights reserved. */ #ifndef _SYS_MACHCPUVAR_H #define _SYS_MACHCPUVAR_H --- 21,31 ---- /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* ! * Copyright 2018 Joyent, Inc. */ #ifndef _SYS_MACHCPUVAR_H #define _SYS_MACHCPUVAR_H
*** 38,47 **** --- 38,50 ---- #include <sys/tss.h> #include <sys/segments.h> #include <sys/rm_platter.h> #include <sys/avintr.h> #include <sys/pte.h> + #include <sys/stddef.h> + #include <sys/debug.h> + #include <sys/cpuvar.h> #ifndef _ASM /* * On a virtualized platform a virtual cpu may not be actually * on a physical cpu, especially in situations where a configuration has
*** 76,85 **** --- 79,154 ---- ulong_t pending_sel[PIL_MAX + 1]; /* event array selectors */ ulong_t pending_evts[PIL_MAX + 1][sizeof (ulong_t) * 8]; ulong_t evt_affinity[sizeof (ulong_t) * 8]; /* service on cpu */ }; + struct kpti_frame { + uint64_t kf_lower_redzone; + + /* Stashed value of %cr3 when we entered the trampoline. */ + greg_t kf_tr_cr3; + + /* + * We use %r13-r14 as scratch registers in the trampoline code, + * so stash those here "below" the rest of the stack so they can be + * pushed/popped if needed. + */ + greg_t kf_r14; + greg_t kf_r13; + + /* + * Part of this struct is used as the HW stack frame when taking an + * interrupt on the user page table. The CPU is going to push a bunch + * of regs onto the stack pointer set in the TSS/IDT (which we set to + * &kf_rsp here). + * + * This is only a temporary holding area for them (we'll move them over + * to the real interrupt stack once we've set %cr3). + * + * Note that these must be cleared during a process switch on this cpu. + */ + greg_t kf_err; /* Bottom of initial hw stack frame */ + greg_t kf_rip; + greg_t kf_cs; + greg_t kf_rflags; + greg_t kf_rsp; + greg_t kf_ss; + + greg_t kf_tr_rsp; /* Top of HW stack frame */ + /* We also write this with the %rsp value on tramp entry */ + + /* Written to 0x1 when this kpti_frame is in use. */ + uint64_t kf_tr_flag; + + uint64_t kf_middle_redzone; + + /* + * The things we need to write to %cr3 to change between page tables. + * These live "above" the HW stack. + */ + greg_t kf_kernel_cr3; + greg_t kf_user_cr3; + greg_t kf_tr_ret_rsp; + + uint64_t kf_unused; /* For 16-byte align */ + + uint64_t kf_upper_redzone; + }; + + /* + * This first value, MACHCPU_SIZE is the size of all the members in the cpu_t + * AND struct machcpu, before we get to the mcpu_pad and the kpti area. + * The KPTI is used to contain per-CPU data that is visible in both sets of + * page-tables, and hence must be page-aligned and page-sized. See + * hat_pcp_setup(). + * + * There is a CTASSERT in os/intr.c that checks these numbers. + */ + #define MACHCPU_SIZE (572 + 1584) + #define MACHCPU_PAD (MMU_PAGESIZE - MACHCPU_SIZE) + #define MACHCPU_PAD2 (MMU_PAGESIZE - 16 - 3 * sizeof (struct kpti_frame)) + struct machcpu { /* * x_call fields - used for interprocessor cross calls */ struct xc_msg *xc_msgbox;
*** 101,110 **** --- 170,181 ---- user_desc_t *mcpu_gdt; /* GDT */ gate_desc_t *mcpu_idt; /* current IDT */ tss_t *mcpu_tss; /* TSS */ + void *mcpu_ldt; + size_t mcpu_ldt_len; kmutex_t mcpu_ppaddr_mutex; caddr_t mcpu_caddr1; /* per cpu CADDR1 */ caddr_t mcpu_caddr2; /* per cpu CADDR2 */ uint64_t mcpu_caddr1pte;
*** 145,154 **** --- 216,234 ---- * if a section of code gets interrupted. * The high order 16 bits will hold the cpu->cpu_id. * The low order bits will be incremented on every interrupt. */ volatile uint32_t mcpu_istamp; + + char mcpu_pad[MACHCPU_PAD]; + + /* This is the start of the page */ + char mcpu_pad2[MACHCPU_PAD2]; + struct kpti_frame mcpu_kpti; + struct kpti_frame mcpu_kpti_flt; + struct kpti_frame mcpu_kpti_dbg; + char mcpu_pad3[16]; }; #define NINTR_THREADS (LOCK_LEVEL-1) /* number of interrupt threads */ #define MWAIT_HALTED (1) /* mcpu_mwait set when halting */ #define MWAIT_RUNNING (0) /* mcpu_mwait set to wakeup */
*** 165,175 **** #define cpu_hat_info cpu_m.mcpu_hat_info #define cpu_ppaddr_mutex cpu_m.mcpu_ppaddr_mutex #define cpu_gdt cpu_m.mcpu_gdt #define cpu_idt cpu_m.mcpu_idt #define cpu_tss cpu_m.mcpu_tss - #define cpu_ldt cpu_m.mcpu_ldt #define cpu_caddr1 cpu_m.mcpu_caddr1 #define cpu_caddr2 cpu_m.mcpu_caddr2 #define cpu_softinfo cpu_m.mcpu_softinfo #define cpu_caddr1pte cpu_m.mcpu_caddr1pte #define cpu_caddr2pte cpu_m.mcpu_caddr2pte --- 245,254 ----