Print this page
OS-7125 Need mitigation of L1TF (CVE-2018-3646)
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
*** 22,54 ****
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
! * Copyright (c) 2018 Joyent, Inc.
*/
/*
* Process switching routines.
*/
- #if defined(__lint)
- #include <sys/thread.h>
- #include <sys/systm.h>
- #include <sys/time.h>
- #else /* __lint */
- #include "assym.h"
- #endif /* __lint */
-
#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/stack.h>
#include <sys/segments.h>
#include <sys/psw.h>
/*
* resume(thread_id_t t);
*
* a thread can only run on one processor at a time. there
* exists a window on MPs where the current thread on one
--- 22,48 ----
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
! * Copyright 2019 Joyent, Inc.
*/
/*
* Process switching routines.
*/
#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/stack.h>
#include <sys/segments.h>
#include <sys/psw.h>
+ #include "assym.h"
+
/*
* resume(thread_id_t t);
*
* a thread can only run on one processor at a time. there
* exists a window on MPs where the current thread on one
*** 72,91 ****
* resume_from_zombie() is the same as resume except the calling thread
* is a zombie and must be put on the deathrow list after the CPU is
* off the stack.
*/
- #if !defined(__lint)
-
#if LWP_PCB_FPU != 0
#error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work
#endif /* LWP_PCB_FPU != 0 */
- #endif /* !__lint */
-
- #if defined(__amd64)
-
/*
* Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
*
* The stack frame must be created before the save of %rsp so that tracebacks
* of swtch()ed-out processes show the process as having last called swtch().
--- 66,79 ----
*** 151,242 ****
movq T_INTR_START(thread_t), %rax; \
cmpxchgq %r14, T_INTR_START(thread_t); \
jnz 0b; \
1:
- #elif defined (__i386)
-
- /*
- * Save non-volatile registers (%ebp, %esi, %edi and %ebx)
- *
- * The stack frame must be created before the save of %esp so that tracebacks
- * of swtch()ed-out processes show the process as having last called swtch().
- */
- #define SAVE_REGS(thread_t, retaddr) \
- movl %ebp, T_EBP(thread_t); \
- movl %ebx, T_EBX(thread_t); \
- movl %esi, T_ESI(thread_t); \
- movl %edi, T_EDI(thread_t); \
- pushl %ebp; \
- movl %esp, %ebp; \
- movl %esp, T_SP(thread_t); \
- movl retaddr, T_PC(thread_t); \
- movl 8(%ebp), %edi; \
- pushl %edi; \
- call __dtrace_probe___sched_off__cpu; \
- addl $CLONGSIZE, %esp
-
- /*
- * Restore non-volatile registers (%ebp, %esi, %edi and %ebx)
- *
- * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
- * already has the effect of putting the stack back the way it was when
- * we came in.
- */
- #define RESTORE_REGS(scratch_reg) \
- movl %gs:CPU_THREAD, scratch_reg; \
- movl T_EBP(scratch_reg), %ebp; \
- movl T_EBX(scratch_reg), %ebx; \
- movl T_ESI(scratch_reg), %esi; \
- movl T_EDI(scratch_reg), %edi
-
- /*
- * Get pointer to a thread's hat structure
- */
- #define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \
- movl T_PROCP(thread_t), hatp; \
- movl P_AS(hatp), scratch_reg; \
- movl A_HAT(scratch_reg), hatp
-
- /*
- * If we are resuming an interrupt thread, store a timestamp in the thread
- * structure. If an interrupt occurs between tsc_read() and its subsequent
- * store, the timestamp will be stale by the time it is stored. We can detect
- * this by doing a compare-and-swap on the thread's timestamp, since any
- * interrupt occurring in this window will put a new timestamp in the thread's
- * t_intr_start field.
- */
- #define STORE_INTR_START(thread_t) \
- testw $T_INTR_THREAD, T_FLAGS(thread_t); \
- jz 1f; \
- pushl %ecx; \
- 0: \
- pushl T_INTR_START(thread_t); \
- pushl T_INTR_START+4(thread_t); \
- call tsc_read; \
- movl %eax, %ebx; \
- movl %edx, %ecx; \
- popl %edx; \
- popl %eax; \
- cmpxchg8b T_INTR_START(thread_t); \
- jnz 0b; \
- popl %ecx; \
- 1:
-
- #endif /* __amd64 */
-
- #if defined(__lint)
-
- /* ARGSUSED */
- void
- resume(kthread_t *t)
- {}
-
- #else /* __lint */
-
- #if defined(__amd64)
-
.global kpti_enable
ENTRY(resume)
movq %gs:CPU_THREAD, %rax
leaq resume_return(%rip), %r11
--- 139,148 ----
*** 434,443 ****
--- 340,351 ----
xorq %rax, %rax
movq %rax, T_USERACC(%r12)
call smap_disable
.nosmap:
+ call ht_mark
+
/*
* Restore non-volatile registers, then have spl0 return to the
* resuming thread's PC after first setting the priority as low as
* possible and blocking all interrupt threads that may be active.
*/
*** 454,659 ****
addq $CLONGSIZE, %rsp
ret
SET_SIZE(_resume_from_idle)
SET_SIZE(resume)
- #elif defined (__i386)
-
- ENTRY(resume)
- movl %gs:CPU_THREAD, %eax
- movl $resume_return, %ecx
-
- /*
- * Save non-volatile registers, and set return address for current
- * thread to resume_return.
- *
- * %edi = t (new thread) when done.
- */
- SAVE_REGS(%eax, %ecx)
-
- LOADCPU(%ebx) /* %ebx = CPU */
- movl CPU_THREAD(%ebx), %esi /* %esi = curthread */
-
- #ifdef DEBUG
- call assert_ints_enabled /* panics if we are cli'd */
- #endif
- /*
- * Call savectx if thread has installed context ops.
- *
- * Note that if we have floating point context, the save op
- * (either fpsave_begin or fpxsave_begin) will issue the
- * async save instruction (fnsave or fxsave respectively)
- * that we fwait for below.
- */
- movl T_CTX(%esi), %eax /* should current thread savectx? */
- testl %eax, %eax
- jz .nosavectx /* skip call when zero */
- pushl %esi /* arg = thread pointer */
- call savectx /* call ctx ops */
- addl $4, %esp /* restore stack pointer */
- .nosavectx:
-
- /*
- * Call savepctx if process has installed context ops.
- */
- movl T_PROCP(%esi), %eax /* %eax = proc */
- cmpl $0, P_PCTX(%eax) /* should current thread savectx? */
- je .nosavepctx /* skip call when zero */
- pushl %eax /* arg = proc pointer */
- call savepctx /* call ctx ops */
- addl $4, %esp
- .nosavepctx:
-
- /*
- * Temporarily switch to the idle thread's stack
- */
- movl CPU_IDLE_THREAD(%ebx), %eax /* idle thread pointer */
-
- /*
- * Set the idle thread as the current thread
- */
- movl T_SP(%eax), %esp /* It is safe to set esp */
- movl %eax, CPU_THREAD(%ebx)
-
- /* switch in the hat context for the new thread */
- GET_THREAD_HATP(%ecx, %edi, %ecx)
- pushl %ecx
- call hat_switch
- addl $4, %esp
-
- /*
- * Clear and unlock previous thread's t_lock
- * to allow it to be dispatched by another processor.
- */
- movb $0, T_LOCK(%esi)
-
- /*
- * IMPORTANT: Registers at this point must be:
- * %edi = new thread
- *
- * Here we are in the idle thread, have dropped the old thread.
- */
- ALTENTRY(_resume_from_idle)
- /*
- * spin until dispatched thread's mutex has
- * been unlocked. this mutex is unlocked when
- * it becomes safe for the thread to run.
- */
- .L4:
- lock
- btsl $0, T_LOCK(%edi) /* lock new thread's mutex */
- jc .L4_2 /* lock did not succeed */
-
- /*
- * Fix CPU structure to indicate new running thread.
- * Set pointer in new thread to the CPU structure.
- */
- LOADCPU(%esi) /* load current CPU pointer */
- movl T_STACK(%edi), %eax /* here to use v pipeline of */
- /* Pentium. Used few lines below */
- cmpl %esi, T_CPU(%edi)
- jne .L5_2
- .L5_1:
- /*
- * Setup esp0 (kernel stack) in TSS to curthread's stack.
- * (Note: Since we don't have saved 'regs' structure for all
- * the threads we can't easily determine if we need to
- * change esp0. So, we simply change the esp0 to bottom
- * of the thread stack and it will work for all cases.)
- */
- movl CPU_TSS(%esi), %ecx
- addl $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */
- #if !defined(__xpv)
- movl %eax, TSS_ESP0(%ecx)
- #else
- pushl %eax
- pushl $KDS_SEL
- call HYPERVISOR_stack_switch
- addl $8, %esp
- #endif /* __xpv */
-
- movl %edi, CPU_THREAD(%esi) /* set CPU's thread pointer */
- mfence /* synchronize with mutex_exit() */
- xorl %ebp, %ebp /* make $<threadlist behave better */
- movl T_LWP(%edi), %eax /* set associated lwp to */
- movl %eax, CPU_LWP(%esi) /* CPU's lwp ptr */
-
- movl T_SP(%edi), %esp /* switch to outgoing thread's stack */
- movl T_PC(%edi), %esi /* saved return addr */
-
- /*
- * Call restorectx if context ops have been installed.
- */
- movl T_CTX(%edi), %eax /* should resumed thread restorectx? */
- testl %eax, %eax
- jz .norestorectx /* skip call when zero */
- pushl %edi /* arg = thread pointer */
- call restorectx /* call ctx ops */
- addl $4, %esp /* restore stack pointer */
- .norestorectx:
-
- /*
- * Call restorepctx if context ops have been installed for the proc.
- */
- movl T_PROCP(%edi), %eax
- cmpl $0, P_PCTX(%eax)
- je .norestorepctx
- pushl %eax /* arg = proc pointer */
- call restorepctx
- addl $4, %esp /* restore stack pointer */
- .norestorepctx:
-
- STORE_INTR_START(%edi)
-
- /*
- * Restore non-volatile registers, then have spl0 return to the
- * resuming thread's PC after first setting the priority as low as
- * possible and blocking all interrupt threads that may be active.
- */
- movl %esi, %eax /* save return address */
- RESTORE_REGS(%ecx)
- pushl %eax /* push return address for spl0() */
- call __dtrace_probe___sched_on__cpu
- jmp spl0
-
- resume_return:
- /*
- * Remove stack frame created in SAVE_REGS()
- */
- addl $CLONGSIZE, %esp
- ret
-
- .L4_2:
- pause
- cmpb $0, T_LOCK(%edi)
- je .L4
- jmp .L4_2
-
- .L5_2:
- /* cp->cpu_stats.sys.cpumigrate++ */
- addl $1, CPU_STATS_SYS_CPUMIGRATE(%esi)
- adcl $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi)
- movl %esi, T_CPU(%edi) /* set new thread's CPU pointer */
- jmp .L5_1
-
- SET_SIZE(_resume_from_idle)
- SET_SIZE(resume)
-
- #endif /* __amd64 */
- #endif /* __lint */
-
- #if defined(__lint)
-
- /* ARGSUSED */
- void
- resume_from_zombie(kthread_t *t)
- {}
-
- #else /* __lint */
-
- #if defined(__amd64)
-
ENTRY(resume_from_zombie)
movq %gs:CPU_THREAD, %rax
leaq resume_from_zombie_return(%rip), %r11
/*
--- 362,371 ----
*** 724,815 ****
*/
addq $CLONGSIZE, %rsp
ret
SET_SIZE(resume_from_zombie)
- #elif defined (__i386)
-
- ENTRY(resume_from_zombie)
- movl %gs:CPU_THREAD, %eax
- movl $resume_from_zombie_return, %ecx
-
- /*
- * Save non-volatile registers, and set return address for current
- * thread to resume_from_zombie_return.
- *
- * %edi = t (new thread) when done.
- */
- SAVE_REGS(%eax, %ecx)
-
- #ifdef DEBUG
- call assert_ints_enabled /* panics if we are cli'd */
- #endif
- movl %gs:CPU_THREAD, %esi /* %esi = curthread */
-
- /* clean up the fp unit. It might be left enabled */
-
- movl %cr0, %eax
- testl $CR0_TS, %eax
- jnz .zfpu_disabled /* if TS already set, nothing to do */
- fninit /* init fpu & discard pending error */
- orl $CR0_TS, %eax
- movl %eax, %cr0
- .zfpu_disabled:
-
- /*
- * Temporarily switch to the idle thread's stack so that the zombie
- * thread's stack can be reclaimed by the reaper.
- */
- movl %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */
- movl T_SP(%eax), %esp /* get onto idle thread stack */
-
- /*
- * Set the idle thread as the current thread.
- */
- movl %eax, %gs:CPU_THREAD
-
- /*
- * switch in the hat context for the new thread
- */
- GET_THREAD_HATP(%ecx, %edi, %ecx)
- pushl %ecx
- call hat_switch
- addl $4, %esp
-
- /*
- * Put the zombie on death-row.
- */
- pushl %esi
- call reapq_add
- addl $4, %esp
- jmp _resume_from_idle /* finish job of resume */
-
- resume_from_zombie_return:
- RESTORE_REGS(%ecx) /* restore non-volatile registers */
- call __dtrace_probe___sched_on__cpu
-
- /*
- * Remove stack frame created in SAVE_REGS()
- */
- addl $CLONGSIZE, %esp
- ret
- SET_SIZE(resume_from_zombie)
-
- #endif /* __amd64 */
- #endif /* __lint */
-
- #if defined(__lint)
-
- /* ARGSUSED */
- void
- resume_from_intr(kthread_t *t)
- {}
-
- #else /* __lint */
-
- #if defined(__amd64)
-
ENTRY(resume_from_intr)
movq %gs:CPU_THREAD, %rax
leaq resume_from_intr_return(%rip), %r11
/*
--- 436,445 ----
*** 832,841 ****
--- 462,473 ----
xorl %eax, %eax
xchgb %al, T_LOCK(%r13)
STORE_INTR_START(%r12)
+ call ht_mark
+
/*
* Restore non-volatile registers, then have spl0 return to the
* resuming thread's PC after first setting the priority as low as
* possible and blocking all interrupt threads that may be active.
*/
*** 851,943 ****
*/
addq $CLONGSIZE, %rsp
ret
SET_SIZE(resume_from_intr)
- #elif defined (__i386)
-
- ENTRY(resume_from_intr)
- movl %gs:CPU_THREAD, %eax
- movl $resume_from_intr_return, %ecx
-
- /*
- * Save non-volatile registers, and set return address for current
- * thread to resume_return.
- *
- * %edi = t (new thread) when done.
- */
- SAVE_REGS(%eax, %ecx)
-
- #ifdef DEBUG
- call assert_ints_enabled /* panics if we are cli'd */
- #endif
- movl %gs:CPU_THREAD, %esi /* %esi = curthread */
- movl %edi, %gs:CPU_THREAD /* set CPU's thread pointer */
- mfence /* synchronize with mutex_exit() */
- movl T_SP(%edi), %esp /* restore resuming thread's sp */
- xorl %ebp, %ebp /* make $<threadlist behave better */
-
- /*
- * Unlock outgoing thread's mutex dispatched by another processor.
- */
- xorl %eax,%eax
- xchgb %al, T_LOCK(%esi)
-
- STORE_INTR_START(%edi)
-
- /*
- * Restore non-volatile registers, then have spl0 return to the
- * resuming thread's PC after first setting the priority as low as
- * possible and blocking all interrupt threads that may be active.
- */
- movl T_PC(%edi), %eax /* saved return addr */
- RESTORE_REGS(%ecx)
- pushl %eax /* push return address for spl0() */
- call __dtrace_probe___sched_on__cpu
- jmp spl0
-
- resume_from_intr_return:
- /*
- * Remove stack frame created in SAVE_REGS()
- */
- addl $CLONGSIZE, %esp
- ret
- SET_SIZE(resume_from_intr)
-
- #endif /* __amd64 */
- #endif /* __lint */
-
- #if defined(__lint)
-
- void
- thread_start(void)
- {}
-
- #else /* __lint */
-
- #if defined(__amd64)
-
ENTRY(thread_start)
popq %rax /* start() */
popq %rdi /* arg */
popq %rsi /* len */
movq %rsp, %rbp
call *%rax
call thread_exit /* destroy thread if it returns. */
/*NOTREACHED*/
SET_SIZE(thread_start)
-
- #elif defined(__i386)
-
- ENTRY(thread_start)
- popl %eax
- movl %esp, %ebp
- addl $8, %ebp
- call *%eax
- addl $8, %esp
- call thread_exit /* destroy thread if it returns. */
- /*NOTREACHED*/
- SET_SIZE(thread_start)
-
- #endif /* __i386 */
-
- #endif /* __lint */
--- 483,496 ----