Print this page
OS-7125 Need mitigation of L1TF (CVE-2018-3646)
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>

*** 22,54 **** * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* ! * Copyright (c) 2018 Joyent, Inc. */ /* * Process switching routines. */ - #if defined(__lint) - #include <sys/thread.h> - #include <sys/systm.h> - #include <sys/time.h> - #else /* __lint */ - #include "assym.h" - #endif /* __lint */ - #include <sys/asm_linkage.h> #include <sys/asm_misc.h> #include <sys/regset.h> #include <sys/privregs.h> #include <sys/stack.h> #include <sys/segments.h> #include <sys/psw.h> /* * resume(thread_id_t t); * * a thread can only run on one processor at a time. there * exists a window on MPs where the current thread on one --- 22,48 ---- * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* ! * Copyright 2019 Joyent, Inc. */ /* * Process switching routines. */ #include <sys/asm_linkage.h> #include <sys/asm_misc.h> #include <sys/regset.h> #include <sys/privregs.h> #include <sys/stack.h> #include <sys/segments.h> #include <sys/psw.h> + #include "assym.h" + /* * resume(thread_id_t t); * * a thread can only run on one processor at a time. there * exists a window on MPs where the current thread on one
*** 72,91 **** * resume_from_zombie() is the same as resume except the calling thread * is a zombie and must be put on the deathrow list after the CPU is * off the stack. */ - #if !defined(__lint) - #if LWP_PCB_FPU != 0 #error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work #endif /* LWP_PCB_FPU != 0 */ - #endif /* !__lint */ - - #if defined(__amd64) - /* * Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15) * * The stack frame must be created before the save of %rsp so that tracebacks * of swtch()ed-out processes show the process as having last called swtch(). --- 66,79 ----
*** 151,242 **** movq T_INTR_START(thread_t), %rax; \ cmpxchgq %r14, T_INTR_START(thread_t); \ jnz 0b; \ 1: - #elif defined (__i386) - - /* - * Save non-volatile registers (%ebp, %esi, %edi and %ebx) - * - * The stack frame must be created before the save of %esp so that tracebacks - * of swtch()ed-out processes show the process as having last called swtch(). - */ - #define SAVE_REGS(thread_t, retaddr) \ - movl %ebp, T_EBP(thread_t); \ - movl %ebx, T_EBX(thread_t); \ - movl %esi, T_ESI(thread_t); \ - movl %edi, T_EDI(thread_t); \ - pushl %ebp; \ - movl %esp, %ebp; \ - movl %esp, T_SP(thread_t); \ - movl retaddr, T_PC(thread_t); \ - movl 8(%ebp), %edi; \ - pushl %edi; \ - call __dtrace_probe___sched_off__cpu; \ - addl $CLONGSIZE, %esp - - /* - * Restore non-volatile registers (%ebp, %esi, %edi and %ebx) - * - * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t - * already has the effect of putting the stack back the way it was when - * we came in. - */ - #define RESTORE_REGS(scratch_reg) \ - movl %gs:CPU_THREAD, scratch_reg; \ - movl T_EBP(scratch_reg), %ebp; \ - movl T_EBX(scratch_reg), %ebx; \ - movl T_ESI(scratch_reg), %esi; \ - movl T_EDI(scratch_reg), %edi - - /* - * Get pointer to a thread's hat structure - */ - #define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \ - movl T_PROCP(thread_t), hatp; \ - movl P_AS(hatp), scratch_reg; \ - movl A_HAT(scratch_reg), hatp - - /* - * If we are resuming an interrupt thread, store a timestamp in the thread - * structure. If an interrupt occurs between tsc_read() and its subsequent - * store, the timestamp will be stale by the time it is stored. We can detect - * this by doing a compare-and-swap on the thread's timestamp, since any - * interrupt occurring in this window will put a new timestamp in the thread's - * t_intr_start field. - */ - #define STORE_INTR_START(thread_t) \ - testw $T_INTR_THREAD, T_FLAGS(thread_t); \ - jz 1f; \ - pushl %ecx; \ - 0: \ - pushl T_INTR_START(thread_t); \ - pushl T_INTR_START+4(thread_t); \ - call tsc_read; \ - movl %eax, %ebx; \ - movl %edx, %ecx; \ - popl %edx; \ - popl %eax; \ - cmpxchg8b T_INTR_START(thread_t); \ - jnz 0b; \ - popl %ecx; \ - 1: - - #endif /* __amd64 */ - - #if defined(__lint) - - /* ARGSUSED */ - void - resume(kthread_t *t) - {} - - #else /* __lint */ - - #if defined(__amd64) - .global kpti_enable ENTRY(resume) movq %gs:CPU_THREAD, %rax leaq resume_return(%rip), %r11 --- 139,148 ----
*** 434,443 **** --- 340,351 ---- xorq %rax, %rax movq %rax, T_USERACC(%r12) call smap_disable .nosmap: + call ht_mark + /* * Restore non-volatile registers, then have spl0 return to the * resuming thread's PC after first setting the priority as low as * possible and blocking all interrupt threads that may be active. */
*** 454,659 **** addq $CLONGSIZE, %rsp ret SET_SIZE(_resume_from_idle) SET_SIZE(resume) - #elif defined (__i386) - - ENTRY(resume) - movl %gs:CPU_THREAD, %eax - movl $resume_return, %ecx - - /* - * Save non-volatile registers, and set return address for current - * thread to resume_return. - * - * %edi = t (new thread) when done. - */ - SAVE_REGS(%eax, %ecx) - - LOADCPU(%ebx) /* %ebx = CPU */ - movl CPU_THREAD(%ebx), %esi /* %esi = curthread */ - - #ifdef DEBUG - call assert_ints_enabled /* panics if we are cli'd */ - #endif - /* - * Call savectx if thread has installed context ops. - * - * Note that if we have floating point context, the save op - * (either fpsave_begin or fpxsave_begin) will issue the - * async save instruction (fnsave or fxsave respectively) - * that we fwait for below. - */ - movl T_CTX(%esi), %eax /* should current thread savectx? */ - testl %eax, %eax - jz .nosavectx /* skip call when zero */ - pushl %esi /* arg = thread pointer */ - call savectx /* call ctx ops */ - addl $4, %esp /* restore stack pointer */ - .nosavectx: - - /* - * Call savepctx if process has installed context ops. - */ - movl T_PROCP(%esi), %eax /* %eax = proc */ - cmpl $0, P_PCTX(%eax) /* should current thread savectx? */ - je .nosavepctx /* skip call when zero */ - pushl %eax /* arg = proc pointer */ - call savepctx /* call ctx ops */ - addl $4, %esp - .nosavepctx: - - /* - * Temporarily switch to the idle thread's stack - */ - movl CPU_IDLE_THREAD(%ebx), %eax /* idle thread pointer */ - - /* - * Set the idle thread as the current thread - */ - movl T_SP(%eax), %esp /* It is safe to set esp */ - movl %eax, CPU_THREAD(%ebx) - - /* switch in the hat context for the new thread */ - GET_THREAD_HATP(%ecx, %edi, %ecx) - pushl %ecx - call hat_switch - addl $4, %esp - - /* - * Clear and unlock previous thread's t_lock - * to allow it to be dispatched by another processor. - */ - movb $0, T_LOCK(%esi) - - /* - * IMPORTANT: Registers at this point must be: - * %edi = new thread - * - * Here we are in the idle thread, have dropped the old thread. - */ - ALTENTRY(_resume_from_idle) - /* - * spin until dispatched thread's mutex has - * been unlocked. this mutex is unlocked when - * it becomes safe for the thread to run. - */ - .L4: - lock - btsl $0, T_LOCK(%edi) /* lock new thread's mutex */ - jc .L4_2 /* lock did not succeed */ - - /* - * Fix CPU structure to indicate new running thread. - * Set pointer in new thread to the CPU structure. - */ - LOADCPU(%esi) /* load current CPU pointer */ - movl T_STACK(%edi), %eax /* here to use v pipeline of */ - /* Pentium. Used few lines below */ - cmpl %esi, T_CPU(%edi) - jne .L5_2 - .L5_1: - /* - * Setup esp0 (kernel stack) in TSS to curthread's stack. - * (Note: Since we don't have saved 'regs' structure for all - * the threads we can't easily determine if we need to - * change esp0. So, we simply change the esp0 to bottom - * of the thread stack and it will work for all cases.) - */ - movl CPU_TSS(%esi), %ecx - addl $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */ - #if !defined(__xpv) - movl %eax, TSS_ESP0(%ecx) - #else - pushl %eax - pushl $KDS_SEL - call HYPERVISOR_stack_switch - addl $8, %esp - #endif /* __xpv */ - - movl %edi, CPU_THREAD(%esi) /* set CPU's thread pointer */ - mfence /* synchronize with mutex_exit() */ - xorl %ebp, %ebp /* make $<threadlist behave better */ - movl T_LWP(%edi), %eax /* set associated lwp to */ - movl %eax, CPU_LWP(%esi) /* CPU's lwp ptr */ - - movl T_SP(%edi), %esp /* switch to outgoing thread's stack */ - movl T_PC(%edi), %esi /* saved return addr */ - - /* - * Call restorectx if context ops have been installed. - */ - movl T_CTX(%edi), %eax /* should resumed thread restorectx? */ - testl %eax, %eax - jz .norestorectx /* skip call when zero */ - pushl %edi /* arg = thread pointer */ - call restorectx /* call ctx ops */ - addl $4, %esp /* restore stack pointer */ - .norestorectx: - - /* - * Call restorepctx if context ops have been installed for the proc. - */ - movl T_PROCP(%edi), %eax - cmpl $0, P_PCTX(%eax) - je .norestorepctx - pushl %eax /* arg = proc pointer */ - call restorepctx - addl $4, %esp /* restore stack pointer */ - .norestorepctx: - - STORE_INTR_START(%edi) - - /* - * Restore non-volatile registers, then have spl0 return to the - * resuming thread's PC after first setting the priority as low as - * possible and blocking all interrupt threads that may be active. - */ - movl %esi, %eax /* save return address */ - RESTORE_REGS(%ecx) - pushl %eax /* push return address for spl0() */ - call __dtrace_probe___sched_on__cpu - jmp spl0 - - resume_return: - /* - * Remove stack frame created in SAVE_REGS() - */ - addl $CLONGSIZE, %esp - ret - - .L4_2: - pause - cmpb $0, T_LOCK(%edi) - je .L4 - jmp .L4_2 - - .L5_2: - /* cp->cpu_stats.sys.cpumigrate++ */ - addl $1, CPU_STATS_SYS_CPUMIGRATE(%esi) - adcl $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi) - movl %esi, T_CPU(%edi) /* set new thread's CPU pointer */ - jmp .L5_1 - - SET_SIZE(_resume_from_idle) - SET_SIZE(resume) - - #endif /* __amd64 */ - #endif /* __lint */ - - #if defined(__lint) - - /* ARGSUSED */ - void - resume_from_zombie(kthread_t *t) - {} - - #else /* __lint */ - - #if defined(__amd64) - ENTRY(resume_from_zombie) movq %gs:CPU_THREAD, %rax leaq resume_from_zombie_return(%rip), %r11 /* --- 362,371 ----
*** 724,815 **** */ addq $CLONGSIZE, %rsp ret SET_SIZE(resume_from_zombie) - #elif defined (__i386) - - ENTRY(resume_from_zombie) - movl %gs:CPU_THREAD, %eax - movl $resume_from_zombie_return, %ecx - - /* - * Save non-volatile registers, and set return address for current - * thread to resume_from_zombie_return. - * - * %edi = t (new thread) when done. - */ - SAVE_REGS(%eax, %ecx) - - #ifdef DEBUG - call assert_ints_enabled /* panics if we are cli'd */ - #endif - movl %gs:CPU_THREAD, %esi /* %esi = curthread */ - - /* clean up the fp unit. It might be left enabled */ - - movl %cr0, %eax - testl $CR0_TS, %eax - jnz .zfpu_disabled /* if TS already set, nothing to do */ - fninit /* init fpu & discard pending error */ - orl $CR0_TS, %eax - movl %eax, %cr0 - .zfpu_disabled: - - /* - * Temporarily switch to the idle thread's stack so that the zombie - * thread's stack can be reclaimed by the reaper. - */ - movl %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */ - movl T_SP(%eax), %esp /* get onto idle thread stack */ - - /* - * Set the idle thread as the current thread. - */ - movl %eax, %gs:CPU_THREAD - - /* - * switch in the hat context for the new thread - */ - GET_THREAD_HATP(%ecx, %edi, %ecx) - pushl %ecx - call hat_switch - addl $4, %esp - - /* - * Put the zombie on death-row. - */ - pushl %esi - call reapq_add - addl $4, %esp - jmp _resume_from_idle /* finish job of resume */ - - resume_from_zombie_return: - RESTORE_REGS(%ecx) /* restore non-volatile registers */ - call __dtrace_probe___sched_on__cpu - - /* - * Remove stack frame created in SAVE_REGS() - */ - addl $CLONGSIZE, %esp - ret - SET_SIZE(resume_from_zombie) - - #endif /* __amd64 */ - #endif /* __lint */ - - #if defined(__lint) - - /* ARGSUSED */ - void - resume_from_intr(kthread_t *t) - {} - - #else /* __lint */ - - #if defined(__amd64) - ENTRY(resume_from_intr) movq %gs:CPU_THREAD, %rax leaq resume_from_intr_return(%rip), %r11 /* --- 436,445 ----
*** 832,841 **** --- 462,473 ---- xorl %eax, %eax xchgb %al, T_LOCK(%r13) STORE_INTR_START(%r12) + call ht_mark + /* * Restore non-volatile registers, then have spl0 return to the * resuming thread's PC after first setting the priority as low as * possible and blocking all interrupt threads that may be active. */
*** 851,943 **** */ addq $CLONGSIZE, %rsp ret SET_SIZE(resume_from_intr) - #elif defined (__i386) - - ENTRY(resume_from_intr) - movl %gs:CPU_THREAD, %eax - movl $resume_from_intr_return, %ecx - - /* - * Save non-volatile registers, and set return address for current - * thread to resume_return. - * - * %edi = t (new thread) when done. - */ - SAVE_REGS(%eax, %ecx) - - #ifdef DEBUG - call assert_ints_enabled /* panics if we are cli'd */ - #endif - movl %gs:CPU_THREAD, %esi /* %esi = curthread */ - movl %edi, %gs:CPU_THREAD /* set CPU's thread pointer */ - mfence /* synchronize with mutex_exit() */ - movl T_SP(%edi), %esp /* restore resuming thread's sp */ - xorl %ebp, %ebp /* make $<threadlist behave better */ - - /* - * Unlock outgoing thread's mutex dispatched by another processor. - */ - xorl %eax,%eax - xchgb %al, T_LOCK(%esi) - - STORE_INTR_START(%edi) - - /* - * Restore non-volatile registers, then have spl0 return to the - * resuming thread's PC after first setting the priority as low as - * possible and blocking all interrupt threads that may be active. - */ - movl T_PC(%edi), %eax /* saved return addr */ - RESTORE_REGS(%ecx) - pushl %eax /* push return address for spl0() */ - call __dtrace_probe___sched_on__cpu - jmp spl0 - - resume_from_intr_return: - /* - * Remove stack frame created in SAVE_REGS() - */ - addl $CLONGSIZE, %esp - ret - SET_SIZE(resume_from_intr) - - #endif /* __amd64 */ - #endif /* __lint */ - - #if defined(__lint) - - void - thread_start(void) - {} - - #else /* __lint */ - - #if defined(__amd64) - ENTRY(thread_start) popq %rax /* start() */ popq %rdi /* arg */ popq %rsi /* len */ movq %rsp, %rbp call *%rax call thread_exit /* destroy thread if it returns. */ /*NOTREACHED*/ SET_SIZE(thread_start) - - #elif defined(__i386) - - ENTRY(thread_start) - popl %eax - movl %esp, %ebp - addl $8, %ebp - call *%eax - addl $8, %esp - call thread_exit /* destroy thread if it returns. */ - /*NOTREACHED*/ - SET_SIZE(thread_start) - - #endif /* __i386 */ - - #endif /* __lint */ --- 483,496 ----