Print this page
10924 Need mitigation of L1TF (CVE-2018-3646)
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Peter Tribble <peter.tribble@gmail.com>

@@ -22,33 +22,27 @@
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 /*
- * Copyright (c) 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*
  * Process switching routines.
  */
 
-#if defined(__lint)
-#include <sys/thread.h>
-#include <sys/systm.h>
-#include <sys/time.h>
-#else   /* __lint */
-#include "assym.h"
-#endif  /* __lint */
-
 #include <sys/asm_linkage.h>
 #include <sys/asm_misc.h>
 #include <sys/regset.h>
 #include <sys/privregs.h>
 #include <sys/stack.h>
 #include <sys/segments.h>
 #include <sys/psw.h>
 
+#include "assym.h"
+
 /*
  * resume(thread_id_t t);
  *
  * a thread can only run on one processor at a time. there
  * exists a window on MPs where the current thread on one

@@ -72,20 +66,14 @@
  * resume_from_zombie() is the same as resume except the calling thread
  * is a zombie and must be put on the deathrow list after the CPU is
  * off the stack.
  */
 
-#if !defined(__lint)
-
 #if LWP_PCB_FPU != 0
 #error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work
 #endif  /* LWP_PCB_FPU != 0 */
 
-#endif  /* !__lint */
-
-#if defined(__amd64)
-
 /*
  * Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
  *
  * The stack frame must be created before the save of %rsp so that tracebacks
  * of swtch()ed-out processes show the process as having last called swtch().

@@ -151,92 +139,10 @@
         movq    T_INTR_START(thread_t), %rax;           \
         cmpxchgq %r14, T_INTR_START(thread_t);          \
         jnz     0b;                                     \
 1:
 
-#elif defined (__i386)
-
-/*
- * Save non-volatile registers (%ebp, %esi, %edi and %ebx)
- *
- * The stack frame must be created before the save of %esp so that tracebacks
- * of swtch()ed-out processes show the process as having last called swtch().
- */
-#define SAVE_REGS(thread_t, retaddr)                    \
-        movl    %ebp, T_EBP(thread_t);                  \
-        movl    %ebx, T_EBX(thread_t);                  \
-        movl    %esi, T_ESI(thread_t);                  \
-        movl    %edi, T_EDI(thread_t);                  \
-        pushl   %ebp;                                   \
-        movl    %esp, %ebp;                             \
-        movl    %esp, T_SP(thread_t);                   \
-        movl    retaddr, T_PC(thread_t);                \
-        movl    8(%ebp), %edi;                          \
-        pushl   %edi;                                   \
-        call    __dtrace_probe___sched_off__cpu;        \
-        addl    $CLONGSIZE, %esp
-
-/*
- * Restore non-volatile registers (%ebp, %esi, %edi and %ebx)
- *
- * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
- * already has the effect of putting the stack back the way it was when
- * we came in.
- */
-#define RESTORE_REGS(scratch_reg)                       \
-        movl    %gs:CPU_THREAD, scratch_reg;            \
-        movl    T_EBP(scratch_reg), %ebp;               \
-        movl    T_EBX(scratch_reg), %ebx;               \
-        movl    T_ESI(scratch_reg), %esi;               \
-        movl    T_EDI(scratch_reg), %edi
-
-/*
- * Get pointer to a thread's hat structure
- */
-#define GET_THREAD_HATP(hatp, thread_t, scratch_reg)    \
-        movl    T_PROCP(thread_t), hatp;                \
-        movl    P_AS(hatp), scratch_reg;                \
-        movl    A_HAT(scratch_reg), hatp
-
-/*
- * If we are resuming an interrupt thread, store a timestamp in the thread
- * structure.  If an interrupt occurs between tsc_read() and its subsequent
- * store, the timestamp will be stale by the time it is stored.  We can detect
- * this by doing a compare-and-swap on the thread's timestamp, since any
- * interrupt occurring in this window will put a new timestamp in the thread's
- * t_intr_start field.
- */
-#define STORE_INTR_START(thread_t)                      \
-        testw   $T_INTR_THREAD, T_FLAGS(thread_t);      \
-        jz      1f;                                     \
-        pushl   %ecx;                                   \
-0:                                                      \
-        pushl   T_INTR_START(thread_t);                 \
-        pushl   T_INTR_START+4(thread_t);               \
-        call    tsc_read;                               \
-        movl    %eax, %ebx;                             \
-        movl    %edx, %ecx;                             \
-        popl    %edx;                                   \
-        popl    %eax;                                   \
-        cmpxchg8b T_INTR_START(thread_t);               \
-        jnz     0b;                                     \
-        popl    %ecx;                                   \
-1:
-
-#endif  /* __amd64 */
-
-#if defined(__lint)
-
-/* ARGSUSED */
-void
-resume(kthread_t *t)
-{}
-
-#else   /* __lint */
-
-#if defined(__amd64)
-
         .global kpti_enable
 
         ENTRY(resume)
         movq    %gs:CPU_THREAD, %rax
         leaq    resume_return(%rip), %r11

@@ -434,10 +340,12 @@
         xorq    %rax, %rax
         movq    %rax, T_USERACC(%r12)
         call    smap_disable
 .nosmap:
 
+        call    ht_mark
+
         /*
          * Restore non-volatile registers, then have spl0 return to the
          * resuming thread's PC after first setting the priority as low as
          * possible and blocking all interrupt threads that may be active.
          */

@@ -454,206 +362,10 @@
         addq    $CLONGSIZE, %rsp
         ret
         SET_SIZE(_resume_from_idle)
         SET_SIZE(resume)
 
-#elif defined (__i386)
-
-        ENTRY(resume)
-        movl    %gs:CPU_THREAD, %eax
-        movl    $resume_return, %ecx
-
-        /*
-         * Save non-volatile registers, and set return address for current
-         * thread to resume_return.
-         *
-         * %edi = t (new thread) when done.
-         */
-        SAVE_REGS(%eax,  %ecx)
-
-        LOADCPU(%ebx)                   /* %ebx = CPU */
-        movl    CPU_THREAD(%ebx), %esi  /* %esi = curthread */
-
-#ifdef DEBUG
-        call    assert_ints_enabled     /* panics if we are cli'd */
-#endif
-        /*
-         * Call savectx if thread has installed context ops.
-         *
-         * Note that if we have floating point context, the save op
-         * (either fpsave_begin or fpxsave_begin) will issue the
-         * async save instruction (fnsave or fxsave respectively)
-         * that we fwait for below.
-         */
-        movl    T_CTX(%esi), %eax       /* should current thread savectx? */
-        testl   %eax, %eax
-        jz      .nosavectx              /* skip call when zero */
-        pushl   %esi                    /* arg = thread pointer */
-        call    savectx                 /* call ctx ops */
-        addl    $4, %esp                /* restore stack pointer */
-.nosavectx:
-
-        /*
-         * Call savepctx if process has installed context ops.
-         */
-        movl    T_PROCP(%esi), %eax     /* %eax = proc */
-        cmpl    $0, P_PCTX(%eax)        /* should current thread savectx? */
-        je      .nosavepctx             /* skip call when zero */
-        pushl   %eax                    /* arg = proc pointer */
-        call    savepctx                /* call ctx ops */
-        addl    $4, %esp
-.nosavepctx:
-
-        /*
-         * Temporarily switch to the idle thread's stack
-         */
-        movl    CPU_IDLE_THREAD(%ebx), %eax     /* idle thread pointer */
-
-        /*
-         * Set the idle thread as the current thread
-         */
-        movl    T_SP(%eax), %esp        /* It is safe to set esp */
-        movl    %eax, CPU_THREAD(%ebx)
-
-        /* switch in the hat context for the new thread */
-        GET_THREAD_HATP(%ecx, %edi, %ecx)
-        pushl   %ecx
-        call    hat_switch
-        addl    $4, %esp
-
-        /*
-         * Clear and unlock previous thread's t_lock
-         * to allow it to be dispatched by another processor.
-         */
-        movb    $0, T_LOCK(%esi)
-
-        /*
-         * IMPORTANT: Registers at this point must be:
-         *       %edi = new thread
-         *
-         * Here we are in the idle thread, have dropped the old thread.
-         */
-        ALTENTRY(_resume_from_idle)
-        /*
-         * spin until dispatched thread's mutex has
-         * been unlocked. this mutex is unlocked when
-         * it becomes safe for the thread to run.
-         */
-.L4:
-        lock
-        btsl    $0, T_LOCK(%edi) /* lock new thread's mutex */
-        jc      .L4_2                   /* lock did not succeed */
-
-        /*
-         * Fix CPU structure to indicate new running thread.
-         * Set pointer in new thread to the CPU structure.
-         */
-        LOADCPU(%esi)                   /* load current CPU pointer */
-        movl    T_STACK(%edi), %eax     /* here to use v pipeline of */
-                                        /* Pentium. Used few lines below */
-        cmpl    %esi, T_CPU(%edi)
-        jne     .L5_2
-.L5_1:
-        /*
-         * Setup esp0 (kernel stack) in TSS to curthread's stack.
-         * (Note: Since we don't have saved 'regs' structure for all
-         *        the threads we can't easily determine if we need to
-         *        change esp0. So, we simply change the esp0 to bottom 
-         *        of the thread stack and it will work for all cases.)
-         */
-        movl    CPU_TSS(%esi), %ecx
-        addl    $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */
-#if !defined(__xpv)
-        movl    %eax, TSS_ESP0(%ecx)
-#else
-        pushl   %eax
-        pushl   $KDS_SEL
-        call    HYPERVISOR_stack_switch
-        addl    $8, %esp
-#endif  /* __xpv */
-
-        movl    %edi, CPU_THREAD(%esi)  /* set CPU's thread pointer */
-        mfence                          /* synchronize with mutex_exit() */
-        xorl    %ebp, %ebp              /* make $<threadlist behave better */
-        movl    T_LWP(%edi), %eax       /* set associated lwp to  */
-        movl    %eax, CPU_LWP(%esi)     /* CPU's lwp ptr */
-
-        movl    T_SP(%edi), %esp        /* switch to outgoing thread's stack */
-        movl    T_PC(%edi), %esi        /* saved return addr */
-
-        /*
-         * Call restorectx if context ops have been installed.
-         */
-        movl    T_CTX(%edi), %eax       /* should resumed thread restorectx? */
-        testl   %eax, %eax
-        jz      .norestorectx           /* skip call when zero */
-        pushl   %edi                    /* arg = thread pointer */
-        call    restorectx              /* call ctx ops */
-        addl    $4, %esp                /* restore stack pointer */
-.norestorectx:
-
-        /*
-         * Call restorepctx if context ops have been installed for the proc.
-         */
-        movl    T_PROCP(%edi), %eax
-        cmpl    $0, P_PCTX(%eax)
-        je      .norestorepctx
-        pushl   %eax                    /* arg = proc pointer */
-        call    restorepctx
-        addl    $4, %esp                /* restore stack pointer */
-.norestorepctx:
-
-        STORE_INTR_START(%edi)
-
-        /*
-         * Restore non-volatile registers, then have spl0 return to the
-         * resuming thread's PC after first setting the priority as low as
-         * possible and blocking all interrupt threads that may be active.
-         */
-        movl    %esi, %eax              /* save return address */
-        RESTORE_REGS(%ecx)
-        pushl   %eax                    /* push return address for spl0() */
-        call    __dtrace_probe___sched_on__cpu
-        jmp     spl0
-
-resume_return:
-        /*
-         * Remove stack frame created in SAVE_REGS()
-         */
-        addl    $CLONGSIZE, %esp
-        ret
-
-.L4_2:
-        pause
-        cmpb    $0, T_LOCK(%edi)
-        je      .L4
-        jmp     .L4_2
-
-.L5_2:
-        /* cp->cpu_stats.sys.cpumigrate++ */
-        addl    $1, CPU_STATS_SYS_CPUMIGRATE(%esi)
-        adcl    $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi)
-        movl    %esi, T_CPU(%edi)       /* set new thread's CPU pointer */
-        jmp     .L5_1
-
-        SET_SIZE(_resume_from_idle)
-        SET_SIZE(resume)
-
-#endif  /* __amd64 */
-#endif  /* __lint */
-
-#if defined(__lint)
-
-/* ARGSUSED */
-void
-resume_from_zombie(kthread_t *t)
-{}
-
-#else   /* __lint */
-
-#if defined(__amd64)
-
         ENTRY(resume_from_zombie)
         movq    %gs:CPU_THREAD, %rax
         leaq    resume_from_zombie_return(%rip), %r11
 
         /*

@@ -724,92 +436,10 @@
          */
         addq    $CLONGSIZE, %rsp
         ret
         SET_SIZE(resume_from_zombie)
 
-#elif defined (__i386)
-
-        ENTRY(resume_from_zombie)
-        movl    %gs:CPU_THREAD, %eax
-        movl    $resume_from_zombie_return, %ecx
-
-        /*
-         * Save non-volatile registers, and set return address for current
-         * thread to resume_from_zombie_return.
-         *
-         * %edi = t (new thread) when done.
-         */
-        SAVE_REGS(%eax, %ecx)
-
-#ifdef DEBUG
-        call    assert_ints_enabled     /* panics if we are cli'd */
-#endif
-        movl    %gs:CPU_THREAD, %esi    /* %esi = curthread */
-
-        /* clean up the fp unit. It might be left enabled */
-
-        movl    %cr0, %eax
-        testl   $CR0_TS, %eax
-        jnz     .zfpu_disabled          /* if TS already set, nothing to do */
-        fninit                          /* init fpu & discard pending error */
-        orl     $CR0_TS, %eax
-        movl    %eax, %cr0
-.zfpu_disabled:
-
-        /*
-         * Temporarily switch to the idle thread's stack so that the zombie
-         * thread's stack can be reclaimed by the reaper.
-         */
-        movl    %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */
-        movl    T_SP(%eax), %esp        /* get onto idle thread stack */
-
-        /*
-         * Set the idle thread as the current thread.
-         */
-        movl    %eax, %gs:CPU_THREAD
-
-        /*
-         * switch in the hat context for the new thread
-         */
-        GET_THREAD_HATP(%ecx, %edi, %ecx)
-        pushl   %ecx
-        call    hat_switch
-        addl    $4, %esp
-
-        /*
-         * Put the zombie on death-row.
-         */
-        pushl   %esi
-        call    reapq_add
-        addl    $4, %esp
-        jmp     _resume_from_idle       /* finish job of resume */
-
-resume_from_zombie_return:
-        RESTORE_REGS(%ecx)              /* restore non-volatile registers */
-        call    __dtrace_probe___sched_on__cpu
-
-        /*
-         * Remove stack frame created in SAVE_REGS()
-         */
-        addl    $CLONGSIZE, %esp
-        ret
-        SET_SIZE(resume_from_zombie)
-
-#endif  /* __amd64 */
-#endif  /* __lint */
-
-#if defined(__lint)
-
-/* ARGSUSED */
-void
-resume_from_intr(kthread_t *t)
-{}
-
-#else   /* __lint */
-
-#if defined(__amd64)
-
         ENTRY(resume_from_intr)
         movq    %gs:CPU_THREAD, %rax
         leaq    resume_from_intr_return(%rip), %r11
 
         /*

@@ -832,10 +462,12 @@
         xorl    %eax, %eax
         xchgb   %al, T_LOCK(%r13)
 
         STORE_INTR_START(%r12)
 
+        call    ht_mark
+
         /*
          * Restore non-volatile registers, then have spl0 return to the
          * resuming thread's PC after first setting the priority as low as
          * possible and blocking all interrupt threads that may be active.
          */

@@ -851,93 +483,14 @@
          */
         addq    $CLONGSIZE, %rsp
         ret
         SET_SIZE(resume_from_intr)
 
-#elif defined (__i386)
-
-        ENTRY(resume_from_intr)
-        movl    %gs:CPU_THREAD, %eax
-        movl    $resume_from_intr_return, %ecx
-
-        /*
-         * Save non-volatile registers, and set return address for current
-         * thread to resume_return.
-         *
-         * %edi = t (new thread) when done.
-         */
-        SAVE_REGS(%eax, %ecx)
-
-#ifdef DEBUG
-        call    assert_ints_enabled     /* panics if we are cli'd */
-#endif
-        movl    %gs:CPU_THREAD, %esi    /* %esi = curthread */
-        movl    %edi, %gs:CPU_THREAD    /* set CPU's thread pointer */
-        mfence                          /* synchronize with mutex_exit() */
-        movl    T_SP(%edi), %esp        /* restore resuming thread's sp */
-        xorl    %ebp, %ebp              /* make $<threadlist behave better */
-
-        /*
-         * Unlock outgoing thread's mutex dispatched by another processor.
-         */
-        xorl    %eax,%eax
-        xchgb   %al, T_LOCK(%esi)
-
-        STORE_INTR_START(%edi)
-
-        /*
-         * Restore non-volatile registers, then have spl0 return to the
-         * resuming thread's PC after first setting the priority as low as
-         * possible and blocking all interrupt threads that may be active.
-         */
-        movl    T_PC(%edi), %eax        /* saved return addr */
-        RESTORE_REGS(%ecx)
-        pushl   %eax                    /* push return address for spl0() */
-        call    __dtrace_probe___sched_on__cpu
-        jmp     spl0
-
-resume_from_intr_return:
-        /*
-         * Remove stack frame created in SAVE_REGS()
-         */
-        addl    $CLONGSIZE, %esp
-        ret
-        SET_SIZE(resume_from_intr)
-
-#endif  /* __amd64 */
-#endif /* __lint */
-
-#if defined(__lint)
-
-void
-thread_start(void)
-{}
-
-#else   /* __lint */
-
-#if defined(__amd64)
-
         ENTRY(thread_start)
         popq    %rax            /* start() */
         popq    %rdi            /* arg */
         popq    %rsi            /* len */
         movq    %rsp, %rbp
         call    *%rax
         call    thread_exit     /* destroy thread if it returns. */
         /*NOTREACHED*/
         SET_SIZE(thread_start)
-
-#elif defined(__i386)
-
-        ENTRY(thread_start)
-        popl    %eax
-        movl    %esp, %ebp
-        addl    $8, %ebp
-        call    *%eax
-        addl    $8, %esp
-        call    thread_exit     /* destroy thread if it returns. */
-        /*NOTREACHED*/
-        SET_SIZE(thread_start)
-
-#endif  /* __i386 */
-
-#endif  /* __lint */