1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 /*
  27  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  28  */
  29 
  30 /*
  31  * Process switching routines.
  32  */
  33 
  34 #if defined(__lint)
  35 #include <sys/thread.h>
  36 #include <sys/systm.h>
  37 #include <sys/time.h>
  38 #else   /* __lint */
  39 #include "assym.h"
  40 #endif  /* __lint */
  41 
  42 #include <sys/asm_linkage.h>
  43 #include <sys/asm_misc.h>
  44 #include <sys/regset.h>
  45 #include <sys/privregs.h>
  46 #include <sys/stack.h>
  47 #include <sys/segments.h>
  48 #include <sys/psw.h>
  49 
  50 /*
  51  * resume(thread_id_t t);
  52  *
  53  * a thread can only run on one processor at a time. there
  54  * exists a window on MPs where the current thread on one
  55  * processor is capable of being dispatched by another processor.
  56  * some overlap between outgoing and incoming threads can happen
  57  * when they are the same thread. in this case where the threads
  58  * are the same, resume() on one processor will spin on the incoming
  59  * thread until resume() on the other processor has finished with
  60  * the outgoing thread.
  61  *
  62  * The MMU context changes when the resuming thread resides in a different
  63  * process.  Kernel threads are known by resume to reside in process 0.
  64  * The MMU context, therefore, only changes when resuming a thread in
  65  * a process different from curproc.
  66  *
  67  * resume_from_intr() is called when the thread being resumed was not 
  68  * passivated by resume (e.g. was interrupted).  This means that the
  69  * resume lock is already held and that a restore context is not needed.
  70  * Also, the MMU context is not changed on the resume in this case.
  71  *
  72  * resume_from_zombie() is the same as resume except the calling thread
  73  * is a zombie and must be put on the deathrow list after the CPU is
  74  * off the stack.
  75  */
  76 
  77 #if !defined(__lint)
  78 
  79 #if LWP_PCB_FPU != 0
  80 #error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work
  81 #endif  /* LWP_PCB_FPU != 0 */
  82 
  83 #endif  /* !__lint */
  84 
  85 #if defined(__amd64)
  86 
  87 /*
  88  * Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
  89  *
  90  * The stack frame must be created before the save of %rsp so that tracebacks
  91  * of swtch()ed-out processes show the process as having last called swtch().
  92  */
  93 #define SAVE_REGS(thread_t, retaddr)                    \
  94         movq    %rbp, T_RBP(thread_t);                  \
  95         movq    %rbx, T_RBX(thread_t);                  \
  96         movq    %r12, T_R12(thread_t);                  \
  97         movq    %r13, T_R13(thread_t);                  \
  98         movq    %r14, T_R14(thread_t);                  \
  99         movq    %r15, T_R15(thread_t);                  \
 100         pushq   %rbp;                                   \
 101         movq    %rsp, %rbp;                             \
 102         movq    %rsp, T_SP(thread_t);                   \
 103         movq    retaddr, T_PC(thread_t);                \
 104         movq    %rdi, %r12;                             \
 105         call    __dtrace_probe___sched_off__cpu
 106 
 107 /*
 108  * Restore non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
 109  *
 110  * We load up %rsp from the label_t as part of the context switch, so
 111  * we don't repeat that here.
 112  *
 113  * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
 114  * already has the effect of putting the stack back the way it was when
 115  * we came in.
 116  */
 117 #define RESTORE_REGS(scratch_reg)                       \
 118         movq    %gs:CPU_THREAD, scratch_reg;            \
 119         movq    T_RBP(scratch_reg), %rbp;               \
 120         movq    T_RBX(scratch_reg), %rbx;               \
 121         movq    T_R12(scratch_reg), %r12;               \
 122         movq    T_R13(scratch_reg), %r13;               \
 123         movq    T_R14(scratch_reg), %r14;               \
 124         movq    T_R15(scratch_reg), %r15
 125 
 126 /*
 127  * Get pointer to a thread's hat structure
 128  */
 129 #define GET_THREAD_HATP(hatp, thread_t, scratch_reg)    \
 130         movq    T_PROCP(thread_t), hatp;                \
 131         movq    P_AS(hatp), scratch_reg;                \
 132         movq    A_HAT(scratch_reg), hatp
 133 
 134 #define TSC_READ()                                      \
 135         call    tsc_read;                               \
 136         movq    %rax, %r14;
 137 
 138 /*
 139  * If we are resuming an interrupt thread, store a timestamp in the thread
 140  * structure.  If an interrupt occurs between tsc_read() and its subsequent
 141  * store, the timestamp will be stale by the time it is stored.  We can detect
 142  * this by doing a compare-and-swap on the thread's timestamp, since any
 143  * interrupt occurring in this window will put a new timestamp in the thread's
 144  * t_intr_start field.
 145  */
 146 #define STORE_INTR_START(thread_t)                      \
 147         testw   $T_INTR_THREAD, T_FLAGS(thread_t);      \
 148         jz      1f;                                     \
 149 0:                                                      \
 150         TSC_READ();                                     \
 151         movq    T_INTR_START(thread_t), %rax;           \
 152         cmpxchgq %r14, T_INTR_START(thread_t);          \
 153         jnz     0b;                                     \
 154 1:
 155 
 156 #elif defined (__i386)
 157 
 158 /*
 159  * Save non-volatile registers (%ebp, %esi, %edi and %ebx)
 160  *
 161  * The stack frame must be created before the save of %esp so that tracebacks
 162  * of swtch()ed-out processes show the process as having last called swtch().
 163  */
 164 #define SAVE_REGS(thread_t, retaddr)                    \
 165         movl    %ebp, T_EBP(thread_t);                  \
 166         movl    %ebx, T_EBX(thread_t);                  \
 167         movl    %esi, T_ESI(thread_t);                  \
 168         movl    %edi, T_EDI(thread_t);                  \
 169         pushl   %ebp;                                   \
 170         movl    %esp, %ebp;                             \
 171         movl    %esp, T_SP(thread_t);                   \
 172         movl    retaddr, T_PC(thread_t);                \
 173         movl    8(%ebp), %edi;                          \
 174         pushl   %edi;                                   \
 175         call    __dtrace_probe___sched_off__cpu;        \
 176         addl    $CLONGSIZE, %esp
 177 
 178 /*
 179  * Restore non-volatile registers (%ebp, %esi, %edi and %ebx)
 180  *
 181  * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
 182  * already has the effect of putting the stack back the way it was when
 183  * we came in.
 184  */
 185 #define RESTORE_REGS(scratch_reg)                       \
 186         movl    %gs:CPU_THREAD, scratch_reg;            \
 187         movl    T_EBP(scratch_reg), %ebp;               \
 188         movl    T_EBX(scratch_reg), %ebx;               \
 189         movl    T_ESI(scratch_reg), %esi;               \
 190         movl    T_EDI(scratch_reg), %edi
 191 
 192 /*
 193  * Get pointer to a thread's hat structure
 194  */
 195 #define GET_THREAD_HATP(hatp, thread_t, scratch_reg)    \
 196         movl    T_PROCP(thread_t), hatp;                \
 197         movl    P_AS(hatp), scratch_reg;                \
 198         movl    A_HAT(scratch_reg), hatp
 199 
 200 /*
 201  * If we are resuming an interrupt thread, store a timestamp in the thread
 202  * structure.  If an interrupt occurs between tsc_read() and its subsequent
 203  * store, the timestamp will be stale by the time it is stored.  We can detect
 204  * this by doing a compare-and-swap on the thread's timestamp, since any
 205  * interrupt occurring in this window will put a new timestamp in the thread's
 206  * t_intr_start field.
 207  */
 208 #define STORE_INTR_START(thread_t)                      \
 209         testw   $T_INTR_THREAD, T_FLAGS(thread_t);      \
 210         jz      1f;                                     \
 211         pushl   %ecx;                                   \
 212 0:                                                      \
 213         pushl   T_INTR_START(thread_t);                 \
 214         pushl   T_INTR_START+4(thread_t);               \
 215         call    tsc_read;                               \
 216         movl    %eax, %ebx;                             \
 217         movl    %edx, %ecx;                             \
 218         popl    %edx;                                   \
 219         popl    %eax;                                   \
 220         cmpxchg8b T_INTR_START(thread_t);               \
 221         jnz     0b;                                     \
 222         popl    %ecx;                                   \
 223 1:
 224 
 225 #endif  /* __amd64 */
 226 
 227 #if defined(__lint)
 228 
 229 /* ARGSUSED */
 230 void
 231 resume(kthread_t *t)
 232 {}
 233 
 234 #else   /* __lint */
 235 
 236 #if defined(__amd64)
 237 
 238         ENTRY(resume)
 239         movq    %gs:CPU_THREAD, %rax
 240         leaq    resume_return(%rip), %r11
 241 
 242         /*
 243          * Deal with SMAP here. A thread may be switched out at any point while
 244          * it is executing. The thread could be under on_fault() or it could be
 245          * pre-empted while performing a copy interruption. If this happens and
 246          * we're not in the context of an interrupt which happens to handle
 247          * saving and restoring rflags correctly, we may lose our SMAP related
 248          * state.
 249          *
 250          * To handle this, as part of being switched out, we first save whether
 251          * or not userland access is allowed ($PS_ACHK in rflags) and store that
 252          * in t_useracc on the kthread_t and unconditionally enable SMAP to
 253          * protect the system.
 254          *
 255          * Later, when the thread finishes resuming, we potentially disable smap
 256          * if PS_ACHK was present in rflags. See uts/intel/ia32/ml/copy.s for
 257          * more information on rflags and SMAP.
 258          */
 259         pushfq
 260         popq    %rsi
 261         andq    $PS_ACHK, %rsi
 262         movq    %rsi, T_USERACC(%rax)
 263         call    smap_enable
 264 
 265         /*
 266          * Save non-volatile registers, and set return address for current
 267          * thread to resume_return.
 268          *
 269          * %r12 = t (new thread) when done
 270          */
 271         SAVE_REGS(%rax, %r11)
 272 
 273 
 274         LOADCPU(%r15)                           /* %r15 = CPU */
 275         movq    CPU_THREAD(%r15), %r13          /* %r13 = curthread */
 276 
 277         /*
 278          * Call savectx if thread has installed context ops.
 279          *
 280          * Note that if we have floating point context, the save op
 281          * (either fpsave_begin or fpxsave_begin) will issue the
 282          * async save instruction (fnsave or fxsave respectively)
 283          * that we fwait for below.
 284          */
 285         cmpq    $0, T_CTX(%r13)         /* should current thread savectx? */
 286         je      .nosavectx              /* skip call when zero */
 287 
 288         movq    %r13, %rdi              /* arg = thread pointer */
 289         call    savectx                 /* call ctx ops */
 290 .nosavectx:
 291 
 292         /*
 293          * Call savepctx if process has installed context ops.
 294          */
 295         movq    T_PROCP(%r13), %r14     /* %r14 = proc */
 296         cmpq    $0, P_PCTX(%r14)         /* should current thread savectx? */
 297         je      .nosavepctx              /* skip call when zero */
 298 
 299         movq    %r14, %rdi              /* arg = proc pointer */
 300         call    savepctx                 /* call ctx ops */
 301 .nosavepctx:
 302 
 303         /*
 304          * Temporarily switch to the idle thread's stack
 305          */
 306         movq    CPU_IDLE_THREAD(%r15), %rax     /* idle thread pointer */
 307 
 308         /* 
 309          * Set the idle thread as the current thread
 310          */
 311         movq    T_SP(%rax), %rsp        /* It is safe to set rsp */
 312         movq    %rax, CPU_THREAD(%r15)
 313 
 314         /*
 315          * Switch in the hat context for the new thread
 316          *
 317          */
 318         GET_THREAD_HATP(%rdi, %r12, %r11)
 319         call    hat_switch
 320 
 321         /* 
 322          * Clear and unlock previous thread's t_lock
 323          * to allow it to be dispatched by another processor.
 324          */
 325         movb    $0, T_LOCK(%r13)
 326 
 327         /*
 328          * IMPORTANT: Registers at this point must be:
 329          *       %r12 = new thread
 330          *
 331          * Here we are in the idle thread, have dropped the old thread.
 332          */
 333         ALTENTRY(_resume_from_idle)
 334         /*
 335          * spin until dispatched thread's mutex has
 336          * been unlocked. this mutex is unlocked when
 337          * it becomes safe for the thread to run.
 338          */
 339 .lock_thread_mutex:
 340         lock
 341         btsl    $0, T_LOCK(%r12)        /* attempt to lock new thread's mutex */
 342         jnc     .thread_mutex_locked    /* got it */
 343 
 344 .spin_thread_mutex:
 345         pause
 346         cmpb    $0, T_LOCK(%r12)        /* check mutex status */
 347         jz      .lock_thread_mutex      /* clear, retry lock */
 348         jmp     .spin_thread_mutex      /* still locked, spin... */
 349 
 350 .thread_mutex_locked:
 351         /*
 352          * Fix CPU structure to indicate new running thread.
 353          * Set pointer in new thread to the CPU structure.
 354          */
 355         LOADCPU(%r13)                   /* load current CPU pointer */
 356         cmpq    %r13, T_CPU(%r12)
 357         je      .setup_cpu
 358 
 359         /* cp->cpu_stats.sys.cpumigrate++ */
 360         incq    CPU_STATS_SYS_CPUMIGRATE(%r13)
 361         movq    %r13, T_CPU(%r12)       /* set new thread's CPU pointer */
 362 
 363 .setup_cpu:
 364         /*
 365          * Setup rsp0 (kernel stack) in TSS to curthread's stack.
 366          * (Note: Since we don't have saved 'regs' structure for all
 367          *        the threads we can't easily determine if we need to
 368          *        change rsp0. So, we simply change the rsp0 to bottom 
 369          *        of the thread stack and it will work for all cases.)
 370          *
 371          * XX64 - Is this correct?
 372          */
 373         movq    CPU_TSS(%r13), %r14
 374         movq    T_STACK(%r12), %rax
 375         addq    $REGSIZE+MINFRAME, %rax /* to the bottom of thread stack */
 376 #if !defined(__xpv)
 377         movq    %rax, TSS_RSP0(%r14)
 378 #else
 379         movl    $KDS_SEL, %edi
 380         movq    %rax, %rsi
 381         call    HYPERVISOR_stack_switch
 382 #endif  /* __xpv */
 383 
 384         movq    %r12, CPU_THREAD(%r13)  /* set CPU's thread pointer */
 385         mfence                          /* synchronize with mutex_exit() */
 386         xorl    %ebp, %ebp              /* make $<threadlist behave better */
 387         movq    T_LWP(%r12), %rax       /* set associated lwp to  */
 388         movq    %rax, CPU_LWP(%r13)     /* CPU's lwp ptr */
 389 
 390         movq    T_SP(%r12), %rsp        /* switch to outgoing thread's stack */
 391         movq    T_PC(%r12), %r13        /* saved return addr */
 392 
 393         /*
 394          * Call restorectx if context ops have been installed.
 395          */
 396         cmpq    $0, T_CTX(%r12)         /* should resumed thread restorectx? */
 397         jz      .norestorectx           /* skip call when zero */
 398         movq    %r12, %rdi              /* arg = thread pointer */
 399         call    restorectx              /* call ctx ops */
 400 .norestorectx:
 401 
 402         /*
 403          * Call restorepctx if context ops have been installed for the proc.
 404          */
 405         movq    T_PROCP(%r12), %rcx
 406         cmpq    $0, P_PCTX(%rcx)
 407         jz      .norestorepctx
 408         movq    %rcx, %rdi
 409         call    restorepctx
 410 .norestorepctx:
 411         
 412         STORE_INTR_START(%r12)
 413 
 414         /*
 415          * If we came into swtch with the ability to access userland pages, go
 416          * ahead and restore that fact by disabling SMAP.  Clear the indicator
 417          * flag out of paranoia.
 418          */
 419         movq    T_USERACC(%r12), %rax   /* should we disable smap? */
 420         cmpq    $0, %rax                /* skip call when zero */
 421         jz      .nosmap
 422         xorq    %rax, %rax
 423         movq    %rax, T_USERACC(%r12)
 424         call    smap_disable
 425 .nosmap:
 426 
 427         /*
 428          * Restore non-volatile registers, then have spl0 return to the
 429          * resuming thread's PC after first setting the priority as low as
 430          * possible and blocking all interrupt threads that may be active.
 431          */
 432         movq    %r13, %rax      /* save return address */       
 433         RESTORE_REGS(%r11)
 434         pushq   %rax            /* push return address for spl0() */
 435         call    __dtrace_probe___sched_on__cpu
 436         jmp     spl0
 437 
 438 resume_return:
 439         /*
 440          * Remove stack frame created in SAVE_REGS()
 441          */
 442         addq    $CLONGSIZE, %rsp
 443         ret
 444         SET_SIZE(_resume_from_idle)
 445         SET_SIZE(resume)
 446 
 447 #elif defined (__i386)
 448 
 449         ENTRY(resume)
 450         movl    %gs:CPU_THREAD, %eax
 451         movl    $resume_return, %ecx
 452 
 453         /*
 454          * Save non-volatile registers, and set return address for current
 455          * thread to resume_return.
 456          *
 457          * %edi = t (new thread) when done.
 458          */
 459         SAVE_REGS(%eax,  %ecx)
 460 
 461         LOADCPU(%ebx)                   /* %ebx = CPU */
 462         movl    CPU_THREAD(%ebx), %esi  /* %esi = curthread */
 463 
 464 #ifdef DEBUG
 465         call    assert_ints_enabled     /* panics if we are cli'd */
 466 #endif
 467         /*
 468          * Call savectx if thread has installed context ops.
 469          *
 470          * Note that if we have floating point context, the save op
 471          * (either fpsave_begin or fpxsave_begin) will issue the
 472          * async save instruction (fnsave or fxsave respectively)
 473          * that we fwait for below.
 474          */
 475         movl    T_CTX(%esi), %eax       /* should current thread savectx? */
 476         testl   %eax, %eax
 477         jz      .nosavectx              /* skip call when zero */
 478         pushl   %esi                    /* arg = thread pointer */
 479         call    savectx                 /* call ctx ops */
 480         addl    $4, %esp                /* restore stack pointer */
 481 .nosavectx:
 482 
 483         /*
 484          * Call savepctx if process has installed context ops.
 485          */
 486         movl    T_PROCP(%esi), %eax     /* %eax = proc */
 487         cmpl    $0, P_PCTX(%eax)        /* should current thread savectx? */
 488         je      .nosavepctx             /* skip call when zero */
 489         pushl   %eax                    /* arg = proc pointer */
 490         call    savepctx                /* call ctx ops */
 491         addl    $4, %esp
 492 .nosavepctx:
 493 
 494         /* 
 495          * Temporarily switch to the idle thread's stack
 496          */
 497         movl    CPU_IDLE_THREAD(%ebx), %eax     /* idle thread pointer */
 498 
 499         /* 
 500          * Set the idle thread as the current thread
 501          */
 502         movl    T_SP(%eax), %esp        /* It is safe to set esp */
 503         movl    %eax, CPU_THREAD(%ebx)
 504 
 505         /* switch in the hat context for the new thread */
 506         GET_THREAD_HATP(%ecx, %edi, %ecx)
 507         pushl   %ecx
 508         call    hat_switch
 509         addl    $4, %esp
 510         
 511         /* 
 512          * Clear and unlock previous thread's t_lock
 513          * to allow it to be dispatched by another processor.
 514          */
 515         movb    $0, T_LOCK(%esi)
 516 
 517         /*
 518          * IMPORTANT: Registers at this point must be:
 519          *       %edi = new thread
 520          *
 521          * Here we are in the idle thread, have dropped the old thread.
 522          */
 523         ALTENTRY(_resume_from_idle)
 524         /*
 525          * spin until dispatched thread's mutex has
 526          * been unlocked. this mutex is unlocked when
 527          * it becomes safe for the thread to run.
 528          */
 529 .L4:
 530         lock
 531         btsl    $0, T_LOCK(%edi) /* lock new thread's mutex */
 532         jc      .L4_2                   /* lock did not succeed */
 533 
 534         /*
 535          * Fix CPU structure to indicate new running thread.
 536          * Set pointer in new thread to the CPU structure.
 537          */
 538         LOADCPU(%esi)                   /* load current CPU pointer */
 539         movl    T_STACK(%edi), %eax     /* here to use v pipeline of */
 540                                         /* Pentium. Used few lines below */
 541         cmpl    %esi, T_CPU(%edi)
 542         jne     .L5_2
 543 .L5_1:
 544         /*
 545          * Setup esp0 (kernel stack) in TSS to curthread's stack.
 546          * (Note: Since we don't have saved 'regs' structure for all
 547          *        the threads we can't easily determine if we need to
 548          *        change esp0. So, we simply change the esp0 to bottom 
 549          *        of the thread stack and it will work for all cases.)
 550          */
 551         movl    CPU_TSS(%esi), %ecx
 552         addl    $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */
 553 #if !defined(__xpv)
 554         movl    %eax, TSS_ESP0(%ecx)
 555 #else
 556         pushl   %eax
 557         pushl   $KDS_SEL
 558         call    HYPERVISOR_stack_switch
 559         addl    $8, %esp
 560 #endif  /* __xpv */
 561 
 562         movl    %edi, CPU_THREAD(%esi)  /* set CPU's thread pointer */
 563         mfence                          /* synchronize with mutex_exit() */
 564         xorl    %ebp, %ebp              /* make $<threadlist behave better */
 565         movl    T_LWP(%edi), %eax       /* set associated lwp to  */
 566         movl    %eax, CPU_LWP(%esi)     /* CPU's lwp ptr */
 567 
 568         movl    T_SP(%edi), %esp        /* switch to outgoing thread's stack */
 569         movl    T_PC(%edi), %esi        /* saved return addr */
 570 
 571         /*
 572          * Call restorectx if context ops have been installed.
 573          */
 574         movl    T_CTX(%edi), %eax       /* should resumed thread restorectx? */
 575         testl   %eax, %eax
 576         jz      .norestorectx           /* skip call when zero */
 577         pushl   %edi                    /* arg = thread pointer */
 578         call    restorectx              /* call ctx ops */
 579         addl    $4, %esp                /* restore stack pointer */
 580 .norestorectx:
 581 
 582         /*
 583          * Call restorepctx if context ops have been installed for the proc.
 584          */
 585         movl    T_PROCP(%edi), %eax
 586         cmpl    $0, P_PCTX(%eax)
 587         je      .norestorepctx
 588         pushl   %eax                    /* arg = proc pointer */
 589         call    restorepctx
 590         addl    $4, %esp                /* restore stack pointer */
 591 .norestorepctx:
 592 
 593         STORE_INTR_START(%edi)
 594 
 595         /*
 596          * Restore non-volatile registers, then have spl0 return to the
 597          * resuming thread's PC after first setting the priority as low as
 598          * possible and blocking all interrupt threads that may be active.
 599          */
 600         movl    %esi, %eax              /* save return address */
 601         RESTORE_REGS(%ecx)
 602         pushl   %eax                    /* push return address for spl0() */
 603         call    __dtrace_probe___sched_on__cpu
 604         jmp     spl0
 605 
 606 resume_return:
 607         /*
 608          * Remove stack frame created in SAVE_REGS()
 609          */
 610         addl    $CLONGSIZE, %esp
 611         ret
 612 
 613 .L4_2:
 614         pause
 615         cmpb    $0, T_LOCK(%edi)
 616         je      .L4
 617         jmp     .L4_2
 618 
 619 .L5_2:
 620         /* cp->cpu_stats.sys.cpumigrate++ */
 621         addl    $1, CPU_STATS_SYS_CPUMIGRATE(%esi)
 622         adcl    $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi)
 623         movl    %esi, T_CPU(%edi)       /* set new thread's CPU pointer */
 624         jmp     .L5_1
 625 
 626         SET_SIZE(_resume_from_idle)
 627         SET_SIZE(resume)
 628 
 629 #endif  /* __amd64 */
 630 #endif  /* __lint */
 631 
 632 #if defined(__lint)
 633 
 634 /* ARGSUSED */
 635 void
 636 resume_from_zombie(kthread_t *t)
 637 {}
 638 
 639 #else   /* __lint */
 640 
 641 #if defined(__amd64)
 642 
 643         ENTRY(resume_from_zombie)
 644         movq    %gs:CPU_THREAD, %rax
 645         leaq    resume_from_zombie_return(%rip), %r11
 646 
 647         /*
 648          * Save non-volatile registers, and set return address for current
 649          * thread to resume_from_zombie_return.
 650          *
 651          * %r12 = t (new thread) when done
 652          */
 653         SAVE_REGS(%rax, %r11)
 654 
 655         movq    %gs:CPU_THREAD, %r13    /* %r13 = curthread */
 656 
 657         /* clean up the fp unit. It might be left enabled */
 658 
 659 #if defined(__xpv)              /* XXPV XXtclayton */
 660         /*
 661          * Remove this after bringup.
 662          * (Too many #gp's for an instrumented hypervisor.)
 663          */
 664         STTS(%rax)
 665 #else
 666         movq    %cr0, %rax
 667         testq   $CR0_TS, %rax
 668         jnz     .zfpu_disabled          /* if TS already set, nothing to do */
 669         fninit                          /* init fpu & discard pending error */
 670         orq     $CR0_TS, %rax
 671         movq    %rax, %cr0
 672 .zfpu_disabled:
 673 
 674 #endif  /* __xpv */
 675 
 676         /* 
 677          * Temporarily switch to the idle thread's stack so that the zombie
 678          * thread's stack can be reclaimed by the reaper.
 679          */
 680         movq    %gs:CPU_IDLE_THREAD, %rax /* idle thread pointer */
 681         movq    T_SP(%rax), %rsp        /* get onto idle thread stack */
 682 
 683         /*
 684          * Sigh. If the idle thread has never run thread_start()
 685          * then t_sp is mis-aligned by thread_load().
 686          */
 687         andq    $_BITNOT(STACK_ALIGN-1), %rsp
 688 
 689         /* 
 690          * Set the idle thread as the current thread.
 691          */
 692         movq    %rax, %gs:CPU_THREAD
 693 
 694         /* switch in the hat context for the new thread */
 695         GET_THREAD_HATP(%rdi, %r12, %r11)
 696         call    hat_switch
 697 
 698         /* 
 699          * Put the zombie on death-row.
 700          */
 701         movq    %r13, %rdi
 702         call    reapq_add
 703 
 704         jmp     _resume_from_idle       /* finish job of resume */
 705 
 706 resume_from_zombie_return:
 707         RESTORE_REGS(%r11)              /* restore non-volatile registers */
 708         call    __dtrace_probe___sched_on__cpu
 709 
 710         /*
 711          * Remove stack frame created in SAVE_REGS()
 712          */
 713         addq    $CLONGSIZE, %rsp
 714         ret
 715         SET_SIZE(resume_from_zombie)
 716 
 717 #elif defined (__i386)
 718 
 719         ENTRY(resume_from_zombie)
 720         movl    %gs:CPU_THREAD, %eax
 721         movl    $resume_from_zombie_return, %ecx
 722 
 723         /*
 724          * Save non-volatile registers, and set return address for current
 725          * thread to resume_from_zombie_return.
 726          *
 727          * %edi = t (new thread) when done.
 728          */
 729         SAVE_REGS(%eax, %ecx)
 730 
 731 #ifdef DEBUG
 732         call    assert_ints_enabled     /* panics if we are cli'd */
 733 #endif
 734         movl    %gs:CPU_THREAD, %esi    /* %esi = curthread */
 735 
 736         /* clean up the fp unit. It might be left enabled */
 737 
 738         movl    %cr0, %eax
 739         testl   $CR0_TS, %eax
 740         jnz     .zfpu_disabled          /* if TS already set, nothing to do */
 741         fninit                          /* init fpu & discard pending error */
 742         orl     $CR0_TS, %eax
 743         movl    %eax, %cr0
 744 .zfpu_disabled:
 745 
 746         /* 
 747          * Temporarily switch to the idle thread's stack so that the zombie
 748          * thread's stack can be reclaimed by the reaper.
 749          */
 750         movl    %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */
 751         movl    T_SP(%eax), %esp        /* get onto idle thread stack */
 752 
 753         /* 
 754          * Set the idle thread as the current thread.
 755          */
 756         movl    %eax, %gs:CPU_THREAD
 757 
 758         /*
 759          * switch in the hat context for the new thread
 760          */
 761         GET_THREAD_HATP(%ecx, %edi, %ecx)
 762         pushl   %ecx
 763         call    hat_switch
 764         addl    $4, %esp
 765 
 766         /* 
 767          * Put the zombie on death-row.
 768          */
 769         pushl   %esi
 770         call    reapq_add
 771         addl    $4, %esp
 772         jmp     _resume_from_idle       /* finish job of resume */
 773 
 774 resume_from_zombie_return:
 775         RESTORE_REGS(%ecx)              /* restore non-volatile registers */
 776         call    __dtrace_probe___sched_on__cpu
 777 
 778         /*
 779          * Remove stack frame created in SAVE_REGS()
 780          */
 781         addl    $CLONGSIZE, %esp
 782         ret
 783         SET_SIZE(resume_from_zombie)
 784 
 785 #endif  /* __amd64 */
 786 #endif  /* __lint */
 787 
 788 #if defined(__lint)
 789 
 790 /* ARGSUSED */
 791 void
 792 resume_from_intr(kthread_t *t)
 793 {}
 794 
 795 #else   /* __lint */
 796 
 797 #if defined(__amd64)
 798 
 799         ENTRY(resume_from_intr)
 800         movq    %gs:CPU_THREAD, %rax
 801         leaq    resume_from_intr_return(%rip), %r11
 802 
 803         /*
 804          * Save non-volatile registers, and set return address for current
 805          * thread to resume_from_intr_return.
 806          *
 807          * %r12 = t (new thread) when done
 808          */
 809         SAVE_REGS(%rax, %r11)
 810 
 811         movq    %gs:CPU_THREAD, %r13    /* %r13 = curthread */
 812         movq    %r12, %gs:CPU_THREAD    /* set CPU's thread pointer */
 813         mfence                          /* synchronize with mutex_exit() */
 814         movq    T_SP(%r12), %rsp        /* restore resuming thread's sp */
 815         xorl    %ebp, %ebp              /* make $<threadlist behave better */
 816 
 817         /* 
 818          * Unlock outgoing thread's mutex dispatched by another processor.
 819          */
 820         xorl    %eax, %eax
 821         xchgb   %al, T_LOCK(%r13)
 822 
 823         STORE_INTR_START(%r12)
 824 
 825         /*
 826          * Restore non-volatile registers, then have spl0 return to the
 827          * resuming thread's PC after first setting the priority as low as
 828          * possible and blocking all interrupt threads that may be active.
 829          */
 830         movq    T_PC(%r12), %rax        /* saved return addr */
 831         RESTORE_REGS(%r11);
 832         pushq   %rax                    /* push return address for spl0() */
 833         call    __dtrace_probe___sched_on__cpu
 834         jmp     spl0
 835 
 836 resume_from_intr_return:
 837         /*
 838          * Remove stack frame created in SAVE_REGS()
 839          */
 840         addq    $CLONGSIZE, %rsp
 841         ret
 842         SET_SIZE(resume_from_intr)
 843 
 844 #elif defined (__i386)
 845 
 846         ENTRY(resume_from_intr)
 847         movl    %gs:CPU_THREAD, %eax
 848         movl    $resume_from_intr_return, %ecx
 849 
 850         /*
 851          * Save non-volatile registers, and set return address for current
 852          * thread to resume_return.
 853          *
 854          * %edi = t (new thread) when done.
 855          */
 856         SAVE_REGS(%eax, %ecx)
 857 
 858 #ifdef DEBUG
 859         call    assert_ints_enabled     /* panics if we are cli'd */
 860 #endif
 861         movl    %gs:CPU_THREAD, %esi    /* %esi = curthread */
 862         movl    %edi, %gs:CPU_THREAD    /* set CPU's thread pointer */
 863         mfence                          /* synchronize with mutex_exit() */
 864         movl    T_SP(%edi), %esp        /* restore resuming thread's sp */
 865         xorl    %ebp, %ebp              /* make $<threadlist behave better */
 866 
 867         /* 
 868          * Unlock outgoing thread's mutex dispatched by another processor.
 869          */
 870         xorl    %eax,%eax
 871         xchgb   %al, T_LOCK(%esi)
 872 
 873         STORE_INTR_START(%edi)
 874 
 875         /*
 876          * Restore non-volatile registers, then have spl0 return to the
 877          * resuming thread's PC after first setting the priority as low as
 878          * possible and blocking all interrupt threads that may be active.
 879          */
 880         movl    T_PC(%edi), %eax        /* saved return addr */
 881         RESTORE_REGS(%ecx)
 882         pushl   %eax                    /* push return address for spl0() */
 883         call    __dtrace_probe___sched_on__cpu
 884         jmp     spl0
 885 
 886 resume_from_intr_return:
 887         /*
 888          * Remove stack frame created in SAVE_REGS()
 889          */
 890         addl    $CLONGSIZE, %esp
 891         ret
 892         SET_SIZE(resume_from_intr)
 893 
 894 #endif  /* __amd64 */
 895 #endif /* __lint */
 896 
 897 #if defined(__lint)
 898 
 899 void
 900 thread_start(void)
 901 {}
 902 
 903 #else   /* __lint */
 904 
 905 #if defined(__amd64)
 906 
 907         ENTRY(thread_start)
 908         popq    %rax            /* start() */
 909         popq    %rdi            /* arg */
 910         popq    %rsi            /* len */
 911         movq    %rsp, %rbp
 912         call    *%rax
 913         call    thread_exit     /* destroy thread if it returns. */
 914         /*NOTREACHED*/
 915         SET_SIZE(thread_start)
 916 
 917 #elif defined(__i386)
 918 
 919         ENTRY(thread_start)
 920         popl    %eax
 921         movl    %esp, %ebp
 922         addl    $8, %ebp
 923         call    *%eax
 924         addl    $8, %esp
 925         call    thread_exit     /* destroy thread if it returns. */
 926         /*NOTREACHED*/
 927         SET_SIZE(thread_start)
 928 
 929 #endif  /* __i386 */
 930 
 931 #endif  /* __lint */