1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
28 */
29
30 /*
31 * Process switching routines.
32 */
33
34 #if defined(__lint)
35 #include <sys/thread.h>
36 #include <sys/systm.h>
37 #include <sys/time.h>
38 #else /* __lint */
39 #include "assym.h"
40 #endif /* __lint */
41
42 #include <sys/asm_linkage.h>
43 #include <sys/asm_misc.h>
44 #include <sys/regset.h>
45 #include <sys/privregs.h>
46 #include <sys/stack.h>
47 #include <sys/segments.h>
48 #include <sys/psw.h>
49
50 /*
51 * resume(thread_id_t t);
52 *
53 * a thread can only run on one processor at a time. there
54 * exists a window on MPs where the current thread on one
55 * processor is capable of being dispatched by another processor.
56 * some overlap between outgoing and incoming threads can happen
57 * when they are the same thread. in this case where the threads
58 * are the same, resume() on one processor will spin on the incoming
59 * thread until resume() on the other processor has finished with
60 * the outgoing thread.
61 *
62 * The MMU context changes when the resuming thread resides in a different
63 * process. Kernel threads are known by resume to reside in process 0.
64 * The MMU context, therefore, only changes when resuming a thread in
65 * a process different from curproc.
66 *
67 * resume_from_intr() is called when the thread being resumed was not
68 * passivated by resume (e.g. was interrupted). This means that the
69 * resume lock is already held and that a restore context is not needed.
70 * Also, the MMU context is not changed on the resume in this case.
71 *
72 * resume_from_zombie() is the same as resume except the calling thread
73 * is a zombie and must be put on the deathrow list after the CPU is
74 * off the stack.
75 */
76
77 #if !defined(__lint)
78
79 #if LWP_PCB_FPU != 0
80 #error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work
81 #endif /* LWP_PCB_FPU != 0 */
82
83 #endif /* !__lint */
84
85 #if defined(__amd64)
86
87 /*
88 * Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
89 *
90 * The stack frame must be created before the save of %rsp so that tracebacks
91 * of swtch()ed-out processes show the process as having last called swtch().
92 */
93 #define SAVE_REGS(thread_t, retaddr) \
94 movq %rbp, T_RBP(thread_t); \
95 movq %rbx, T_RBX(thread_t); \
96 movq %r12, T_R12(thread_t); \
97 movq %r13, T_R13(thread_t); \
98 movq %r14, T_R14(thread_t); \
99 movq %r15, T_R15(thread_t); \
100 pushq %rbp; \
101 movq %rsp, %rbp; \
102 movq %rsp, T_SP(thread_t); \
103 movq retaddr, T_PC(thread_t); \
104 movq %rdi, %r12; \
105 call __dtrace_probe___sched_off__cpu
106
107 /*
108 * Restore non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
109 *
110 * We load up %rsp from the label_t as part of the context switch, so
111 * we don't repeat that here.
112 *
113 * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
114 * already has the effect of putting the stack back the way it was when
115 * we came in.
116 */
117 #define RESTORE_REGS(scratch_reg) \
118 movq %gs:CPU_THREAD, scratch_reg; \
119 movq T_RBP(scratch_reg), %rbp; \
120 movq T_RBX(scratch_reg), %rbx; \
121 movq T_R12(scratch_reg), %r12; \
122 movq T_R13(scratch_reg), %r13; \
123 movq T_R14(scratch_reg), %r14; \
124 movq T_R15(scratch_reg), %r15
125
126 /*
127 * Get pointer to a thread's hat structure
128 */
129 #define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \
130 movq T_PROCP(thread_t), hatp; \
131 movq P_AS(hatp), scratch_reg; \
132 movq A_HAT(scratch_reg), hatp
133
134 #define TSC_READ() \
135 call tsc_read; \
136 movq %rax, %r14;
137
138 /*
139 * If we are resuming an interrupt thread, store a timestamp in the thread
140 * structure. If an interrupt occurs between tsc_read() and its subsequent
141 * store, the timestamp will be stale by the time it is stored. We can detect
142 * this by doing a compare-and-swap on the thread's timestamp, since any
143 * interrupt occurring in this window will put a new timestamp in the thread's
144 * t_intr_start field.
145 */
146 #define STORE_INTR_START(thread_t) \
147 testw $T_INTR_THREAD, T_FLAGS(thread_t); \
148 jz 1f; \
149 0: \
150 TSC_READ(); \
151 movq T_INTR_START(thread_t), %rax; \
152 cmpxchgq %r14, T_INTR_START(thread_t); \
153 jnz 0b; \
154 1:
155
156 #elif defined (__i386)
157
158 /*
159 * Save non-volatile registers (%ebp, %esi, %edi and %ebx)
160 *
161 * The stack frame must be created before the save of %esp so that tracebacks
162 * of swtch()ed-out processes show the process as having last called swtch().
163 */
164 #define SAVE_REGS(thread_t, retaddr) \
165 movl %ebp, T_EBP(thread_t); \
166 movl %ebx, T_EBX(thread_t); \
167 movl %esi, T_ESI(thread_t); \
168 movl %edi, T_EDI(thread_t); \
169 pushl %ebp; \
170 movl %esp, %ebp; \
171 movl %esp, T_SP(thread_t); \
172 movl retaddr, T_PC(thread_t); \
173 movl 8(%ebp), %edi; \
174 pushl %edi; \
175 call __dtrace_probe___sched_off__cpu; \
176 addl $CLONGSIZE, %esp
177
178 /*
179 * Restore non-volatile registers (%ebp, %esi, %edi and %ebx)
180 *
181 * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
182 * already has the effect of putting the stack back the way it was when
183 * we came in.
184 */
185 #define RESTORE_REGS(scratch_reg) \
186 movl %gs:CPU_THREAD, scratch_reg; \
187 movl T_EBP(scratch_reg), %ebp; \
188 movl T_EBX(scratch_reg), %ebx; \
189 movl T_ESI(scratch_reg), %esi; \
190 movl T_EDI(scratch_reg), %edi
191
192 /*
193 * Get pointer to a thread's hat structure
194 */
195 #define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \
196 movl T_PROCP(thread_t), hatp; \
197 movl P_AS(hatp), scratch_reg; \
198 movl A_HAT(scratch_reg), hatp
199
200 /*
201 * If we are resuming an interrupt thread, store a timestamp in the thread
202 * structure. If an interrupt occurs between tsc_read() and its subsequent
203 * store, the timestamp will be stale by the time it is stored. We can detect
204 * this by doing a compare-and-swap on the thread's timestamp, since any
205 * interrupt occurring in this window will put a new timestamp in the thread's
206 * t_intr_start field.
207 */
208 #define STORE_INTR_START(thread_t) \
209 testw $T_INTR_THREAD, T_FLAGS(thread_t); \
210 jz 1f; \
211 pushl %ecx; \
212 0: \
213 pushl T_INTR_START(thread_t); \
214 pushl T_INTR_START+4(thread_t); \
215 call tsc_read; \
216 movl %eax, %ebx; \
217 movl %edx, %ecx; \
218 popl %edx; \
219 popl %eax; \
220 cmpxchg8b T_INTR_START(thread_t); \
221 jnz 0b; \
222 popl %ecx; \
223 1:
224
225 #endif /* __amd64 */
226
227 #if defined(__lint)
228
229 /* ARGSUSED */
230 void
231 resume(kthread_t *t)
232 {}
233
234 #else /* __lint */
235
236 #if defined(__amd64)
237
238 ENTRY(resume)
239 movq %gs:CPU_THREAD, %rax
240 leaq resume_return(%rip), %r11
241
242 /*
243 * Deal with SMAP here. A thread may be switched out at any point while
244 * it is executing. The thread could be under on_fault() or it could be
245 * pre-empted while performing a copy interruption. If this happens and
246 * we're not in the context of an interrupt which happens to handle
247 * saving and restoring rflags correctly, we may lose our SMAP related
248 * state.
249 *
250 * To handle this, as part of being switched out, we first save whether
251 * or not userland access is allowed ($PS_ACHK in rflags) and store that
252 * in t_useracc on the kthread_t and unconditionally enable SMAP to
253 * protect the system.
254 *
255 * Later, when the thread finishes resuming, we potentially disable smap
256 * if PS_ACHK was present in rflags. See uts/intel/ia32/ml/copy.s for
257 * more information on rflags and SMAP.
258 */
259 pushfq
260 popq %rsi
261 andq $PS_ACHK, %rsi
262 movq %rsi, T_USERACC(%rax)
263 call smap_enable
264
265 /*
266 * Save non-volatile registers, and set return address for current
267 * thread to resume_return.
268 *
269 * %r12 = t (new thread) when done
270 */
271 SAVE_REGS(%rax, %r11)
272
273
274 LOADCPU(%r15) /* %r15 = CPU */
275 movq CPU_THREAD(%r15), %r13 /* %r13 = curthread */
276
277 /*
278 * Call savectx if thread has installed context ops.
279 *
280 * Note that if we have floating point context, the save op
281 * (either fpsave_begin or fpxsave_begin) will issue the
282 * async save instruction (fnsave or fxsave respectively)
283 * that we fwait for below.
284 */
285 cmpq $0, T_CTX(%r13) /* should current thread savectx? */
286 je .nosavectx /* skip call when zero */
287
288 movq %r13, %rdi /* arg = thread pointer */
289 call savectx /* call ctx ops */
290 .nosavectx:
291
292 /*
293 * Call savepctx if process has installed context ops.
294 */
295 movq T_PROCP(%r13), %r14 /* %r14 = proc */
296 cmpq $0, P_PCTX(%r14) /* should current thread savectx? */
297 je .nosavepctx /* skip call when zero */
298
299 movq %r14, %rdi /* arg = proc pointer */
300 call savepctx /* call ctx ops */
301 .nosavepctx:
302
303 /*
304 * Temporarily switch to the idle thread's stack
305 */
306 movq CPU_IDLE_THREAD(%r15), %rax /* idle thread pointer */
307
308 /*
309 * Set the idle thread as the current thread
310 */
311 movq T_SP(%rax), %rsp /* It is safe to set rsp */
312 movq %rax, CPU_THREAD(%r15)
313
314 /*
315 * Switch in the hat context for the new thread
316 *
317 */
318 GET_THREAD_HATP(%rdi, %r12, %r11)
319 call hat_switch
320
321 /*
322 * Clear and unlock previous thread's t_lock
323 * to allow it to be dispatched by another processor.
324 */
325 movb $0, T_LOCK(%r13)
326
327 /*
328 * IMPORTANT: Registers at this point must be:
329 * %r12 = new thread
330 *
331 * Here we are in the idle thread, have dropped the old thread.
332 */
333 ALTENTRY(_resume_from_idle)
334 /*
335 * spin until dispatched thread's mutex has
336 * been unlocked. this mutex is unlocked when
337 * it becomes safe for the thread to run.
338 */
339 .lock_thread_mutex:
340 lock
341 btsl $0, T_LOCK(%r12) /* attempt to lock new thread's mutex */
342 jnc .thread_mutex_locked /* got it */
343
344 .spin_thread_mutex:
345 pause
346 cmpb $0, T_LOCK(%r12) /* check mutex status */
347 jz .lock_thread_mutex /* clear, retry lock */
348 jmp .spin_thread_mutex /* still locked, spin... */
349
350 .thread_mutex_locked:
351 /*
352 * Fix CPU structure to indicate new running thread.
353 * Set pointer in new thread to the CPU structure.
354 */
355 LOADCPU(%r13) /* load current CPU pointer */
356 cmpq %r13, T_CPU(%r12)
357 je .setup_cpu
358
359 /* cp->cpu_stats.sys.cpumigrate++ */
360 incq CPU_STATS_SYS_CPUMIGRATE(%r13)
361 movq %r13, T_CPU(%r12) /* set new thread's CPU pointer */
362
363 .setup_cpu:
364 /*
365 * Setup rsp0 (kernel stack) in TSS to curthread's stack.
366 * (Note: Since we don't have saved 'regs' structure for all
367 * the threads we can't easily determine if we need to
368 * change rsp0. So, we simply change the rsp0 to bottom
369 * of the thread stack and it will work for all cases.)
370 *
371 * XX64 - Is this correct?
372 */
373 movq CPU_TSS(%r13), %r14
374 movq T_STACK(%r12), %rax
375 addq $REGSIZE+MINFRAME, %rax /* to the bottom of thread stack */
376 #if !defined(__xpv)
377 movq %rax, TSS_RSP0(%r14)
378 #else
379 movl $KDS_SEL, %edi
380 movq %rax, %rsi
381 call HYPERVISOR_stack_switch
382 #endif /* __xpv */
383
384 movq %r12, CPU_THREAD(%r13) /* set CPU's thread pointer */
385 mfence /* synchronize with mutex_exit() */
386 xorl %ebp, %ebp /* make $<threadlist behave better */
387 movq T_LWP(%r12), %rax /* set associated lwp to */
388 movq %rax, CPU_LWP(%r13) /* CPU's lwp ptr */
389
390 movq T_SP(%r12), %rsp /* switch to outgoing thread's stack */
391 movq T_PC(%r12), %r13 /* saved return addr */
392
393 /*
394 * Call restorectx if context ops have been installed.
395 */
396 cmpq $0, T_CTX(%r12) /* should resumed thread restorectx? */
397 jz .norestorectx /* skip call when zero */
398 movq %r12, %rdi /* arg = thread pointer */
399 call restorectx /* call ctx ops */
400 .norestorectx:
401
402 /*
403 * Call restorepctx if context ops have been installed for the proc.
404 */
405 movq T_PROCP(%r12), %rcx
406 cmpq $0, P_PCTX(%rcx)
407 jz .norestorepctx
408 movq %rcx, %rdi
409 call restorepctx
410 .norestorepctx:
411
412 STORE_INTR_START(%r12)
413
414 /*
415 * If we came into swtch with the ability to access userland pages, go
416 * ahead and restore that fact by disabling SMAP. Clear the indicator
417 * flag out of paranoia.
418 */
419 movq T_USERACC(%r12), %rax /* should we disable smap? */
420 cmpq $0, %rax /* skip call when zero */
421 jz .nosmap
422 xorq %rax, %rax
423 movq %rax, T_USERACC(%r12)
424 call smap_disable
425 .nosmap:
426
427 /*
428 * Restore non-volatile registers, then have spl0 return to the
429 * resuming thread's PC after first setting the priority as low as
430 * possible and blocking all interrupt threads that may be active.
431 */
432 movq %r13, %rax /* save return address */
433 RESTORE_REGS(%r11)
434 pushq %rax /* push return address for spl0() */
435 call __dtrace_probe___sched_on__cpu
436 jmp spl0
437
438 resume_return:
439 /*
440 * Remove stack frame created in SAVE_REGS()
441 */
442 addq $CLONGSIZE, %rsp
443 ret
444 SET_SIZE(_resume_from_idle)
445 SET_SIZE(resume)
446
447 #elif defined (__i386)
448
449 ENTRY(resume)
450 movl %gs:CPU_THREAD, %eax
451 movl $resume_return, %ecx
452
453 /*
454 * Save non-volatile registers, and set return address for current
455 * thread to resume_return.
456 *
457 * %edi = t (new thread) when done.
458 */
459 SAVE_REGS(%eax, %ecx)
460
461 LOADCPU(%ebx) /* %ebx = CPU */
462 movl CPU_THREAD(%ebx), %esi /* %esi = curthread */
463
464 #ifdef DEBUG
465 call assert_ints_enabled /* panics if we are cli'd */
466 #endif
467 /*
468 * Call savectx if thread has installed context ops.
469 *
470 * Note that if we have floating point context, the save op
471 * (either fpsave_begin or fpxsave_begin) will issue the
472 * async save instruction (fnsave or fxsave respectively)
473 * that we fwait for below.
474 */
475 movl T_CTX(%esi), %eax /* should current thread savectx? */
476 testl %eax, %eax
477 jz .nosavectx /* skip call when zero */
478 pushl %esi /* arg = thread pointer */
479 call savectx /* call ctx ops */
480 addl $4, %esp /* restore stack pointer */
481 .nosavectx:
482
483 /*
484 * Call savepctx if process has installed context ops.
485 */
486 movl T_PROCP(%esi), %eax /* %eax = proc */
487 cmpl $0, P_PCTX(%eax) /* should current thread savectx? */
488 je .nosavepctx /* skip call when zero */
489 pushl %eax /* arg = proc pointer */
490 call savepctx /* call ctx ops */
491 addl $4, %esp
492 .nosavepctx:
493
494 /*
495 * Temporarily switch to the idle thread's stack
496 */
497 movl CPU_IDLE_THREAD(%ebx), %eax /* idle thread pointer */
498
499 /*
500 * Set the idle thread as the current thread
501 */
502 movl T_SP(%eax), %esp /* It is safe to set esp */
503 movl %eax, CPU_THREAD(%ebx)
504
505 /* switch in the hat context for the new thread */
506 GET_THREAD_HATP(%ecx, %edi, %ecx)
507 pushl %ecx
508 call hat_switch
509 addl $4, %esp
510
511 /*
512 * Clear and unlock previous thread's t_lock
513 * to allow it to be dispatched by another processor.
514 */
515 movb $0, T_LOCK(%esi)
516
517 /*
518 * IMPORTANT: Registers at this point must be:
519 * %edi = new thread
520 *
521 * Here we are in the idle thread, have dropped the old thread.
522 */
523 ALTENTRY(_resume_from_idle)
524 /*
525 * spin until dispatched thread's mutex has
526 * been unlocked. this mutex is unlocked when
527 * it becomes safe for the thread to run.
528 */
529 .L4:
530 lock
531 btsl $0, T_LOCK(%edi) /* lock new thread's mutex */
532 jc .L4_2 /* lock did not succeed */
533
534 /*
535 * Fix CPU structure to indicate new running thread.
536 * Set pointer in new thread to the CPU structure.
537 */
538 LOADCPU(%esi) /* load current CPU pointer */
539 movl T_STACK(%edi), %eax /* here to use v pipeline of */
540 /* Pentium. Used few lines below */
541 cmpl %esi, T_CPU(%edi)
542 jne .L5_2
543 .L5_1:
544 /*
545 * Setup esp0 (kernel stack) in TSS to curthread's stack.
546 * (Note: Since we don't have saved 'regs' structure for all
547 * the threads we can't easily determine if we need to
548 * change esp0. So, we simply change the esp0 to bottom
549 * of the thread stack and it will work for all cases.)
550 */
551 movl CPU_TSS(%esi), %ecx
552 addl $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */
553 #if !defined(__xpv)
554 movl %eax, TSS_ESP0(%ecx)
555 #else
556 pushl %eax
557 pushl $KDS_SEL
558 call HYPERVISOR_stack_switch
559 addl $8, %esp
560 #endif /* __xpv */
561
562 movl %edi, CPU_THREAD(%esi) /* set CPU's thread pointer */
563 mfence /* synchronize with mutex_exit() */
564 xorl %ebp, %ebp /* make $<threadlist behave better */
565 movl T_LWP(%edi), %eax /* set associated lwp to */
566 movl %eax, CPU_LWP(%esi) /* CPU's lwp ptr */
567
568 movl T_SP(%edi), %esp /* switch to outgoing thread's stack */
569 movl T_PC(%edi), %esi /* saved return addr */
570
571 /*
572 * Call restorectx if context ops have been installed.
573 */
574 movl T_CTX(%edi), %eax /* should resumed thread restorectx? */
575 testl %eax, %eax
576 jz .norestorectx /* skip call when zero */
577 pushl %edi /* arg = thread pointer */
578 call restorectx /* call ctx ops */
579 addl $4, %esp /* restore stack pointer */
580 .norestorectx:
581
582 /*
583 * Call restorepctx if context ops have been installed for the proc.
584 */
585 movl T_PROCP(%edi), %eax
586 cmpl $0, P_PCTX(%eax)
587 je .norestorepctx
588 pushl %eax /* arg = proc pointer */
589 call restorepctx
590 addl $4, %esp /* restore stack pointer */
591 .norestorepctx:
592
593 STORE_INTR_START(%edi)
594
595 /*
596 * Restore non-volatile registers, then have spl0 return to the
597 * resuming thread's PC after first setting the priority as low as
598 * possible and blocking all interrupt threads that may be active.
599 */
600 movl %esi, %eax /* save return address */
601 RESTORE_REGS(%ecx)
602 pushl %eax /* push return address for spl0() */
603 call __dtrace_probe___sched_on__cpu
604 jmp spl0
605
606 resume_return:
607 /*
608 * Remove stack frame created in SAVE_REGS()
609 */
610 addl $CLONGSIZE, %esp
611 ret
612
613 .L4_2:
614 pause
615 cmpb $0, T_LOCK(%edi)
616 je .L4
617 jmp .L4_2
618
619 .L5_2:
620 /* cp->cpu_stats.sys.cpumigrate++ */
621 addl $1, CPU_STATS_SYS_CPUMIGRATE(%esi)
622 adcl $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi)
623 movl %esi, T_CPU(%edi) /* set new thread's CPU pointer */
624 jmp .L5_1
625
626 SET_SIZE(_resume_from_idle)
627 SET_SIZE(resume)
628
629 #endif /* __amd64 */
630 #endif /* __lint */
631
632 #if defined(__lint)
633
634 /* ARGSUSED */
635 void
636 resume_from_zombie(kthread_t *t)
637 {}
638
639 #else /* __lint */
640
641 #if defined(__amd64)
642
643 ENTRY(resume_from_zombie)
644 movq %gs:CPU_THREAD, %rax
645 leaq resume_from_zombie_return(%rip), %r11
646
647 /*
648 * Save non-volatile registers, and set return address for current
649 * thread to resume_from_zombie_return.
650 *
651 * %r12 = t (new thread) when done
652 */
653 SAVE_REGS(%rax, %r11)
654
655 movq %gs:CPU_THREAD, %r13 /* %r13 = curthread */
656
657 /* clean up the fp unit. It might be left enabled */
658
659 #if defined(__xpv) /* XXPV XXtclayton */
660 /*
661 * Remove this after bringup.
662 * (Too many #gp's for an instrumented hypervisor.)
663 */
664 STTS(%rax)
665 #else
666 movq %cr0, %rax
667 testq $CR0_TS, %rax
668 jnz .zfpu_disabled /* if TS already set, nothing to do */
669 fninit /* init fpu & discard pending error */
670 orq $CR0_TS, %rax
671 movq %rax, %cr0
672 .zfpu_disabled:
673
674 #endif /* __xpv */
675
676 /*
677 * Temporarily switch to the idle thread's stack so that the zombie
678 * thread's stack can be reclaimed by the reaper.
679 */
680 movq %gs:CPU_IDLE_THREAD, %rax /* idle thread pointer */
681 movq T_SP(%rax), %rsp /* get onto idle thread stack */
682
683 /*
684 * Sigh. If the idle thread has never run thread_start()
685 * then t_sp is mis-aligned by thread_load().
686 */
687 andq $_BITNOT(STACK_ALIGN-1), %rsp
688
689 /*
690 * Set the idle thread as the current thread.
691 */
692 movq %rax, %gs:CPU_THREAD
693
694 /* switch in the hat context for the new thread */
695 GET_THREAD_HATP(%rdi, %r12, %r11)
696 call hat_switch
697
698 /*
699 * Put the zombie on death-row.
700 */
701 movq %r13, %rdi
702 call reapq_add
703
704 jmp _resume_from_idle /* finish job of resume */
705
706 resume_from_zombie_return:
707 RESTORE_REGS(%r11) /* restore non-volatile registers */
708 call __dtrace_probe___sched_on__cpu
709
710 /*
711 * Remove stack frame created in SAVE_REGS()
712 */
713 addq $CLONGSIZE, %rsp
714 ret
715 SET_SIZE(resume_from_zombie)
716
717 #elif defined (__i386)
718
719 ENTRY(resume_from_zombie)
720 movl %gs:CPU_THREAD, %eax
721 movl $resume_from_zombie_return, %ecx
722
723 /*
724 * Save non-volatile registers, and set return address for current
725 * thread to resume_from_zombie_return.
726 *
727 * %edi = t (new thread) when done.
728 */
729 SAVE_REGS(%eax, %ecx)
730
731 #ifdef DEBUG
732 call assert_ints_enabled /* panics if we are cli'd */
733 #endif
734 movl %gs:CPU_THREAD, %esi /* %esi = curthread */
735
736 /* clean up the fp unit. It might be left enabled */
737
738 movl %cr0, %eax
739 testl $CR0_TS, %eax
740 jnz .zfpu_disabled /* if TS already set, nothing to do */
741 fninit /* init fpu & discard pending error */
742 orl $CR0_TS, %eax
743 movl %eax, %cr0
744 .zfpu_disabled:
745
746 /*
747 * Temporarily switch to the idle thread's stack so that the zombie
748 * thread's stack can be reclaimed by the reaper.
749 */
750 movl %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */
751 movl T_SP(%eax), %esp /* get onto idle thread stack */
752
753 /*
754 * Set the idle thread as the current thread.
755 */
756 movl %eax, %gs:CPU_THREAD
757
758 /*
759 * switch in the hat context for the new thread
760 */
761 GET_THREAD_HATP(%ecx, %edi, %ecx)
762 pushl %ecx
763 call hat_switch
764 addl $4, %esp
765
766 /*
767 * Put the zombie on death-row.
768 */
769 pushl %esi
770 call reapq_add
771 addl $4, %esp
772 jmp _resume_from_idle /* finish job of resume */
773
774 resume_from_zombie_return:
775 RESTORE_REGS(%ecx) /* restore non-volatile registers */
776 call __dtrace_probe___sched_on__cpu
777
778 /*
779 * Remove stack frame created in SAVE_REGS()
780 */
781 addl $CLONGSIZE, %esp
782 ret
783 SET_SIZE(resume_from_zombie)
784
785 #endif /* __amd64 */
786 #endif /* __lint */
787
788 #if defined(__lint)
789
790 /* ARGSUSED */
791 void
792 resume_from_intr(kthread_t *t)
793 {}
794
795 #else /* __lint */
796
797 #if defined(__amd64)
798
799 ENTRY(resume_from_intr)
800 movq %gs:CPU_THREAD, %rax
801 leaq resume_from_intr_return(%rip), %r11
802
803 /*
804 * Save non-volatile registers, and set return address for current
805 * thread to resume_from_intr_return.
806 *
807 * %r12 = t (new thread) when done
808 */
809 SAVE_REGS(%rax, %r11)
810
811 movq %gs:CPU_THREAD, %r13 /* %r13 = curthread */
812 movq %r12, %gs:CPU_THREAD /* set CPU's thread pointer */
813 mfence /* synchronize with mutex_exit() */
814 movq T_SP(%r12), %rsp /* restore resuming thread's sp */
815 xorl %ebp, %ebp /* make $<threadlist behave better */
816
817 /*
818 * Unlock outgoing thread's mutex dispatched by another processor.
819 */
820 xorl %eax, %eax
821 xchgb %al, T_LOCK(%r13)
822
823 STORE_INTR_START(%r12)
824
825 /*
826 * Restore non-volatile registers, then have spl0 return to the
827 * resuming thread's PC after first setting the priority as low as
828 * possible and blocking all interrupt threads that may be active.
829 */
830 movq T_PC(%r12), %rax /* saved return addr */
831 RESTORE_REGS(%r11);
832 pushq %rax /* push return address for spl0() */
833 call __dtrace_probe___sched_on__cpu
834 jmp spl0
835
836 resume_from_intr_return:
837 /*
838 * Remove stack frame created in SAVE_REGS()
839 */
840 addq $CLONGSIZE, %rsp
841 ret
842 SET_SIZE(resume_from_intr)
843
844 #elif defined (__i386)
845
846 ENTRY(resume_from_intr)
847 movl %gs:CPU_THREAD, %eax
848 movl $resume_from_intr_return, %ecx
849
850 /*
851 * Save non-volatile registers, and set return address for current
852 * thread to resume_return.
853 *
854 * %edi = t (new thread) when done.
855 */
856 SAVE_REGS(%eax, %ecx)
857
858 #ifdef DEBUG
859 call assert_ints_enabled /* panics if we are cli'd */
860 #endif
861 movl %gs:CPU_THREAD, %esi /* %esi = curthread */
862 movl %edi, %gs:CPU_THREAD /* set CPU's thread pointer */
863 mfence /* synchronize with mutex_exit() */
864 movl T_SP(%edi), %esp /* restore resuming thread's sp */
865 xorl %ebp, %ebp /* make $<threadlist behave better */
866
867 /*
868 * Unlock outgoing thread's mutex dispatched by another processor.
869 */
870 xorl %eax,%eax
871 xchgb %al, T_LOCK(%esi)
872
873 STORE_INTR_START(%edi)
874
875 /*
876 * Restore non-volatile registers, then have spl0 return to the
877 * resuming thread's PC after first setting the priority as low as
878 * possible and blocking all interrupt threads that may be active.
879 */
880 movl T_PC(%edi), %eax /* saved return addr */
881 RESTORE_REGS(%ecx)
882 pushl %eax /* push return address for spl0() */
883 call __dtrace_probe___sched_on__cpu
884 jmp spl0
885
886 resume_from_intr_return:
887 /*
888 * Remove stack frame created in SAVE_REGS()
889 */
890 addl $CLONGSIZE, %esp
891 ret
892 SET_SIZE(resume_from_intr)
893
894 #endif /* __amd64 */
895 #endif /* __lint */
896
897 #if defined(__lint)
898
899 void
900 thread_start(void)
901 {}
902
903 #else /* __lint */
904
905 #if defined(__amd64)
906
907 ENTRY(thread_start)
908 popq %rax /* start() */
909 popq %rdi /* arg */
910 popq %rsi /* len */
911 movq %rsp, %rbp
912 call *%rax
913 call thread_exit /* destroy thread if it returns. */
914 /*NOTREACHED*/
915 SET_SIZE(thread_start)
916
917 #elif defined(__i386)
918
919 ENTRY(thread_start)
920 popl %eax
921 movl %esp, %ebp
922 addl $8, %ebp
923 call *%eax
924 addl $8, %esp
925 call thread_exit /* destroy thread if it returns. */
926 /*NOTREACHED*/
927 SET_SIZE(thread_start)
928
929 #endif /* __i386 */
930
931 #endif /* __lint */