Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/swtch.s
+++ new/usr/src/uts/intel/ia32/ml/swtch.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
23 23 * Use is subject to license terms.
24 24 */
25 25
26 26 /*
27 - * Copyright (c) 2013, Joyent, Inc. All rights reserved.
27 + * Copyright (c) 2018 Joyent, Inc.
28 28 */
29 29
30 30 /*
31 31 * Process switching routines.
32 32 */
33 33
34 34 #if defined(__lint)
35 35 #include <sys/thread.h>
36 36 #include <sys/systm.h>
37 37 #include <sys/time.h>
38 38 #else /* __lint */
39 39 #include "assym.h"
40 40 #endif /* __lint */
41 41
42 42 #include <sys/asm_linkage.h>
43 43 #include <sys/asm_misc.h>
44 44 #include <sys/regset.h>
45 45 #include <sys/privregs.h>
46 46 #include <sys/stack.h>
47 47 #include <sys/segments.h>
48 48 #include <sys/psw.h>
49 49
50 50 /*
51 51 * resume(thread_id_t t);
52 52 *
53 53 * a thread can only run on one processor at a time. there
54 54 * exists a window on MPs where the current thread on one
55 55 * processor is capable of being dispatched by another processor.
56 56 * some overlap between outgoing and incoming threads can happen
|
↓ open down ↓ |
19 lines elided |
↑ open up ↑ |
57 57 * when they are the same thread. in this case where the threads
58 58 * are the same, resume() on one processor will spin on the incoming
59 59 * thread until resume() on the other processor has finished with
60 60 * the outgoing thread.
61 61 *
62 62 * The MMU context changes when the resuming thread resides in a different
63 63 * process. Kernel threads are known by resume to reside in process 0.
64 64 * The MMU context, therefore, only changes when resuming a thread in
65 65 * a process different from curproc.
66 66 *
67 - * resume_from_intr() is called when the thread being resumed was not
67 + * resume_from_intr() is called when the thread being resumed was not
68 68 * passivated by resume (e.g. was interrupted). This means that the
69 69 * resume lock is already held and that a restore context is not needed.
70 70 * Also, the MMU context is not changed on the resume in this case.
71 71 *
72 72 * resume_from_zombie() is the same as resume except the calling thread
73 73 * is a zombie and must be put on the deathrow list after the CPU is
74 74 * off the stack.
75 75 */
76 76
77 77 #if !defined(__lint)
78 78
79 79 #if LWP_PCB_FPU != 0
80 80 #error LWP_PCB_FPU MUST be defined as 0 for code in swtch.s to work
81 81 #endif /* LWP_PCB_FPU != 0 */
82 82
83 83 #endif /* !__lint */
84 84
85 85 #if defined(__amd64)
86 86
87 87 /*
88 88 * Save non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
89 89 *
90 90 * The stack frame must be created before the save of %rsp so that tracebacks
91 91 * of swtch()ed-out processes show the process as having last called swtch().
92 92 */
93 93 #define SAVE_REGS(thread_t, retaddr) \
94 94 movq %rbp, T_RBP(thread_t); \
95 95 movq %rbx, T_RBX(thread_t); \
96 96 movq %r12, T_R12(thread_t); \
97 97 movq %r13, T_R13(thread_t); \
98 98 movq %r14, T_R14(thread_t); \
99 99 movq %r15, T_R15(thread_t); \
100 100 pushq %rbp; \
101 101 movq %rsp, %rbp; \
102 102 movq %rsp, T_SP(thread_t); \
103 103 movq retaddr, T_PC(thread_t); \
104 104 movq %rdi, %r12; \
105 105 call __dtrace_probe___sched_off__cpu
106 106
107 107 /*
108 108 * Restore non-volatile regs other than %rsp (%rbx, %rbp, and %r12 - %r15)
109 109 *
110 110 * We load up %rsp from the label_t as part of the context switch, so
111 111 * we don't repeat that here.
112 112 *
113 113 * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
114 114 * already has the effect of putting the stack back the way it was when
115 115 * we came in.
116 116 */
117 117 #define RESTORE_REGS(scratch_reg) \
118 118 movq %gs:CPU_THREAD, scratch_reg; \
119 119 movq T_RBP(scratch_reg), %rbp; \
120 120 movq T_RBX(scratch_reg), %rbx; \
121 121 movq T_R12(scratch_reg), %r12; \
122 122 movq T_R13(scratch_reg), %r13; \
123 123 movq T_R14(scratch_reg), %r14; \
124 124 movq T_R15(scratch_reg), %r15
125 125
126 126 /*
127 127 * Get pointer to a thread's hat structure
128 128 */
129 129 #define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \
130 130 movq T_PROCP(thread_t), hatp; \
131 131 movq P_AS(hatp), scratch_reg; \
132 132 movq A_HAT(scratch_reg), hatp
133 133
134 134 #define TSC_READ() \
135 135 call tsc_read; \
136 136 movq %rax, %r14;
137 137
138 138 /*
139 139 * If we are resuming an interrupt thread, store a timestamp in the thread
140 140 * structure. If an interrupt occurs between tsc_read() and its subsequent
141 141 * store, the timestamp will be stale by the time it is stored. We can detect
142 142 * this by doing a compare-and-swap on the thread's timestamp, since any
143 143 * interrupt occurring in this window will put a new timestamp in the thread's
144 144 * t_intr_start field.
145 145 */
146 146 #define STORE_INTR_START(thread_t) \
147 147 testw $T_INTR_THREAD, T_FLAGS(thread_t); \
148 148 jz 1f; \
149 149 0: \
150 150 TSC_READ(); \
151 151 movq T_INTR_START(thread_t), %rax; \
152 152 cmpxchgq %r14, T_INTR_START(thread_t); \
153 153 jnz 0b; \
154 154 1:
155 155
156 156 #elif defined (__i386)
157 157
158 158 /*
159 159 * Save non-volatile registers (%ebp, %esi, %edi and %ebx)
160 160 *
161 161 * The stack frame must be created before the save of %esp so that tracebacks
162 162 * of swtch()ed-out processes show the process as having last called swtch().
163 163 */
164 164 #define SAVE_REGS(thread_t, retaddr) \
165 165 movl %ebp, T_EBP(thread_t); \
166 166 movl %ebx, T_EBX(thread_t); \
167 167 movl %esi, T_ESI(thread_t); \
168 168 movl %edi, T_EDI(thread_t); \
169 169 pushl %ebp; \
170 170 movl %esp, %ebp; \
171 171 movl %esp, T_SP(thread_t); \
172 172 movl retaddr, T_PC(thread_t); \
173 173 movl 8(%ebp), %edi; \
174 174 pushl %edi; \
175 175 call __dtrace_probe___sched_off__cpu; \
176 176 addl $CLONGSIZE, %esp
177 177
178 178 /*
179 179 * Restore non-volatile registers (%ebp, %esi, %edi and %ebx)
180 180 *
181 181 * We don't do a 'leave,' because reloading %rsp/%rbp from the label_t
182 182 * already has the effect of putting the stack back the way it was when
183 183 * we came in.
184 184 */
185 185 #define RESTORE_REGS(scratch_reg) \
186 186 movl %gs:CPU_THREAD, scratch_reg; \
187 187 movl T_EBP(scratch_reg), %ebp; \
188 188 movl T_EBX(scratch_reg), %ebx; \
189 189 movl T_ESI(scratch_reg), %esi; \
190 190 movl T_EDI(scratch_reg), %edi
191 191
192 192 /*
193 193 * Get pointer to a thread's hat structure
194 194 */
195 195 #define GET_THREAD_HATP(hatp, thread_t, scratch_reg) \
196 196 movl T_PROCP(thread_t), hatp; \
197 197 movl P_AS(hatp), scratch_reg; \
198 198 movl A_HAT(scratch_reg), hatp
199 199
200 200 /*
201 201 * If we are resuming an interrupt thread, store a timestamp in the thread
202 202 * structure. If an interrupt occurs between tsc_read() and its subsequent
203 203 * store, the timestamp will be stale by the time it is stored. We can detect
204 204 * this by doing a compare-and-swap on the thread's timestamp, since any
205 205 * interrupt occurring in this window will put a new timestamp in the thread's
206 206 * t_intr_start field.
207 207 */
208 208 #define STORE_INTR_START(thread_t) \
209 209 testw $T_INTR_THREAD, T_FLAGS(thread_t); \
210 210 jz 1f; \
211 211 pushl %ecx; \
212 212 0: \
213 213 pushl T_INTR_START(thread_t); \
214 214 pushl T_INTR_START+4(thread_t); \
215 215 call tsc_read; \
216 216 movl %eax, %ebx; \
217 217 movl %edx, %ecx; \
218 218 popl %edx; \
219 219 popl %eax; \
220 220 cmpxchg8b T_INTR_START(thread_t); \
221 221 jnz 0b; \
222 222 popl %ecx; \
223 223 1:
224 224
225 225 #endif /* __amd64 */
226 226
227 227 #if defined(__lint)
|
↓ open down ↓ |
150 lines elided |
↑ open up ↑ |
228 228
229 229 /* ARGSUSED */
230 230 void
231 231 resume(kthread_t *t)
232 232 {}
233 233
234 234 #else /* __lint */
235 235
236 236 #if defined(__amd64)
237 237
238 + .global kpti_enable
239 +
238 240 ENTRY(resume)
239 241 movq %gs:CPU_THREAD, %rax
240 242 leaq resume_return(%rip), %r11
241 243
242 244 /*
243 245 * Deal with SMAP here. A thread may be switched out at any point while
244 246 * it is executing. The thread could be under on_fault() or it could be
245 247 * pre-empted while performing a copy interruption. If this happens and
246 248 * we're not in the context of an interrupt which happens to handle
247 249 * saving and restoring rflags correctly, we may lose our SMAP related
248 250 * state.
249 251 *
250 252 * To handle this, as part of being switched out, we first save whether
251 253 * or not userland access is allowed ($PS_ACHK in rflags) and store that
252 254 * in t_useracc on the kthread_t and unconditionally enable SMAP to
253 255 * protect the system.
254 256 *
255 257 * Later, when the thread finishes resuming, we potentially disable smap
256 258 * if PS_ACHK was present in rflags. See uts/intel/ia32/ml/copy.s for
257 259 * more information on rflags and SMAP.
258 260 */
259 261 pushfq
260 262 popq %rsi
261 263 andq $PS_ACHK, %rsi
262 264 movq %rsi, T_USERACC(%rax)
263 265 call smap_enable
264 266
265 267 /*
266 268 * Save non-volatile registers, and set return address for current
267 269 * thread to resume_return.
268 270 *
269 271 * %r12 = t (new thread) when done
270 272 */
271 273 SAVE_REGS(%rax, %r11)
272 274
273 275
274 276 LOADCPU(%r15) /* %r15 = CPU */
275 277 movq CPU_THREAD(%r15), %r13 /* %r13 = curthread */
276 278
277 279 /*
278 280 * Call savectx if thread has installed context ops.
279 281 *
280 282 * Note that if we have floating point context, the save op
281 283 * (either fpsave_begin or fpxsave_begin) will issue the
282 284 * async save instruction (fnsave or fxsave respectively)
283 285 * that we fwait for below.
284 286 */
285 287 cmpq $0, T_CTX(%r13) /* should current thread savectx? */
286 288 je .nosavectx /* skip call when zero */
287 289
288 290 movq %r13, %rdi /* arg = thread pointer */
289 291 call savectx /* call ctx ops */
290 292 .nosavectx:
291 293
292 294 /*
293 295 * Call savepctx if process has installed context ops.
294 296 */
295 297 movq T_PROCP(%r13), %r14 /* %r14 = proc */
296 298 cmpq $0, P_PCTX(%r14) /* should current thread savectx? */
297 299 je .nosavepctx /* skip call when zero */
|
↓ open down ↓ |
50 lines elided |
↑ open up ↑ |
298 300
299 301 movq %r14, %rdi /* arg = proc pointer */
300 302 call savepctx /* call ctx ops */
301 303 .nosavepctx:
302 304
303 305 /*
304 306 * Temporarily switch to the idle thread's stack
305 307 */
306 308 movq CPU_IDLE_THREAD(%r15), %rax /* idle thread pointer */
307 309
308 - /*
310 + /*
309 311 * Set the idle thread as the current thread
310 312 */
311 313 movq T_SP(%rax), %rsp /* It is safe to set rsp */
312 314 movq %rax, CPU_THREAD(%r15)
313 315
314 316 /*
315 317 * Switch in the hat context for the new thread
316 318 *
317 319 */
318 320 GET_THREAD_HATP(%rdi, %r12, %r11)
319 321 call hat_switch
320 322
321 - /*
323 + /*
322 324 * Clear and unlock previous thread's t_lock
323 325 * to allow it to be dispatched by another processor.
324 326 */
325 327 movb $0, T_LOCK(%r13)
326 328
327 329 /*
328 330 * IMPORTANT: Registers at this point must be:
329 331 * %r12 = new thread
330 332 *
331 333 * Here we are in the idle thread, have dropped the old thread.
332 334 */
333 335 ALTENTRY(_resume_from_idle)
334 336 /*
335 337 * spin until dispatched thread's mutex has
336 338 * been unlocked. this mutex is unlocked when
337 339 * it becomes safe for the thread to run.
338 340 */
339 341 .lock_thread_mutex:
340 342 lock
341 343 btsl $0, T_LOCK(%r12) /* attempt to lock new thread's mutex */
342 344 jnc .thread_mutex_locked /* got it */
343 345
344 346 .spin_thread_mutex:
345 347 pause
346 348 cmpb $0, T_LOCK(%r12) /* check mutex status */
347 349 jz .lock_thread_mutex /* clear, retry lock */
348 350 jmp .spin_thread_mutex /* still locked, spin... */
349 351
350 352 .thread_mutex_locked:
351 353 /*
352 354 * Fix CPU structure to indicate new running thread.
353 355 * Set pointer in new thread to the CPU structure.
354 356 */
|
↓ open down ↓ |
23 lines elided |
↑ open up ↑ |
355 357 LOADCPU(%r13) /* load current CPU pointer */
356 358 cmpq %r13, T_CPU(%r12)
357 359 je .setup_cpu
358 360
359 361 /* cp->cpu_stats.sys.cpumigrate++ */
360 362 incq CPU_STATS_SYS_CPUMIGRATE(%r13)
361 363 movq %r13, T_CPU(%r12) /* set new thread's CPU pointer */
362 364
363 365 .setup_cpu:
364 366 /*
365 - * Setup rsp0 (kernel stack) in TSS to curthread's stack.
366 - * (Note: Since we don't have saved 'regs' structure for all
367 - * the threads we can't easily determine if we need to
368 - * change rsp0. So, we simply change the rsp0 to bottom
369 - * of the thread stack and it will work for all cases.)
367 + * Setup rsp0 (kernel stack) in TSS to curthread's saved regs
368 + * structure. If this thread doesn't have a regs structure above
369 + * the stack -- that is, if lwp_stk_init() was never called for the
370 + * thread -- this will set rsp0 to the wrong value, but it's harmless
371 + * as it's a kernel thread, and it won't actually attempt to implicitly
372 + * use the rsp0 via a privilege change.
370 373 *
371 - * XX64 - Is this correct?
374 + * Note that when we have KPTI enabled on amd64, we never use this
375 + * value at all (since all the interrupts have an IST set).
372 376 */
373 377 movq CPU_TSS(%r13), %r14
378 +#if !defined(__xpv)
379 + cmpq $1, kpti_enable
380 + jne 1f
381 + leaq CPU_KPTI_TR_RSP(%r13), %rax
382 + jmp 2f
383 +1:
374 384 movq T_STACK(%r12), %rax
375 385 addq $REGSIZE+MINFRAME, %rax /* to the bottom of thread stack */
376 -#if !defined(__xpv)
386 +2:
377 387 movq %rax, TSS_RSP0(%r14)
378 388 #else
389 + movq T_STACK(%r12), %rax
390 + addq $REGSIZE+MINFRAME, %rax /* to the bottom of thread stack */
379 391 movl $KDS_SEL, %edi
380 392 movq %rax, %rsi
381 393 call HYPERVISOR_stack_switch
382 394 #endif /* __xpv */
383 395
384 396 movq %r12, CPU_THREAD(%r13) /* set CPU's thread pointer */
385 397 mfence /* synchronize with mutex_exit() */
386 398 xorl %ebp, %ebp /* make $<threadlist behave better */
387 399 movq T_LWP(%r12), %rax /* set associated lwp to */
388 400 movq %rax, CPU_LWP(%r13) /* CPU's lwp ptr */
389 401
390 402 movq T_SP(%r12), %rsp /* switch to outgoing thread's stack */
391 403 movq T_PC(%r12), %r13 /* saved return addr */
392 404
393 405 /*
394 406 * Call restorectx if context ops have been installed.
395 407 */
396 408 cmpq $0, T_CTX(%r12) /* should resumed thread restorectx? */
397 409 jz .norestorectx /* skip call when zero */
398 410 movq %r12, %rdi /* arg = thread pointer */
399 411 call restorectx /* call ctx ops */
400 412 .norestorectx:
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
401 413
402 414 /*
403 415 * Call restorepctx if context ops have been installed for the proc.
404 416 */
405 417 movq T_PROCP(%r12), %rcx
406 418 cmpq $0, P_PCTX(%rcx)
407 419 jz .norestorepctx
408 420 movq %rcx, %rdi
409 421 call restorepctx
410 422 .norestorepctx:
411 -
423 +
412 424 STORE_INTR_START(%r12)
413 425
414 426 /*
415 427 * If we came into swtch with the ability to access userland pages, go
416 428 * ahead and restore that fact by disabling SMAP. Clear the indicator
417 429 * flag out of paranoia.
418 430 */
419 431 movq T_USERACC(%r12), %rax /* should we disable smap? */
420 432 cmpq $0, %rax /* skip call when zero */
421 433 jz .nosmap
422 434 xorq %rax, %rax
423 435 movq %rax, T_USERACC(%r12)
424 436 call smap_disable
425 437 .nosmap:
426 438
427 439 /*
428 440 * Restore non-volatile registers, then have spl0 return to the
429 441 * resuming thread's PC after first setting the priority as low as
430 442 * possible and blocking all interrupt threads that may be active.
431 443 */
432 - movq %r13, %rax /* save return address */
444 + movq %r13, %rax /* save return address */
433 445 RESTORE_REGS(%r11)
434 446 pushq %rax /* push return address for spl0() */
435 447 call __dtrace_probe___sched_on__cpu
436 448 jmp spl0
437 449
438 450 resume_return:
439 451 /*
440 452 * Remove stack frame created in SAVE_REGS()
441 453 */
442 454 addq $CLONGSIZE, %rsp
443 455 ret
444 456 SET_SIZE(_resume_from_idle)
445 457 SET_SIZE(resume)
446 458
447 459 #elif defined (__i386)
448 460
449 461 ENTRY(resume)
450 462 movl %gs:CPU_THREAD, %eax
451 463 movl $resume_return, %ecx
452 464
453 465 /*
454 466 * Save non-volatile registers, and set return address for current
455 467 * thread to resume_return.
456 468 *
457 469 * %edi = t (new thread) when done.
458 470 */
459 471 SAVE_REGS(%eax, %ecx)
460 472
461 473 LOADCPU(%ebx) /* %ebx = CPU */
462 474 movl CPU_THREAD(%ebx), %esi /* %esi = curthread */
463 475
464 476 #ifdef DEBUG
465 477 call assert_ints_enabled /* panics if we are cli'd */
466 478 #endif
467 479 /*
468 480 * Call savectx if thread has installed context ops.
469 481 *
470 482 * Note that if we have floating point context, the save op
471 483 * (either fpsave_begin or fpxsave_begin) will issue the
472 484 * async save instruction (fnsave or fxsave respectively)
473 485 * that we fwait for below.
474 486 */
475 487 movl T_CTX(%esi), %eax /* should current thread savectx? */
476 488 testl %eax, %eax
477 489 jz .nosavectx /* skip call when zero */
478 490 pushl %esi /* arg = thread pointer */
479 491 call savectx /* call ctx ops */
480 492 addl $4, %esp /* restore stack pointer */
481 493 .nosavectx:
482 494
483 495 /*
|
↓ open down ↓ |
41 lines elided |
↑ open up ↑ |
484 496 * Call savepctx if process has installed context ops.
485 497 */
486 498 movl T_PROCP(%esi), %eax /* %eax = proc */
487 499 cmpl $0, P_PCTX(%eax) /* should current thread savectx? */
488 500 je .nosavepctx /* skip call when zero */
489 501 pushl %eax /* arg = proc pointer */
490 502 call savepctx /* call ctx ops */
491 503 addl $4, %esp
492 504 .nosavepctx:
493 505
494 - /*
506 + /*
495 507 * Temporarily switch to the idle thread's stack
496 508 */
497 509 movl CPU_IDLE_THREAD(%ebx), %eax /* idle thread pointer */
498 510
499 - /*
511 + /*
500 512 * Set the idle thread as the current thread
501 513 */
502 514 movl T_SP(%eax), %esp /* It is safe to set esp */
503 515 movl %eax, CPU_THREAD(%ebx)
504 516
505 517 /* switch in the hat context for the new thread */
506 518 GET_THREAD_HATP(%ecx, %edi, %ecx)
507 519 pushl %ecx
508 520 call hat_switch
509 521 addl $4, %esp
510 -
511 - /*
522 +
523 + /*
512 524 * Clear and unlock previous thread's t_lock
513 525 * to allow it to be dispatched by another processor.
514 526 */
515 527 movb $0, T_LOCK(%esi)
516 528
517 529 /*
518 530 * IMPORTANT: Registers at this point must be:
519 531 * %edi = new thread
520 532 *
521 533 * Here we are in the idle thread, have dropped the old thread.
522 534 */
523 535 ALTENTRY(_resume_from_idle)
524 536 /*
525 537 * spin until dispatched thread's mutex has
526 538 * been unlocked. this mutex is unlocked when
527 539 * it becomes safe for the thread to run.
528 540 */
529 541 .L4:
530 542 lock
531 543 btsl $0, T_LOCK(%edi) /* lock new thread's mutex */
532 544 jc .L4_2 /* lock did not succeed */
533 545
534 546 /*
535 547 * Fix CPU structure to indicate new running thread.
536 548 * Set pointer in new thread to the CPU structure.
537 549 */
538 550 LOADCPU(%esi) /* load current CPU pointer */
539 551 movl T_STACK(%edi), %eax /* here to use v pipeline of */
540 552 /* Pentium. Used few lines below */
541 553 cmpl %esi, T_CPU(%edi)
542 554 jne .L5_2
543 555 .L5_1:
544 556 /*
545 557 * Setup esp0 (kernel stack) in TSS to curthread's stack.
546 558 * (Note: Since we don't have saved 'regs' structure for all
547 559 * the threads we can't easily determine if we need to
548 560 * change esp0. So, we simply change the esp0 to bottom
549 561 * of the thread stack and it will work for all cases.)
550 562 */
551 563 movl CPU_TSS(%esi), %ecx
552 564 addl $REGSIZE+MINFRAME, %eax /* to the bottom of thread stack */
553 565 #if !defined(__xpv)
554 566 movl %eax, TSS_ESP0(%ecx)
555 567 #else
556 568 pushl %eax
557 569 pushl $KDS_SEL
558 570 call HYPERVISOR_stack_switch
559 571 addl $8, %esp
560 572 #endif /* __xpv */
561 573
562 574 movl %edi, CPU_THREAD(%esi) /* set CPU's thread pointer */
563 575 mfence /* synchronize with mutex_exit() */
564 576 xorl %ebp, %ebp /* make $<threadlist behave better */
565 577 movl T_LWP(%edi), %eax /* set associated lwp to */
566 578 movl %eax, CPU_LWP(%esi) /* CPU's lwp ptr */
567 579
568 580 movl T_SP(%edi), %esp /* switch to outgoing thread's stack */
569 581 movl T_PC(%edi), %esi /* saved return addr */
570 582
571 583 /*
572 584 * Call restorectx if context ops have been installed.
573 585 */
574 586 movl T_CTX(%edi), %eax /* should resumed thread restorectx? */
575 587 testl %eax, %eax
576 588 jz .norestorectx /* skip call when zero */
577 589 pushl %edi /* arg = thread pointer */
578 590 call restorectx /* call ctx ops */
579 591 addl $4, %esp /* restore stack pointer */
580 592 .norestorectx:
581 593
582 594 /*
583 595 * Call restorepctx if context ops have been installed for the proc.
584 596 */
585 597 movl T_PROCP(%edi), %eax
586 598 cmpl $0, P_PCTX(%eax)
587 599 je .norestorepctx
588 600 pushl %eax /* arg = proc pointer */
589 601 call restorepctx
590 602 addl $4, %esp /* restore stack pointer */
591 603 .norestorepctx:
592 604
593 605 STORE_INTR_START(%edi)
594 606
595 607 /*
596 608 * Restore non-volatile registers, then have spl0 return to the
597 609 * resuming thread's PC after first setting the priority as low as
598 610 * possible and blocking all interrupt threads that may be active.
599 611 */
600 612 movl %esi, %eax /* save return address */
601 613 RESTORE_REGS(%ecx)
602 614 pushl %eax /* push return address for spl0() */
603 615 call __dtrace_probe___sched_on__cpu
604 616 jmp spl0
605 617
606 618 resume_return:
607 619 /*
608 620 * Remove stack frame created in SAVE_REGS()
609 621 */
610 622 addl $CLONGSIZE, %esp
611 623 ret
612 624
613 625 .L4_2:
614 626 pause
615 627 cmpb $0, T_LOCK(%edi)
616 628 je .L4
617 629 jmp .L4_2
618 630
619 631 .L5_2:
620 632 /* cp->cpu_stats.sys.cpumigrate++ */
621 633 addl $1, CPU_STATS_SYS_CPUMIGRATE(%esi)
622 634 adcl $0, CPU_STATS_SYS_CPUMIGRATE+4(%esi)
623 635 movl %esi, T_CPU(%edi) /* set new thread's CPU pointer */
624 636 jmp .L5_1
625 637
626 638 SET_SIZE(_resume_from_idle)
627 639 SET_SIZE(resume)
628 640
629 641 #endif /* __amd64 */
630 642 #endif /* __lint */
631 643
632 644 #if defined(__lint)
633 645
634 646 /* ARGSUSED */
635 647 void
636 648 resume_from_zombie(kthread_t *t)
637 649 {}
638 650
639 651 #else /* __lint */
640 652
641 653 #if defined(__amd64)
642 654
643 655 ENTRY(resume_from_zombie)
644 656 movq %gs:CPU_THREAD, %rax
645 657 leaq resume_from_zombie_return(%rip), %r11
646 658
647 659 /*
648 660 * Save non-volatile registers, and set return address for current
649 661 * thread to resume_from_zombie_return.
650 662 *
651 663 * %r12 = t (new thread) when done
652 664 */
653 665 SAVE_REGS(%rax, %r11)
654 666
655 667 movq %gs:CPU_THREAD, %r13 /* %r13 = curthread */
656 668
657 669 /* clean up the fp unit. It might be left enabled */
658 670
659 671 #if defined(__xpv) /* XXPV XXtclayton */
660 672 /*
661 673 * Remove this after bringup.
662 674 * (Too many #gp's for an instrumented hypervisor.)
663 675 */
664 676 STTS(%rax)
665 677 #else
|
↓ open down ↓ |
144 lines elided |
↑ open up ↑ |
666 678 movq %cr0, %rax
667 679 testq $CR0_TS, %rax
668 680 jnz .zfpu_disabled /* if TS already set, nothing to do */
669 681 fninit /* init fpu & discard pending error */
670 682 orq $CR0_TS, %rax
671 683 movq %rax, %cr0
672 684 .zfpu_disabled:
673 685
674 686 #endif /* __xpv */
675 687
676 - /*
688 + /*
677 689 * Temporarily switch to the idle thread's stack so that the zombie
678 690 * thread's stack can be reclaimed by the reaper.
679 691 */
680 692 movq %gs:CPU_IDLE_THREAD, %rax /* idle thread pointer */
681 693 movq T_SP(%rax), %rsp /* get onto idle thread stack */
682 694
683 695 /*
684 696 * Sigh. If the idle thread has never run thread_start()
685 697 * then t_sp is mis-aligned by thread_load().
686 698 */
687 699 andq $_BITNOT(STACK_ALIGN-1), %rsp
688 700
689 - /*
701 + /*
690 702 * Set the idle thread as the current thread.
691 703 */
692 704 movq %rax, %gs:CPU_THREAD
693 705
694 706 /* switch in the hat context for the new thread */
695 707 GET_THREAD_HATP(%rdi, %r12, %r11)
696 708 call hat_switch
697 709
698 - /*
710 + /*
699 711 * Put the zombie on death-row.
700 712 */
701 713 movq %r13, %rdi
702 714 call reapq_add
703 715
704 716 jmp _resume_from_idle /* finish job of resume */
705 717
706 718 resume_from_zombie_return:
707 719 RESTORE_REGS(%r11) /* restore non-volatile registers */
708 720 call __dtrace_probe___sched_on__cpu
709 721
710 722 /*
711 723 * Remove stack frame created in SAVE_REGS()
712 724 */
713 725 addq $CLONGSIZE, %rsp
714 726 ret
715 727 SET_SIZE(resume_from_zombie)
716 728
717 729 #elif defined (__i386)
718 730
719 731 ENTRY(resume_from_zombie)
720 732 movl %gs:CPU_THREAD, %eax
721 733 movl $resume_from_zombie_return, %ecx
722 734
723 735 /*
724 736 * Save non-volatile registers, and set return address for current
725 737 * thread to resume_from_zombie_return.
726 738 *
727 739 * %edi = t (new thread) when done.
728 740 */
729 741 SAVE_REGS(%eax, %ecx)
730 742
731 743 #ifdef DEBUG
732 744 call assert_ints_enabled /* panics if we are cli'd */
733 745 #endif
734 746 movl %gs:CPU_THREAD, %esi /* %esi = curthread */
735 747
|
↓ open down ↓ |
27 lines elided |
↑ open up ↑ |
736 748 /* clean up the fp unit. It might be left enabled */
737 749
738 750 movl %cr0, %eax
739 751 testl $CR0_TS, %eax
740 752 jnz .zfpu_disabled /* if TS already set, nothing to do */
741 753 fninit /* init fpu & discard pending error */
742 754 orl $CR0_TS, %eax
743 755 movl %eax, %cr0
744 756 .zfpu_disabled:
745 757
746 - /*
758 + /*
747 759 * Temporarily switch to the idle thread's stack so that the zombie
748 760 * thread's stack can be reclaimed by the reaper.
749 761 */
750 762 movl %gs:CPU_IDLE_THREAD, %eax /* idle thread pointer */
751 763 movl T_SP(%eax), %esp /* get onto idle thread stack */
752 764
753 - /*
765 + /*
754 766 * Set the idle thread as the current thread.
755 767 */
756 768 movl %eax, %gs:CPU_THREAD
757 769
758 770 /*
759 771 * switch in the hat context for the new thread
760 772 */
761 773 GET_THREAD_HATP(%ecx, %edi, %ecx)
762 774 pushl %ecx
763 775 call hat_switch
764 776 addl $4, %esp
765 777
766 - /*
778 + /*
767 779 * Put the zombie on death-row.
768 780 */
769 781 pushl %esi
770 782 call reapq_add
771 783 addl $4, %esp
772 784 jmp _resume_from_idle /* finish job of resume */
773 785
774 786 resume_from_zombie_return:
775 787 RESTORE_REGS(%ecx) /* restore non-volatile registers */
776 788 call __dtrace_probe___sched_on__cpu
777 789
778 790 /*
779 791 * Remove stack frame created in SAVE_REGS()
780 792 */
781 793 addl $CLONGSIZE, %esp
782 794 ret
783 795 SET_SIZE(resume_from_zombie)
784 796
785 797 #endif /* __amd64 */
786 798 #endif /* __lint */
787 799
788 800 #if defined(__lint)
789 801
790 802 /* ARGSUSED */
791 803 void
792 804 resume_from_intr(kthread_t *t)
793 805 {}
794 806
795 807 #else /* __lint */
796 808
797 809 #if defined(__amd64)
798 810
799 811 ENTRY(resume_from_intr)
800 812 movq %gs:CPU_THREAD, %rax
801 813 leaq resume_from_intr_return(%rip), %r11
802 814
803 815 /*
804 816 * Save non-volatile registers, and set return address for current
805 817 * thread to resume_from_intr_return.
806 818 *
|
↓ open down ↓ |
30 lines elided |
↑ open up ↑ |
807 819 * %r12 = t (new thread) when done
808 820 */
809 821 SAVE_REGS(%rax, %r11)
810 822
811 823 movq %gs:CPU_THREAD, %r13 /* %r13 = curthread */
812 824 movq %r12, %gs:CPU_THREAD /* set CPU's thread pointer */
813 825 mfence /* synchronize with mutex_exit() */
814 826 movq T_SP(%r12), %rsp /* restore resuming thread's sp */
815 827 xorl %ebp, %ebp /* make $<threadlist behave better */
816 828
817 - /*
829 + /*
818 830 * Unlock outgoing thread's mutex dispatched by another processor.
819 831 */
820 832 xorl %eax, %eax
821 833 xchgb %al, T_LOCK(%r13)
822 834
823 835 STORE_INTR_START(%r12)
824 836
825 837 /*
826 838 * Restore non-volatile registers, then have spl0 return to the
827 839 * resuming thread's PC after first setting the priority as low as
828 840 * possible and blocking all interrupt threads that may be active.
829 841 */
830 842 movq T_PC(%r12), %rax /* saved return addr */
831 843 RESTORE_REGS(%r11);
832 844 pushq %rax /* push return address for spl0() */
833 845 call __dtrace_probe___sched_on__cpu
834 846 jmp spl0
835 847
836 848 resume_from_intr_return:
837 849 /*
838 850 * Remove stack frame created in SAVE_REGS()
839 851 */
840 852 addq $CLONGSIZE, %rsp
841 853 ret
842 854 SET_SIZE(resume_from_intr)
843 855
844 856 #elif defined (__i386)
845 857
846 858 ENTRY(resume_from_intr)
847 859 movl %gs:CPU_THREAD, %eax
848 860 movl $resume_from_intr_return, %ecx
849 861
850 862 /*
851 863 * Save non-volatile registers, and set return address for current
852 864 * thread to resume_return.
853 865 *
854 866 * %edi = t (new thread) when done.
855 867 */
856 868 SAVE_REGS(%eax, %ecx)
|
↓ open down ↓ |
29 lines elided |
↑ open up ↑ |
857 869
858 870 #ifdef DEBUG
859 871 call assert_ints_enabled /* panics if we are cli'd */
860 872 #endif
861 873 movl %gs:CPU_THREAD, %esi /* %esi = curthread */
862 874 movl %edi, %gs:CPU_THREAD /* set CPU's thread pointer */
863 875 mfence /* synchronize with mutex_exit() */
864 876 movl T_SP(%edi), %esp /* restore resuming thread's sp */
865 877 xorl %ebp, %ebp /* make $<threadlist behave better */
866 878
867 - /*
879 + /*
868 880 * Unlock outgoing thread's mutex dispatched by another processor.
869 881 */
870 882 xorl %eax,%eax
871 883 xchgb %al, T_LOCK(%esi)
872 884
873 885 STORE_INTR_START(%edi)
874 886
875 887 /*
876 888 * Restore non-volatile registers, then have spl0 return to the
877 889 * resuming thread's PC after first setting the priority as low as
878 890 * possible and blocking all interrupt threads that may be active.
879 891 */
880 892 movl T_PC(%edi), %eax /* saved return addr */
881 893 RESTORE_REGS(%ecx)
882 894 pushl %eax /* push return address for spl0() */
883 895 call __dtrace_probe___sched_on__cpu
884 896 jmp spl0
885 897
886 898 resume_from_intr_return:
887 899 /*
888 900 * Remove stack frame created in SAVE_REGS()
889 901 */
890 902 addl $CLONGSIZE, %esp
891 903 ret
892 904 SET_SIZE(resume_from_intr)
893 905
894 906 #endif /* __amd64 */
895 907 #endif /* __lint */
896 908
897 909 #if defined(__lint)
898 910
899 911 void
900 912 thread_start(void)
901 913 {}
902 914
903 915 #else /* __lint */
904 916
905 917 #if defined(__amd64)
906 918
907 919 ENTRY(thread_start)
908 920 popq %rax /* start() */
909 921 popq %rdi /* arg */
910 922 popq %rsi /* len */
911 923 movq %rsp, %rbp
912 924 call *%rax
913 925 call thread_exit /* destroy thread if it returns. */
914 926 /*NOTREACHED*/
915 927 SET_SIZE(thread_start)
916 928
917 929 #elif defined(__i386)
918 930
919 931 ENTRY(thread_start)
920 932 popl %eax
921 933 movl %esp, %ebp
922 934 addl $8, %ebp
923 935 call *%eax
924 936 addl $8, %esp
925 937 call thread_exit /* destroy thread if it returns. */
926 938 /*NOTREACHED*/
927 939 SET_SIZE(thread_start)
928 940
929 941 #endif /* __i386 */
930 942
931 943 #endif /* __lint */
|
↓ open down ↓ |
54 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX