Print this page
8956 Implement KPTI
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Robert Mustacchi <rm@joyent.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/i86pc/ml/syscall_asm_amd64.s
+++ new/usr/src/uts/i86pc/ml/syscall_asm_amd64.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 - * Copyright 2015 Joyent, Inc.
23 + * Copyright 2018 Joyent, Inc.
24 24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 25 */
26 26
27 27 #include <sys/asm_linkage.h>
28 28 #include <sys/asm_misc.h>
29 29 #include <sys/regset.h>
30 30 #include <sys/privregs.h>
31 31 #include <sys/psw.h>
32 32 #include <sys/machbrand.h>
33 33
34 34 #if defined(__lint)
35 35
36 36 #include <sys/types.h>
37 37 #include <sys/thread.h>
38 38 #include <sys/systm.h>
39 39
40 40 #else /* __lint */
41 41
42 42 #include <sys/segments.h>
43 43 #include <sys/pcb.h>
44 44 #include <sys/trap.h>
45 45 #include <sys/ftrace.h>
46 46 #include <sys/traptrace.h>
47 47 #include <sys/clock.h>
48 48 #include <sys/model.h>
49 49 #include <sys/panic.h>
50 50
51 51 #if defined(__xpv)
52 52 #include <sys/hypervisor.h>
53 53 #endif
54 54
55 55 #include "assym.h"
56 56
57 57 #endif /* __lint */
58 58
59 59 /*
60 60 * We implement five flavours of system call entry points
61 61 *
62 62 * - syscall/sysretq (amd64 generic)
63 63 * - syscall/sysretl (i386 plus SYSC bit)
64 64 * - sysenter/sysexit (i386 plus SEP bit)
65 65 * - int/iret (i386 generic)
66 66 * - lcall/iret (i386 generic)
67 67 *
68 68 * The current libc included in Solaris uses int/iret as the base unoptimized
69 69 * kernel entry method. Older libc implementations and legacy binaries may use
70 70 * the lcall call gate, so it must continue to be supported.
71 71 *
72 72 * System calls that use an lcall call gate are processed in trap() via a
73 73 * segment-not-present trap, i.e. lcalls are extremely slow(!).
74 74 *
75 75 * The basic pattern used in the 32-bit SYSC handler at this point in time is
76 76 * to have the bare minimum of assembler, and get to the C handlers as
77 77 * quickly as possible.
78 78 *
79 79 * The 64-bit handler is much closer to the sparcv9 handler; that's
80 80 * because of passing arguments in registers. The 32-bit world still
81 81 * passes arguments on the stack -- that makes that handler substantially
82 82 * more complex.
83 83 *
84 84 * The two handlers share a few code fragments which are broken
85 85 * out into preprocessor macros below.
86 86 *
87 87 * XX64 come back and speed all this up later. The 32-bit stuff looks
88 88 * especially easy to speed up the argument copying part ..
89 89 *
90 90 *
91 91 * Notes about segment register usage (c.f. the 32-bit kernel)
92 92 *
93 93 * In the 32-bit kernel, segment registers are dutifully saved and
94 94 * restored on all mode transitions because the kernel uses them directly.
95 95 * When the processor is running in 64-bit mode, segment registers are
96 96 * largely ignored.
97 97 *
98 98 * %cs and %ss
99 99 * controlled by the hardware mechanisms that make mode transitions
100 100 *
101 101 * The remaining segment registers have to either be pointing at a valid
102 102 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
103 103 *
104 104 * %ds and %es
105 105 * always ignored
106 106 *
107 107 * %fs and %gs
108 108 * fsbase and gsbase are used to control the place they really point at.
109 109 * The kernel only depends on %gs, and controls its own gsbase via swapgs
110 110 *
111 111 * Note that loading segment registers is still costly because the GDT
112 112 * lookup still happens (this is because the hardware can't know that we're
113 113 * not setting up these segment registers for a 32-bit program). Thus we
114 114 * avoid doing this in the syscall path, and defer them to lwp context switch
115 115 * handlers, so the register values remain virtualized to the lwp.
116 116 */
117 117
118 118 #if defined(SYSCALLTRACE)
119 119 #define ORL_SYSCALLTRACE(r32) \
120 120 orl syscalltrace(%rip), r32
121 121 #else
122 122 #define ORL_SYSCALLTRACE(r32)
123 123 #endif
124 124
125 125 /*
126 126 * In the 32-bit kernel, we do absolutely nothing before getting into the
127 127 * brand callback checks. In 64-bit land, we do swapgs and then come here.
128 128 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
129 129 * are still unused.
130 130 *
131 131 * Check if a brand_mach_ops callback is defined for the specified callback_id
132 132 * type. If so invoke it with the kernel's %gs value loaded and the following
133 133 * data on the stack:
134 134 *
135 135 * stack: --------------------------------------
136 136 * 32 | callback pointer |
137 137 * | 24 | user (or interrupt) stack pointer |
138 138 * | 16 | lwp pointer |
139 139 * v 8 | userland return address |
140 140 * 0 | callback wrapper return addr |
141 141 * --------------------------------------
142 142 *
143 143 * Since we're pushing the userland return address onto the kernel stack
144 144 * we need to get that address without accessing the user's stack (since we
145 145 * can't trust that data). There are different ways to get the userland
146 146 * return address depending on how the syscall trap was made:
147 147 *
148 148 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
149 149 * b) For sys_sysenter the return address is in %rdx.
150 150 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
151 151 * the stack pointer points at the state saved when we took the interrupt:
152 152 * ------------------------
153 153 * | | user's %ss |
154 154 * | | user's %esp |
155 155 * | | EFLAGS register |
156 156 * v | user's %cs |
157 157 * | user's %eip |
158 158 * ------------------------
159 159 *
160 160 * The 2nd parameter to the BRAND_CALLBACK macro is either the
161 161 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
162 162 * used to generate the proper code to get the userland return address for
163 163 * each syscall entry point.
164 164 *
165 165 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
166 166 * is available as a scratch register within the callback. If the callback
167 167 * returns within the kernel then this macro will restore %r15. If the
168 168 * callback is going to return directly to userland then it should restore
169 169 * %r15 before returning to userland.
170 170 */
171 171 #define BRAND_URET_FROM_REG(rip_reg) \
172 172 pushq rip_reg /* push the return address */
173 173
174 174 /*
175 175 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
176 176 * is currently pointing at the user return address (%eip).
177 177 */
178 178 #define BRAND_URET_FROM_INTR_STACK() \
179 179 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
180 180 pushq (%r15) /* push the return address */
181 181
182 182 #define BRAND_CALLBACK(callback_id, push_userland_ret) \
183 183 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
184 184 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
185 185 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
186 186 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
187 187 subq $16, %rsp /* save space for 2 pointers */ ;\
188 188 pushq %r14 /* save %r14 */ ;\
189 189 movq %gs:CPU_RTMP_RSP, %r14 ;\
190 190 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
191 191 popq %r14 /* restore %r14 */ ;\
192 192 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
193 193 pushq %r15 /* push the lwp pointer */ ;\
194 194 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
195 195 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
196 196 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
197 197 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
198 198 cmpq $0, %r15 ;\
199 199 je 1f ;\
200 200 movq %r15, 16(%rsp) /* save the callback pointer */ ;\
201 201 push_userland_ret /* push the return address */ ;\
202 202 call *24(%rsp) /* call callback */ ;\
203 203 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
204 204 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
205 205
206 206 #define MSTATE_TRANSITION(from, to) \
207 207 movl $from, %edi; \
208 208 movl $to, %esi; \
209 209 call syscall_mstate
210 210
211 211 /*
212 212 * Check to see if a simple (direct) return is possible i.e.
213 213 *
214 214 * if (t->t_post_sys_ast | syscalltrace |
215 215 * lwp->lwp_pcb.pcb_rupdate == 1)
216 216 * do full version ;
217 217 *
218 218 * Preconditions:
219 219 * - t is curthread
220 220 * Postconditions:
221 221 * - condition code NE is set if post-sys is too complex
222 222 * - rtmp is zeroed if it isn't (we rely on this!)
223 223 * - ltmp is smashed
224 224 */
225 225 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
226 226 movq T_LWP(t), ltmp; \
227 227 movzbl PCB_RUPDATE(ltmp), rtmp; \
228 228 ORL_SYSCALLTRACE(rtmp); \
229 229 orl T_POST_SYS_AST(t), rtmp; \
230 230 cmpl $0, rtmp
231 231
232 232 /*
233 233 * Fix up the lwp, thread, and eflags for a successful return
234 234 *
235 235 * Preconditions:
236 236 * - zwreg contains zero
237 237 */
238 238 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
239 239 movb $LWP_USER, LWP_STATE(lwp); \
240 240 movw zwreg, T_SYSNUM(t); \
241 241 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
242 242
243 243 /*
244 244 * ASSERT(lwptoregs(lwp) == rp);
245 245 *
246 246 * This may seem obvious, but very odd things happen if this
247 247 * assertion is false
248 248 *
249 249 * Preconditions:
250 250 * (%rsp is ready for normal call sequence)
251 251 * Postconditions (if assertion is true):
252 252 * %r11 is smashed
253 253 *
254 254 * ASSERT(rp->r_cs == descnum)
255 255 *
256 256 * The code selector is written into the regs structure when the
257 257 * lwp stack is created. We use this ASSERT to validate that
258 258 * the regs structure really matches how we came in.
259 259 *
260 260 * Preconditions:
261 261 * (%rsp is ready for normal call sequence)
262 262 * Postconditions (if assertion is true):
263 263 * -none-
264 264 *
265 265 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
266 266 *
267 267 * If this is false, it meant that we returned to userland without
268 268 * updating the segment registers as we were supposed to.
269 269 *
270 270 * Note that we must ensure no interrupts or other traps intervene
271 271 * between entering privileged mode and performing the assertion,
272 272 * otherwise we may perform a context switch on the thread, which
273 273 * will end up setting pcb_rupdate to 1 again.
274 274 */
275 275 #if defined(DEBUG)
276 276
277 277 #if !defined(__lint)
278 278
279 279 __lwptoregs_msg:
280 280 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
281 281
282 282 __codesel_msg:
283 283 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
284 284
285 285 __no_rupdate_msg:
286 286 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
287 287
288 288 #endif /* !__lint */
289 289
290 290 #define ASSERT_LWPTOREGS(lwp, rp) \
291 291 movq LWP_REGS(lwp), %r11; \
292 292 cmpq rp, %r11; \
293 293 je 7f; \
294 294 leaq __lwptoregs_msg(%rip), %rdi; \
295 295 movl $__LINE__, %esi; \
296 296 movq lwp, %rdx; \
297 297 movq %r11, %rcx; \
298 298 movq rp, %r8; \
299 299 xorl %eax, %eax; \
300 300 call panic; \
301 301 7:
302 302
303 303 #define ASSERT_NO_RUPDATE_PENDING(lwp) \
304 304 testb $0x1, PCB_RUPDATE(lwp); \
305 305 je 8f; \
306 306 movq lwp, %rdx; \
307 307 leaq __no_rupdate_msg(%rip), %rdi; \
308 308 movl $__LINE__, %esi; \
309 309 xorl %eax, %eax; \
310 310 call panic; \
311 311 8:
312 312
313 313 #else
314 314 #define ASSERT_LWPTOREGS(lwp, rp)
315 315 #define ASSERT_NO_RUPDATE_PENDING(lwp)
316 316 #endif
317 317
318 318 /*
319 319 * Do the traptrace thing and restore any registers we used
320 320 * in situ. Assumes that %rsp is pointing at the base of
321 321 * the struct regs, obviously ..
322 322 */
323 323 #ifdef TRAPTRACE
324 324 #define SYSCALL_TRAPTRACE(ttype) \
325 325 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
326 326 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
327 327 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
328 328 movq REGOFF_RAX(%rsp), %rax; \
329 329 movq REGOFF_RBX(%rsp), %rbx; \
330 330 movq REGOFF_RCX(%rsp), %rcx; \
331 331 movq REGOFF_RDX(%rsp), %rdx; \
332 332 movl %eax, TTR_SYSNUM(%rdi); \
333 333 movq REGOFF_RDI(%rsp), %rdi
334 334
335 335 #define SYSCALL_TRAPTRACE32(ttype) \
336 336 SYSCALL_TRAPTRACE(ttype); \
337 337 /* paranoia: clean the top 32-bits of the registers */ \
338 338 orl %eax, %eax; \
339 339 orl %ebx, %ebx; \
340 340 orl %ecx, %ecx; \
341 341 orl %edx, %edx; \
342 342 orl %edi, %edi
343 343 #else /* TRAPTRACE */
344 344 #define SYSCALL_TRAPTRACE(ttype)
345 345 #define SYSCALL_TRAPTRACE32(ttype)
346 346 #endif /* TRAPTRACE */
347 347
348 348 /*
349 349 * The 64-bit libc syscall wrapper does this:
350 350 *
351 351 * fn(<args>)
352 352 * {
353 353 * movq %rcx, %r10 -- because syscall smashes %rcx
354 354 * movl $CODE, %eax
355 355 * syscall
356 356 * <error processing>
357 357 * }
358 358 *
359 359 * Thus when we come into the kernel:
360 360 *
361 361 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
362 362 * %rax is the syscall number
363 363 * %r12-%r15 contain caller state
364 364 *
365 365 * The syscall instruction arranges that:
366 366 *
367 367 * %rcx contains the return %rip
368 368 * %r11d contains bottom 32-bits of %rflags
369 369 * %rflags is masked (as determined by the SFMASK msr)
370 370 * %cs is set to UCS_SEL (as determined by the STAR msr)
371 371 * %ss is set to UDS_SEL (as determined by the STAR msr)
372 372 * %rip is set to sys_syscall (as determined by the LSTAR msr)
373 373 *
374 374 * Or in other words, we have no registers available at all.
375 375 * Only swapgs can save us!
376 376 *
377 377 * Under the hypervisor, the swapgs has happened already. However, the
378 378 * state of the world is very different from that we're familiar with.
379 379 *
380 380 * In particular, we have a stack structure like that for interrupt
381 381 * gates, except that the %cs and %ss registers are modified for reasons
382 382 * that are not entirely clear. Critically, the %rcx/%r11 values do
383 383 * *not* reflect the usage of those registers under a 'real' syscall[1];
384 384 * the stack, therefore, looks like this:
385 385 *
386 386 * 0x0(rsp) potentially junk %rcx
387 387 * 0x8(rsp) potentially junk %r11
388 388 * 0x10(rsp) user %rip
389 389 * 0x18(rsp) modified %cs
390 390 * 0x20(rsp) user %rflags
391 391 * 0x28(rsp) user %rsp
392 392 * 0x30(rsp) modified %ss
393 393 *
394 394 *
395 395 * and before continuing on, we must load the %rip into %rcx and the
396 396 * %rflags into %r11.
397 397 *
398 398 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
399 399 * Sigh.
400 400 */
401 401 #if defined(__xpv)
402 402 #define XPV_SYSCALL_PROD \
403 403 movq 0x10(%rsp), %rcx; \
404 404 movq 0x20(%rsp), %r11; \
405 405 movq 0x28(%rsp), %rsp
406 406 #else
407 407 #define XPV_SYSCALL_PROD /* nothing */
408 408 #endif
409 409
410 410 #if defined(__lint)
411 411
412 412 /*ARGSUSED*/
413 413 void
414 414 sys_syscall()
415 415 {}
416 416
417 417 void
418 418 _allsyscalls()
419 419 {}
420 420
421 421 size_t _allsyscalls_size;
422 422
423 423 #else /* __lint */
424 424
425 425 ENTRY_NP2(brand_sys_syscall,_allsyscalls)
426 426 SWAPGS /* kernel gsbase */
427 427 XPV_SYSCALL_PROD
428 428 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
429 429 jmp noprod_sys_syscall
430 430
431 431 ALTENTRY(sys_syscall)
432 432 SWAPGS /* kernel gsbase */
433 433 XPV_SYSCALL_PROD
434 434
435 435 noprod_sys_syscall:
436 436 movq %r15, %gs:CPU_RTMP_R15
437 437 movq %rsp, %gs:CPU_RTMP_RSP
438 438
439 439 movq %gs:CPU_THREAD, %r15
440 440 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
441 441
442 442 ASSERT_UPCALL_MASK_IS_SET
443 443
444 444 movl $UCS_SEL, REGOFF_CS(%rsp)
445 445 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
446 446 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
447 447 movl $UDS_SEL, REGOFF_SS(%rsp)
448 448
449 449 movl %eax, %eax /* wrapper: sysc# -> %eax */
450 450 movq %rdi, REGOFF_RDI(%rsp)
451 451 movq %rsi, REGOFF_RSI(%rsp)
452 452 movq %rdx, REGOFF_RDX(%rsp)
453 453 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
454 454 movq %r10, %rcx /* arg[3] for direct calls */
455 455
456 456 movq %r8, REGOFF_R8(%rsp)
457 457 movq %r9, REGOFF_R9(%rsp)
458 458 movq %rax, REGOFF_RAX(%rsp)
459 459 movq %rbx, REGOFF_RBX(%rsp)
460 460
461 461 movq %rbp, REGOFF_RBP(%rsp)
462 462 movq %r10, REGOFF_R10(%rsp)
463 463 movq %gs:CPU_RTMP_RSP, %r11
464 464 movq %r11, REGOFF_RSP(%rsp)
465 465 movq %r12, REGOFF_R12(%rsp)
466 466
467 467 movq %r13, REGOFF_R13(%rsp)
468 468 movq %r14, REGOFF_R14(%rsp)
469 469 movq %gs:CPU_RTMP_R15, %r10
470 470 movq %r10, REGOFF_R15(%rsp)
471 471 movq $0, REGOFF_SAVFP(%rsp)
472 472 movq $0, REGOFF_SAVPC(%rsp)
473 473
474 474 /*
475 475 * Copy these registers here in case we end up stopped with
476 476 * someone (like, say, /proc) messing with our register state.
477 477 * We don't -restore- them unless we have to in update_sregs.
478 478 *
479 479 * Since userland -can't- change fsbase or gsbase directly,
480 480 * and capturing them involves two serializing instructions,
481 481 * we don't bother to capture them here.
482 482 */
483 483 xorl %ebx, %ebx
|
↓ open down ↓ |
450 lines elided |
↑ open up ↑ |
484 484 movw %ds, %bx
485 485 movq %rbx, REGOFF_DS(%rsp)
486 486 movw %es, %bx
487 487 movq %rbx, REGOFF_ES(%rsp)
488 488 movw %fs, %bx
489 489 movq %rbx, REGOFF_FS(%rsp)
490 490 movw %gs, %bx
491 491 movq %rbx, REGOFF_GS(%rsp)
492 492
493 493 /*
494 + * If we're trying to use TRAPTRACE though, I take that back: we're
495 + * probably debugging some problem in the SWAPGS logic and want to know
496 + * what the incoming gsbase was.
497 + *
498 + * Since we already did SWAPGS, record the KGSBASE.
499 + */
500 +#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
501 + movl $MSR_AMD_KGSBASE, %ecx
502 + rdmsr
503 + movl %eax, REGOFF_GSBASE(%rsp)
504 + movl %edx, REGOFF_GSBASE+4(%rsp)
505 +#endif
506 +
507 + /*
494 508 * Machine state saved in the regs structure on the stack
495 509 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
496 510 * %eax is the syscall number
497 511 * %rsp is the thread's stack, %r15 is curthread
498 512 * REG_RSP(%rsp) is the user's stack
499 513 */
500 514
501 515 SYSCALL_TRAPTRACE($TT_SYSC64)
502 516
503 517 movq %rsp, %rbp
504 518
505 519 movq T_LWP(%r15), %r14
506 520 ASSERT_NO_RUPDATE_PENDING(%r14)
507 521 ENABLE_INTR_FLAGS
508 522
509 523 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
510 524 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
511 525
512 526 ASSERT_LWPTOREGS(%r14, %rsp)
513 527
514 528 movb $LWP_SYS, LWP_STATE(%r14)
515 529 incq LWP_RU_SYSC(%r14)
516 530 movb $NORMALRETURN, LWP_EOSYS(%r14)
517 531
518 532 incq %gs:CPU_STATS_SYS_SYSCALL
519 533
520 534 movw %ax, T_SYSNUM(%r15)
521 535 movzbl T_PRE_SYS(%r15), %ebx
522 536 ORL_SYSCALLTRACE(%ebx)
523 537 testl %ebx, %ebx
524 538 jne _syscall_pre
525 539
526 540 _syscall_invoke:
527 541 movq REGOFF_RDI(%rbp), %rdi
528 542 movq REGOFF_RSI(%rbp), %rsi
529 543 movq REGOFF_RDX(%rbp), %rdx
530 544 movq REGOFF_RCX(%rbp), %rcx
531 545 movq REGOFF_R8(%rbp), %r8
532 546 movq REGOFF_R9(%rbp), %r9
533 547
534 548 cmpl $NSYSCALL, %eax
535 549 jae _syscall_ill
536 550 shll $SYSENT_SIZE_SHIFT, %eax
537 551 leaq sysent(%rax), %rbx
538 552
539 553 call *SY_CALLC(%rbx)
540 554
541 555 movq %rax, %r12
542 556 movq %rdx, %r13
543 557
544 558 /*
545 559 * If the handler returns two ints, then we need to split the
546 560 * 64-bit return value into two 32-bit values.
547 561 */
548 562 testw $SE_32RVAL2, SY_FLAGS(%rbx)
549 563 je 5f
550 564 movq %r12, %r13
551 565 shrq $32, %r13 /* upper 32-bits into %edx */
552 566 movl %r12d, %r12d /* lower 32-bits into %eax */
553 567 5:
554 568 /*
555 569 * Optimistically assume that there's no post-syscall
556 570 * work to do. (This is to avoid having to call syscall_mstate()
557 571 * with interrupts disabled)
558 572 */
559 573 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
560 574
561 575 /*
562 576 * We must protect ourselves from being descheduled here;
563 577 * If we were, and we ended up on another cpu, or another
564 578 * lwp got in ahead of us, it could change the segment
565 579 * registers without us noticing before we return to userland.
566 580 */
567 581 CLI(%r14)
568 582 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
569 583 jne _syscall_post
570 584
571 585 /*
572 586 * We need to protect ourselves against non-canonical return values
573 587 * because Intel doesn't check for them on sysret (AMD does). Canonical
574 588 * addresses on current amd64 processors only use 48-bits for VAs; an
575 589 * address is canonical if all upper bits (47-63) are identical. If we
576 590 * find a non-canonical %rip, we opt to go through the full
577 591 * _syscall_post path which takes us into an iretq which is not
578 592 * susceptible to the same problems sysret is.
579 593 *
580 594 * We're checking for a canonical address by first doing an arithmetic
581 595 * shift. This will fill in the remaining bits with the value of bit 63.
582 596 * If the address were canonical, the register would now have either all
583 597 * zeroes or all ones in it. Therefore we add one (inducing overflow)
584 598 * and compare against 1. A canonical address will either be zero or one
585 599 * at this point, hence the use of ja.
586 600 *
587 601 * At this point, r12 and r13 have the return value so we can't use
588 602 * those registers.
589 603 */
590 604 movq REGOFF_RIP(%rsp), %rcx
591 605 sarq $47, %rcx
592 606 incq %rcx
593 607 cmpq $1, %rcx
594 608 ja _syscall_post
595 609
596 610
597 611 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
598 612
599 613 movq %r12, REGOFF_RAX(%rsp)
600 614 movq %r13, REGOFF_RDX(%rsp)
601 615
602 616 /*
603 617 * To get back to userland, we need the return %rip in %rcx and
604 618 * the return %rfl in %r11d. The sysretq instruction also arranges
605 619 * to fix up %cs and %ss; everything else is our responsibility.
606 620 */
607 621 movq REGOFF_RDI(%rsp), %rdi
608 622 movq REGOFF_RSI(%rsp), %rsi
609 623 movq REGOFF_RDX(%rsp), %rdx
610 624 /* %rcx used to restore %rip value */
611 625
612 626 movq REGOFF_R8(%rsp), %r8
613 627 movq REGOFF_R9(%rsp), %r9
614 628 movq REGOFF_RAX(%rsp), %rax
615 629 movq REGOFF_RBX(%rsp), %rbx
616 630
617 631 movq REGOFF_RBP(%rsp), %rbp
618 632 movq REGOFF_R10(%rsp), %r10
619 633 /* %r11 used to restore %rfl value */
620 634 movq REGOFF_R12(%rsp), %r12
621 635
622 636 movq REGOFF_R13(%rsp), %r13
623 637 movq REGOFF_R14(%rsp), %r14
624 638 movq REGOFF_R15(%rsp), %r15
625 639
626 640 movq REGOFF_RIP(%rsp), %rcx
627 641 movl REGOFF_RFL(%rsp), %r11d
628 642
629 643 #if defined(__xpv)
630 644 addq $REGOFF_RIP, %rsp
631 645 #else
632 646 movq REGOFF_RSP(%rsp), %rsp
633 647 #endif
634 648
635 649 /*
636 650 * There can be no instructions between the ALTENTRY below and
637 651 * SYSRET or we could end up breaking brand support. See label usage
638 652 * in sn1_brand_syscall_callback for an example.
639 653 */
640 654 ASSERT_UPCALL_MASK_IS_SET
641 655 #if defined(__xpv)
642 656 SYSRETQ
643 657 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
644 658
645 659 /*
646 660 * We can only get here after executing a brand syscall
647 661 * interposition callback handler and simply need to
648 662 * "sysretq" back to userland. On the hypervisor this
649 663 * involves the iret hypercall which requires us to construct
650 664 * just enough of the stack needed for the hypercall.
651 665 * (rip, cs, rflags, rsp, ss).
652 666 */
653 667 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
654 668 movq %gs:CPU_THREAD, %r11
655 669 movq T_STACK(%r11), %rsp
656 670
657 671 movq %rcx, REGOFF_RIP(%rsp)
658 672 movl $UCS_SEL, REGOFF_CS(%rsp)
659 673 movq %gs:CPU_RTMP_RSP, %r11
660 674 movq %r11, REGOFF_RSP(%rsp)
661 675 pushfq
662 676 popq %r11 /* hypercall enables ints */
663 677 movq %r11, REGOFF_RFL(%rsp)
|
↓ open down ↓ |
160 lines elided |
↑ open up ↑ |
664 678 movl $UDS_SEL, REGOFF_SS(%rsp)
665 679 addq $REGOFF_RIP, %rsp
666 680 /*
667 681 * XXPV: see comment in SYSRETQ definition for future optimization
668 682 * we could take.
669 683 */
670 684 ASSERT_UPCALL_MASK_IS_SET
671 685 SYSRETQ
672 686 #else
673 687 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
674 - SWAPGS /* user gsbase */
675 - SYSRETQ
688 + jmp tr_sysretq
676 689 #endif
677 690 /*NOTREACHED*/
678 691 SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
679 692
680 693 _syscall_pre:
681 694 call pre_syscall
682 695 movl %eax, %r12d
683 696 testl %eax, %eax
684 697 jne _syscall_post_call
685 698 /*
686 699 * Didn't abort, so reload the syscall args and invoke the handler.
687 700 */
688 701 movzwl T_SYSNUM(%r15), %eax
689 702 jmp _syscall_invoke
690 703
691 704 _syscall_ill:
692 705 call nosys
693 706 movq %rax, %r12
694 707 movq %rdx, %r13
695 708 jmp _syscall_post_call
696 709
697 710 _syscall_post:
698 711 STI
699 712 /*
700 713 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
701 714 * so that we can account for the extra work it takes us to finish.
702 715 */
703 716 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
704 717 _syscall_post_call:
705 718 movq %r12, %rdi
706 719 movq %r13, %rsi
707 720 call post_syscall
708 721 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
709 722 jmp _sys_rtt
710 723 SET_SIZE(sys_syscall)
711 724 SET_SIZE(brand_sys_syscall)
712 725
713 726 #endif /* __lint */
714 727
715 728 #if defined(__lint)
716 729
717 730 /*ARGSUSED*/
718 731 void
719 732 sys_syscall32()
720 733 {}
721 734
722 735 #else /* __lint */
723 736
724 737 ENTRY_NP(brand_sys_syscall32)
725 738 SWAPGS /* kernel gsbase */
726 739 XPV_TRAP_POP
727 740 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
728 741 jmp nopop_sys_syscall32
729 742
730 743 ALTENTRY(sys_syscall32)
731 744 SWAPGS /* kernel gsbase */
732 745 XPV_TRAP_POP
733 746
734 747 nopop_sys_syscall32:
735 748 movl %esp, %r10d
736 749 movq %gs:CPU_THREAD, %r15
737 750 movq T_STACK(%r15), %rsp
738 751 movl %eax, %eax
739 752
740 753 movl $U32CS_SEL, REGOFF_CS(%rsp)
741 754 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
742 755 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
743 756 movq %r10, REGOFF_RSP(%rsp)
744 757 movl $UDS_SEL, REGOFF_SS(%rsp)
745 758
746 759 _syscall32_save:
747 760 movl %edi, REGOFF_RDI(%rsp)
748 761 movl %esi, REGOFF_RSI(%rsp)
749 762 movl %ebp, REGOFF_RBP(%rsp)
750 763 movl %ebx, REGOFF_RBX(%rsp)
751 764 movl %edx, REGOFF_RDX(%rsp)
752 765 movl %ecx, REGOFF_RCX(%rsp)
753 766 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
754 767 movq $0, REGOFF_SAVFP(%rsp)
755 768 movq $0, REGOFF_SAVPC(%rsp)
756 769
757 770 /*
758 771 * Copy these registers here in case we end up stopped with
759 772 * someone (like, say, /proc) messing with our register state.
760 773 * We don't -restore- them unless we have to in update_sregs.
761 774 *
762 775 * Since userland -can't- change fsbase or gsbase directly,
763 776 * we don't bother to capture them here.
764 777 */
765 778 xorl %ebx, %ebx
|
↓ open down ↓ |
80 lines elided |
↑ open up ↑ |
766 779 movw %ds, %bx
767 780 movq %rbx, REGOFF_DS(%rsp)
768 781 movw %es, %bx
769 782 movq %rbx, REGOFF_ES(%rsp)
770 783 movw %fs, %bx
771 784 movq %rbx, REGOFF_FS(%rsp)
772 785 movw %gs, %bx
773 786 movq %rbx, REGOFF_GS(%rsp)
774 787
775 788 /*
789 + * If we're trying to use TRAPTRACE though, I take that back: we're
790 + * probably debugging some problem in the SWAPGS logic and want to know
791 + * what the incoming gsbase was.
792 + *
793 + * Since we already did SWAPGS, record the KGSBASE.
794 + */
795 +#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
796 + movl $MSR_AMD_KGSBASE, %ecx
797 + rdmsr
798 + movl %eax, REGOFF_GSBASE(%rsp)
799 + movl %edx, REGOFF_GSBASE+4(%rsp)
800 +#endif
801 +
802 + /*
776 803 * Application state saved in the regs structure on the stack
777 804 * %eax is the syscall number
778 805 * %rsp is the thread's stack, %r15 is curthread
779 806 * REG_RSP(%rsp) is the user's stack
780 807 */
781 808
782 809 SYSCALL_TRAPTRACE32($TT_SYSC)
783 810
784 811 movq %rsp, %rbp
785 812
786 813 movq T_LWP(%r15), %r14
787 814 ASSERT_NO_RUPDATE_PENDING(%r14)
788 815
789 816 ENABLE_INTR_FLAGS
790 817
791 818 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
792 819 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
793 820
794 821 ASSERT_LWPTOREGS(%r14, %rsp)
795 822
796 823 incq %gs:CPU_STATS_SYS_SYSCALL
797 824
798 825 /*
799 826 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
800 827 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
801 828 * more succinctly:
802 829 *
803 830 * SA(MAXSYSARGS * sizeof (long)) == 64
804 831 */
805 832 #define SYS_DROP 64 /* drop for args */
806 833 subq $SYS_DROP, %rsp
807 834 movb $LWP_SYS, LWP_STATE(%r14)
808 835 movq %r15, %rdi
809 836 movq %rsp, %rsi
810 837 call syscall_entry
811 838
812 839 /*
813 840 * Fetch the arguments copied onto the kernel stack and put
814 841 * them in the right registers to invoke a C-style syscall handler.
815 842 * %rax contains the handler address.
816 843 *
817 844 * Ideas for making all this go faster of course include simply
818 845 * forcibly fetching 6 arguments from the user stack under lofault
819 846 * protection, reverting to copyin_args only when watchpoints
820 847 * are in effect.
821 848 *
822 849 * (If we do this, make sure that exec and libthread leave
823 850 * enough space at the top of the stack to ensure that we'll
824 851 * never do a fetch from an invalid page.)
825 852 *
826 853 * Lots of ideas here, but they won't really help with bringup B-)
827 854 * Correctness can't wait, performance can wait a little longer ..
828 855 */
829 856
830 857 movq %rax, %rbx
831 858 movl 0(%rsp), %edi
832 859 movl 8(%rsp), %esi
833 860 movl 0x10(%rsp), %edx
834 861 movl 0x18(%rsp), %ecx
835 862 movl 0x20(%rsp), %r8d
836 863 movl 0x28(%rsp), %r9d
837 864
838 865 call *SY_CALLC(%rbx)
839 866
840 867 movq %rbp, %rsp /* pop the args */
841 868
842 869 /*
843 870 * amd64 syscall handlers -always- return a 64-bit value in %rax.
844 871 * On the 32-bit kernel, they always return that value in %eax:%edx
845 872 * as required by the 32-bit ABI.
846 873 *
847 874 * Simulate the same behaviour by unconditionally splitting the
848 875 * return value in the same way.
849 876 */
850 877 movq %rax, %r13
851 878 shrq $32, %r13 /* upper 32-bits into %edx */
852 879 movl %eax, %r12d /* lower 32-bits into %eax */
853 880
854 881 /*
855 882 * Optimistically assume that there's no post-syscall
856 883 * work to do. (This is to avoid having to call syscall_mstate()
857 884 * with interrupts disabled)
858 885 */
859 886 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
860 887
861 888 /*
862 889 * We must protect ourselves from being descheduled here;
863 890 * If we were, and we ended up on another cpu, or another
864 891 * lwp got in ahead of us, it could change the segment
865 892 * registers without us noticing before we return to userland.
866 893 */
867 894 CLI(%r14)
868 895 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
869 896 jne _full_syscall_postsys32
870 897 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
871 898
872 899 /*
873 900 * To get back to userland, we need to put the return %rip in %rcx and
874 901 * the return %rfl in %r11d. The sysret instruction also arranges
875 902 * to fix up %cs and %ss; everything else is our responsibility.
876 903 */
877 904
878 905 movl %r12d, %eax /* %eax: rval1 */
879 906 movl REGOFF_RBX(%rsp), %ebx
880 907 /* %ecx used for return pointer */
881 908 movl %r13d, %edx /* %edx: rval2 */
|
↓ open down ↓ |
96 lines elided |
↑ open up ↑ |
882 909 movl REGOFF_RBP(%rsp), %ebp
883 910 movl REGOFF_RSI(%rsp), %esi
884 911 movl REGOFF_RDI(%rsp), %edi
885 912
886 913 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
887 914 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
888 915 movl REGOFF_RSP(%rsp), %esp
889 916
890 917 ASSERT_UPCALL_MASK_IS_SET
891 918 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
892 - SWAPGS /* user gsbase */
893 - SYSRETL
919 + jmp tr_sysretl
894 920 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
895 921 /*NOTREACHED*/
896 922
897 923 _full_syscall_postsys32:
898 924 STI
899 925 /*
900 926 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
901 927 * so that we can account for the extra work it takes us to finish.
902 928 */
903 929 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
904 930 movq %r15, %rdi
905 931 movq %r12, %rsi /* rval1 - %eax */
906 932 movq %r13, %rdx /* rval2 - %edx */
907 933 call syscall_exit
908 934 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
909 935 jmp _sys_rtt
910 936 SET_SIZE(sys_syscall32)
911 937 SET_SIZE(brand_sys_syscall32)
912 938
913 939 #endif /* __lint */
914 940
915 941 /*
916 942 * System call handler via the sysenter instruction
917 943 * Used only for 32-bit system calls on the 64-bit kernel.
918 944 *
919 945 * The caller in userland has arranged that:
920 946 *
921 947 * - %eax contains the syscall number
922 948 * - %ecx contains the user %esp
923 949 * - %edx contains the return %eip
924 950 * - the user stack contains the args to the syscall
925 951 *
926 952 * Hardware and (privileged) initialization code have arranged that by
927 953 * the time the sysenter instructions completes:
|
↓ open down ↓ |
24 lines elided |
↑ open up ↑ |
928 954 *
929 955 * - %rip is pointing to sys_sysenter (below).
930 956 * - %cs and %ss are set to kernel text and stack (data) selectors.
931 957 * - %rsp is pointing at the lwp's stack
932 958 * - interrupts have been disabled.
933 959 *
934 960 * Note that we are unable to return both "rvals" to userland with
935 961 * this call, as %edx is used by the sysexit instruction.
936 962 *
937 963 * One final complication in this routine is its interaction with
938 - * single-stepping in a debugger. For most of the system call mechanisms,
939 - * the CPU automatically clears the single-step flag before we enter the
940 - * kernel. The sysenter mechanism does not clear the flag, so a user
941 - * single-stepping through a libc routine may suddenly find themself
942 - * single-stepping through the kernel. To detect this, kmdb compares the
943 - * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
944 - * If it finds that we have single-stepped to a sysenter entry point, it
945 - * explicitly clears the flag and executes the sys_sysenter routine.
964 + * single-stepping in a debugger. For most of the system call mechanisms, the
965 + * CPU automatically clears the single-step flag before we enter the kernel.
966 + * The sysenter mechanism does not clear the flag, so a user single-stepping
967 + * through a libc routine may suddenly find themself single-stepping through the
968 + * kernel. To detect this, kmdb and trap() both compare the trap %pc to the
969 + * [brand_]sys_enter addresses on each single-step trap. If it finds that we
970 + * have single-stepped to a sysenter entry point, it explicitly clears the flag
971 + * and executes the sys_sysenter routine.
946 972 *
947 - * One final complication in this final complication is the fact that we
948 - * have two different entry points for sysenter: brand_sys_sysenter and
949 - * sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping
950 - * through the kernel with kmdb, we will eventually hit the instruction at
951 - * sys_sysenter. kmdb cannot distinguish between that valid single-step
952 - * and the undesirable one mentioned above. To avoid this situation, we
953 - * simply add a jump over the instruction at sys_sysenter to make it
954 - * impossible to single-step to it.
973 + * One final complication in this final complication is the fact that we have
974 + * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
975 + * If we enter at brand_sys_sysenter and start single-stepping through the
976 + * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
977 + * kmdb cannot distinguish between that valid single-step and the undesirable
978 + * one mentioned above. To avoid this situation, we simply add a jump over the
979 + * instruction at sys_sysenter to make it impossible to single-step to it.
955 980 */
956 981 #if defined(__lint)
957 982
958 983 void
959 984 sys_sysenter()
960 985 {}
961 986
962 987 #else /* __lint */
963 988
964 989 ENTRY_NP(brand_sys_sysenter)
965 990 SWAPGS /* kernel gsbase */
966 991 ALTENTRY(_brand_sys_sysenter_post_swapgs)
992 +
967 993 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
968 994 /*
969 995 * Jump over sys_sysenter to allow single-stepping as described
970 996 * above.
971 997 */
972 998 jmp _sys_sysenter_post_swapgs
973 999
974 1000 ALTENTRY(sys_sysenter)
975 1001 SWAPGS /* kernel gsbase */
976 -
977 1002 ALTENTRY(_sys_sysenter_post_swapgs)
1003 +
978 1004 movq %gs:CPU_THREAD, %r15
979 1005
980 1006 movl $U32CS_SEL, REGOFF_CS(%rsp)
981 1007 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
982 1008 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
1009 + /*
1010 + * NOTE: none of the instructions that run before we get here should
1011 + * clobber bits in (R)FLAGS! This includes the kpti trampoline.
1012 + */
983 1013 pushfq
984 1014 popq %r10
985 1015 movl $UDS_SEL, REGOFF_SS(%rsp)
986 1016
987 1017 /*
988 1018 * Set the interrupt flag before storing the flags to the
989 1019 * flags image on the stack so we can return to user with
990 1020 * interrupts enabled if we return via sys_rtt_syscall32
991 1021 */
992 1022 orq $PS_IE, %r10
993 1023 movq %r10, REGOFF_RFL(%rsp)
994 1024
995 1025 movl %edi, REGOFF_RDI(%rsp)
996 1026 movl %esi, REGOFF_RSI(%rsp)
997 1027 movl %ebp, REGOFF_RBP(%rsp)
998 1028 movl %ebx, REGOFF_RBX(%rsp)
999 1029 movl %edx, REGOFF_RDX(%rsp)
1000 1030 movl %ecx, REGOFF_RCX(%rsp)
1001 1031 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
1002 1032 movq $0, REGOFF_SAVFP(%rsp)
1003 1033 movq $0, REGOFF_SAVPC(%rsp)
1004 1034
1005 1035 /*
1006 1036 * Copy these registers here in case we end up stopped with
1007 1037 * someone (like, say, /proc) messing with our register state.
1008 1038 * We don't -restore- them unless we have to in update_sregs.
1009 1039 *
1010 1040 * Since userland -can't- change fsbase or gsbase directly,
1011 1041 * we don't bother to capture them here.
1012 1042 */
1013 1043 xorl %ebx, %ebx
|
↓ open down ↓ |
21 lines elided |
↑ open up ↑ |
1014 1044 movw %ds, %bx
1015 1045 movq %rbx, REGOFF_DS(%rsp)
1016 1046 movw %es, %bx
1017 1047 movq %rbx, REGOFF_ES(%rsp)
1018 1048 movw %fs, %bx
1019 1049 movq %rbx, REGOFF_FS(%rsp)
1020 1050 movw %gs, %bx
1021 1051 movq %rbx, REGOFF_GS(%rsp)
1022 1052
1023 1053 /*
1054 + * If we're trying to use TRAPTRACE though, I take that back: we're
1055 + * probably debugging some problem in the SWAPGS logic and want to know
1056 + * what the incoming gsbase was.
1057 + *
1058 + * Since we already did SWAPGS, record the KGSBASE.
1059 + */
1060 +#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
1061 + movl $MSR_AMD_KGSBASE, %ecx
1062 + rdmsr
1063 + movl %eax, REGOFF_GSBASE(%rsp)
1064 + movl %edx, REGOFF_GSBASE+4(%rsp)
1065 +#endif
1066 +
1067 + /*
1024 1068 * Application state saved in the regs structure on the stack
1025 1069 * %eax is the syscall number
1026 1070 * %rsp is the thread's stack, %r15 is curthread
1027 1071 * REG_RSP(%rsp) is the user's stack
1028 1072 */
1029 1073
1030 1074 SYSCALL_TRAPTRACE($TT_SYSENTER)
1031 1075
1032 1076 movq %rsp, %rbp
1033 1077
1034 1078 movq T_LWP(%r15), %r14
1035 1079 ASSERT_NO_RUPDATE_PENDING(%r14)
1036 1080
1037 1081 ENABLE_INTR_FLAGS
1038 1082
1039 1083 /*
1040 1084 * Catch 64-bit process trying to issue sysenter instruction
1041 1085 * on Nocona based systems.
1042 1086 */
1043 1087 movq LWP_PROCP(%r14), %rax
1044 1088 cmpq $DATAMODEL_ILP32, P_MODEL(%rax)
1045 1089 je 7f
1046 1090
1047 1091 /*
1048 1092 * For a non-32-bit process, simulate a #ud, since that's what
1049 1093 * native hardware does. The traptrace entry (above) will
1050 1094 * let you know what really happened.
1051 1095 */
1052 1096 movq $T_ILLINST, REGOFF_TRAPNO(%rsp)
1053 1097 movq REGOFF_CS(%rsp), %rdi
1054 1098 movq %rdi, REGOFF_ERR(%rsp)
1055 1099 movq %rsp, %rdi
1056 1100 movq REGOFF_RIP(%rsp), %rsi
1057 1101 movl %gs:CPU_ID, %edx
1058 1102 call trap
1059 1103 jmp _sys_rtt
1060 1104 7:
1061 1105
1062 1106 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1063 1107 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */
1064 1108
1065 1109 ASSERT_LWPTOREGS(%r14, %rsp)
1066 1110
1067 1111 incq %gs:CPU_STATS_SYS_SYSCALL
1068 1112
1069 1113 /*
1070 1114 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1071 1115 * placed into 64-bit (long) arg slots, plus one 64-bit
1072 1116 * (long) arg count, maintaining 16 byte alignment.
1073 1117 */
1074 1118 subq $SYS_DROP, %rsp
1075 1119 movb $LWP_SYS, LWP_STATE(%r14)
1076 1120 movq %r15, %rdi
1077 1121 movq %rsp, %rsi
1078 1122 call syscall_entry
1079 1123
1080 1124 /*
1081 1125 * Fetch the arguments copied onto the kernel stack and put
1082 1126 * them in the right registers to invoke a C-style syscall handler.
1083 1127 * %rax contains the handler address.
1084 1128 */
1085 1129 movq %rax, %rbx
1086 1130 movl 0(%rsp), %edi
1087 1131 movl 8(%rsp), %esi
1088 1132 movl 0x10(%rsp), %edx
1089 1133 movl 0x18(%rsp), %ecx
1090 1134 movl 0x20(%rsp), %r8d
1091 1135 movl 0x28(%rsp), %r9d
1092 1136
1093 1137 call *SY_CALLC(%rbx)
1094 1138
1095 1139 movq %rbp, %rsp /* pop the args */
1096 1140
1097 1141 /*
1098 1142 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1099 1143 * On the 32-bit kernel, the always return that value in %eax:%edx
1100 1144 * as required by the 32-bit ABI.
1101 1145 *
1102 1146 * Simulate the same behaviour by unconditionally splitting the
1103 1147 * return value in the same way.
1104 1148 */
1105 1149 movq %rax, %r13
1106 1150 shrq $32, %r13 /* upper 32-bits into %edx */
1107 1151 movl %eax, %r12d /* lower 32-bits into %eax */
1108 1152
1109 1153 /*
1110 1154 * Optimistically assume that there's no post-syscall
|
↓ open down ↓ |
77 lines elided |
↑ open up ↑ |
1111 1155 * work to do. (This is to avoid having to call syscall_mstate()
1112 1156 * with interrupts disabled)
1113 1157 */
1114 1158 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1115 1159
1116 1160 /*
1117 1161 * We must protect ourselves from being descheduled here;
1118 1162 * If we were, and we ended up on another cpu, or another
1119 1163 * lwp got int ahead of us, it could change the segment
1120 1164 * registers without us noticing before we return to userland.
1165 + *
1166 + * This cli is undone in the tr_sysexit trampoline code.
1121 1167 */
1122 1168 cli
1123 1169 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1124 1170 jne _full_syscall_postsys32
1125 1171 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1126 1172
1127 1173 /*
1128 1174 * To get back to userland, load up the 32-bit registers and
1129 1175 * sysexit back where we came from.
1130 1176 */
1131 1177
1132 1178 /*
1133 1179 * Interrupts will be turned on by the 'sti' executed just before
1134 1180 * sysexit. The following ensures that restoring the user's rflags
1135 1181 * doesn't enable interrupts too soon.
1136 1182 */
1137 1183 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1138 1184
1139 1185 /*
1140 1186 * (There's no point in loading up %edx because the sysexit
1141 1187 * mechanism smashes it.)
1142 1188 */
1143 1189 movl %r12d, %eax
|
↓ open down ↓ |
13 lines elided |
↑ open up ↑ |
1144 1190 movl REGOFF_RBX(%rsp), %ebx
1145 1191 movl REGOFF_RBP(%rsp), %ebp
1146 1192 movl REGOFF_RSI(%rsp), %esi
1147 1193 movl REGOFF_RDI(%rsp), %edi
1148 1194
1149 1195 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */
1150 1196 pushq REGOFF_RFL(%rsp)
1151 1197 popfq
1152 1198 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */
1153 1199 ALTENTRY(sys_sysenter_swapgs_sysexit)
1154 - swapgs
1155 - sti
1156 - sysexit
1200 + jmp tr_sysexit
1157 1201 SET_SIZE(sys_sysenter_swapgs_sysexit)
1158 1202 SET_SIZE(sys_sysenter)
1159 1203 SET_SIZE(_sys_sysenter_post_swapgs)
1160 1204 SET_SIZE(brand_sys_sysenter)
1161 1205
1162 1206 #endif /* __lint */
1163 1207
1164 1208 /*
1165 1209 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1166 1210 * the generic i386 libc to do system calls. We do a small amount of setup
1167 1211 * before jumping into the existing sys_syscall32 path.
1168 1212 */
1169 1213 #if defined(__lint)
1170 1214
1171 1215 /*ARGSUSED*/
1172 1216 void
1173 1217 sys_syscall_int()
1174 1218 {}
1175 1219
1176 1220 #else /* __lint */
1177 1221
1178 1222 ENTRY_NP(brand_sys_syscall_int)
1179 1223 SWAPGS /* kernel gsbase */
1180 1224 XPV_TRAP_POP
1181 1225 call smap_enable
1182 1226 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1183 1227 jmp nopop_syscall_int
1184 1228
1185 1229 ALTENTRY(sys_syscall_int)
1186 1230 SWAPGS /* kernel gsbase */
1187 1231 XPV_TRAP_POP
1188 1232 call smap_enable
1189 1233
1190 1234 nopop_syscall_int:
1191 1235 movq %gs:CPU_THREAD, %r15
1192 1236 movq T_STACK(%r15), %rsp
1193 1237 movl %eax, %eax
1194 1238 /*
1195 1239 * Set t_post_sys on this thread to force ourselves out via the slow
1196 1240 * path. It might be possible at some later date to optimize this out
|
↓ open down ↓ |
30 lines elided |
↑ open up ↑ |
1197 1241 * and use a faster return mechanism.
1198 1242 */
1199 1243 movb $1, T_POST_SYS(%r15)
1200 1244 CLEAN_CS
1201 1245 jmp _syscall32_save
1202 1246 /*
1203 1247 * There should be no instructions between this label and SWAPGS/IRET
1204 1248 * or we could end up breaking branded zone support. See the usage of
1205 1249 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1206 1250 * for examples.
1251 + *
1252 + * We want to swapgs to maintain the invariant that all entries into
1253 + * tr_iret_user are done on the user gsbase.
1207 1254 */
1208 - ALTENTRY(sys_sysint_swapgs_iret)
1209 - SWAPGS /* user gsbase */
1210 - IRET
1255 + ALTENTRY(sys_sysint_swapgs_iret)
1256 + SWAPGS
1257 + jmp tr_iret_user
1211 1258 /*NOTREACHED*/
1212 1259 SET_SIZE(sys_sysint_swapgs_iret)
1213 1260 SET_SIZE(sys_syscall_int)
1214 1261 SET_SIZE(brand_sys_syscall_int)
1215 1262
1216 1263 #endif /* __lint */
1217 1264
1218 1265 /*
1219 1266 * Legacy 32-bit applications and old libc implementations do lcalls;
1220 1267 * we should never get here because the LDT entry containing the syscall
1221 1268 * segment descriptor has the "segment present" bit cleared, which means
1222 1269 * we end up processing those system calls in trap() via a not-present trap.
1223 1270 *
1224 1271 * We do it this way because a call gate unhelpfully does -nothing- to the
1225 1272 * interrupt flag bit, so an interrupt can run us just after the lcall
1226 1273 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and
1227 1274 * INTR_POP paths would have to be slightly more complex to dance around
1228 1275 * this problem, and end up depending explicitly on the first
1229 1276 * instruction of this handler being either swapgs or cli.
1230 1277 */
1231 1278
1232 1279 #if defined(__lint)
1233 1280
1234 1281 /*ARGSUSED*/
1235 1282 void
1236 1283 sys_lcall32()
1237 1284 {}
1238 1285
1239 1286 #else /* __lint */
1240 1287
1241 1288 ENTRY_NP(sys_lcall32)
1242 1289 SWAPGS /* kernel gsbase */
1243 1290 pushq $0
1244 1291 pushq %rbp
1245 1292 movq %rsp, %rbp
1246 1293 leaq __lcall_panic_str(%rip), %rdi
1247 1294 xorl %eax, %eax
1248 1295 call panic
1249 1296 SET_SIZE(sys_lcall32)
1250 1297
1251 1298 __lcall_panic_str:
1252 1299 .string "sys_lcall32: shouldn't be here!"
1253 1300
1254 1301 /*
1255 1302 * Declare a uintptr_t which covers the entire pc range of syscall
1256 1303 * handlers for the stack walkers that need this.
1257 1304 */
1258 1305 .align CPTRSIZE
1259 1306 .globl _allsyscalls_size
1260 1307 .type _allsyscalls_size, @object
1261 1308 _allsyscalls_size:
1262 1309 .NWORD . - _allsyscalls
1263 1310 SET_SIZE(_allsyscalls_size)
1264 1311
1265 1312 #endif /* __lint */
1266 1313
1267 1314 /*
1268 1315 * These are the thread context handlers for lwps using sysenter/sysexit.
1269 1316 */
1270 1317
1271 1318 #if defined(__lint)
1272 1319
1273 1320 /*ARGSUSED*/
1274 1321 void
1275 1322 sep_save(void *ksp)
1276 1323 {}
1277 1324
1278 1325 /*ARGSUSED*/
1279 1326 void
1280 1327 sep_restore(void *ksp)
1281 1328 {}
1282 1329
1283 1330 #else /* __lint */
1284 1331
1285 1332 /*
1286 1333 * setting this value to zero as we switch away causes the
1287 1334 * stack-pointer-on-sysenter to be NULL, ensuring that we
1288 1335 * don't silently corrupt another (preempted) thread stack
1289 1336 * when running an lwp that (somehow) didn't get sep_restore'd
1290 1337 */
1291 1338 ENTRY_NP(sep_save)
1292 1339 xorl %edx, %edx
1293 1340 xorl %eax, %eax
1294 1341 movl $MSR_INTC_SEP_ESP, %ecx
1295 1342 wrmsr
1296 1343 ret
1297 1344 SET_SIZE(sep_save)
1298 1345
1299 1346 /*
1300 1347 * Update the kernel stack pointer as we resume onto this cpu.
1301 1348 */
1302 1349 ENTRY_NP(sep_restore)
1303 1350 movq %rdi, %rdx
1304 1351 shrq $32, %rdx
1305 1352 movl %edi, %eax
1306 1353 movl $MSR_INTC_SEP_ESP, %ecx
1307 1354 wrmsr
1308 1355 ret
1309 1356 SET_SIZE(sep_restore)
1310 1357
1311 1358 #endif /* __lint */
|
↓ open down ↓ |
91 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX