1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2019 Joyent, Inc.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 #include <sys/asm_linkage.h>
28 #include <sys/asm_misc.h>
29 #include <sys/regset.h>
30 #include <sys/privregs.h>
31 #include <sys/psw.h>
32 #include <sys/machbrand.h>
33
34 #if defined(__lint)
35
36 #include <sys/types.h>
37 #include <sys/thread.h>
38 #include <sys/systm.h>
39
40 #else /* __lint */
41
42 #include <sys/segments.h>
43 #include <sys/pcb.h>
44 #include <sys/trap.h>
45 #include <sys/ftrace.h>
46 #include <sys/traptrace.h>
47 #include <sys/clock.h>
48 #include <sys/model.h>
49 #include <sys/panic.h>
50
51 #if defined(__xpv)
52 #include <sys/hypervisor.h>
53 #endif
54
55 #include "assym.h"
56
57 #endif /* __lint */
58
59 /*
60 * We implement five flavours of system call entry points
61 *
62 * - syscall/sysretq (amd64 generic)
63 * - syscall/sysretl (i386 plus SYSC bit)
64 * - sysenter/sysexit (i386 plus SEP bit)
65 * - int/iret (i386 generic)
66 * - lcall/iret (i386 generic)
67 *
68 * The current libc included in Solaris uses int/iret as the base unoptimized
69 * kernel entry method. Older libc implementations and legacy binaries may use
70 * the lcall call gate, so it must continue to be supported.
71 *
72 * System calls that use an lcall call gate are processed in trap() via a
73 * segment-not-present trap, i.e. lcalls are extremely slow(!).
74 *
75 * The basic pattern used in the 32-bit SYSC handler at this point in time is
76 * to have the bare minimum of assembler, and get to the C handlers as
77 * quickly as possible.
78 *
79 * The 64-bit handler is much closer to the sparcv9 handler; that's
80 * because of passing arguments in registers. The 32-bit world still
81 * passes arguments on the stack -- that makes that handler substantially
82 * more complex.
83 *
84 * The two handlers share a few code fragments which are broken
85 * out into preprocessor macros below.
86 *
87 * XX64 come back and speed all this up later. The 32-bit stuff looks
88 * especially easy to speed up the argument copying part ..
89 *
90 *
91 * Notes about segment register usage (c.f. the 32-bit kernel)
92 *
93 * In the 32-bit kernel, segment registers are dutifully saved and
94 * restored on all mode transitions because the kernel uses them directly.
95 * When the processor is running in 64-bit mode, segment registers are
96 * largely ignored.
97 *
98 * %cs and %ss
99 * controlled by the hardware mechanisms that make mode transitions
100 *
101 * The remaining segment registers have to either be pointing at a valid
102 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
103 *
104 * %ds and %es
105 * always ignored
106 *
107 * %fs and %gs
108 * fsbase and gsbase are used to control the place they really point at.
109 * The kernel only depends on %gs, and controls its own gsbase via swapgs
110 *
111 * Note that loading segment registers is still costly because the GDT
112 * lookup still happens (this is because the hardware can't know that we're
113 * not setting up these segment registers for a 32-bit program). Thus we
114 * avoid doing this in the syscall path, and defer them to lwp context switch
115 * handlers, so the register values remain virtualized to the lwp.
116 */
117
118 #if defined(SYSCALLTRACE)
119 #define ORL_SYSCALLTRACE(r32) \
120 orl syscalltrace(%rip), r32
121 #else
122 #define ORL_SYSCALLTRACE(r32)
123 #endif
124
125 /*
126 * In the 32-bit kernel, we do absolutely nothing before getting into the
127 * brand callback checks. In 64-bit land, we do swapgs and then come here.
128 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
129 * are still unused.
130 *
131 * Check if a brand_mach_ops callback is defined for the specified callback_id
132 * type. If so invoke it with the kernel's %gs value loaded and the following
133 * data on the stack:
134 *
135 * stack: --------------------------------------
136 * 32 | callback pointer |
137 * | 24 | user (or interrupt) stack pointer |
138 * | 16 | lwp pointer |
139 * v 8 | userland return address |
140 * 0 | callback wrapper return addr |
141 * --------------------------------------
142 *
143 * Since we're pushing the userland return address onto the kernel stack
144 * we need to get that address without accessing the user's stack (since we
145 * can't trust that data). There are different ways to get the userland
146 * return address depending on how the syscall trap was made:
147 *
148 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
149 * b) For sys_sysenter the return address is in %rdx.
150 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
151 * the stack pointer points at the state saved when we took the interrupt:
152 * ------------------------
153 * | | user's %ss |
154 * | | user's %esp |
155 * | | EFLAGS register |
156 * v | user's %cs |
157 * | user's %eip |
158 * ------------------------
159 *
160 * The 2nd parameter to the BRAND_CALLBACK macro is either the
161 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
162 * used to generate the proper code to get the userland return address for
163 * each syscall entry point.
164 *
165 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
166 * is available as a scratch register within the callback. If the callback
167 * returns within the kernel then this macro will restore %r15. If the
168 * callback is going to return directly to userland then it should restore
169 * %r15 before returning to userland.
170 */
171 #define BRAND_URET_FROM_REG(rip_reg) \
172 pushq rip_reg /* push the return address */
173
174 /*
175 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
176 * is currently pointing at the user return address (%eip).
177 */
178 #define BRAND_URET_FROM_INTR_STACK() \
179 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
180 pushq (%r15) /* push the return address */
181
182 #define BRAND_CALLBACK(callback_id, push_userland_ret) \
183 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
184 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
185 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
186 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
187 subq $16, %rsp /* save space for 2 pointers */ ;\
188 pushq %r14 /* save %r14 */ ;\
189 movq %gs:CPU_RTMP_RSP, %r14 ;\
190 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
191 popq %r14 /* restore %r14 */ ;\
192 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
193 pushq %r15 /* push the lwp pointer */ ;\
194 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
195 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
196 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
197 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
198 cmpq $0, %r15 ;\
199 je 1f ;\
200 movq %r15, 16(%rsp) /* save the callback pointer */ ;\
201 push_userland_ret /* push the return address */ ;\
202 movq 24(%rsp), %r15 /* load callback pointer */ ;\
203 INDIRECT_CALL_REG(r15) /* call callback */ ;\
204 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
205 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
206
207 #define MSTATE_TRANSITION(from, to) \
208 movl $from, %edi; \
209 movl $to, %esi; \
210 call syscall_mstate
211
212 /*
213 * Check to see if a simple (direct) return is possible i.e.
214 *
215 * if (t->t_post_sys_ast | syscalltrace |
216 * lwp->lwp_pcb.pcb_rupdate == 1)
217 * do full version ;
218 *
219 * Preconditions:
220 * - t is curthread
221 * Postconditions:
222 * - condition code NE is set if post-sys is too complex
223 * - rtmp is zeroed if it isn't (we rely on this!)
224 * - ltmp is smashed
225 */
226 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
227 movq T_LWP(t), ltmp; \
228 movzbl PCB_RUPDATE(ltmp), rtmp; \
229 ORL_SYSCALLTRACE(rtmp); \
230 orl T_POST_SYS_AST(t), rtmp; \
231 cmpl $0, rtmp
232
233 /*
234 * Fix up the lwp, thread, and eflags for a successful return
235 *
236 * Preconditions:
237 * - zwreg contains zero
238 */
239 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
240 movb $LWP_USER, LWP_STATE(lwp); \
241 movw zwreg, T_SYSNUM(t); \
242 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
243
244 /*
245 * ASSERT(lwptoregs(lwp) == rp);
246 *
247 * This may seem obvious, but very odd things happen if this
248 * assertion is false
249 *
250 * Preconditions:
251 * (%rsp is ready for normal call sequence)
252 * Postconditions (if assertion is true):
253 * %r11 is smashed
254 *
255 * ASSERT(rp->r_cs == descnum)
256 *
257 * The code selector is written into the regs structure when the
258 * lwp stack is created. We use this ASSERT to validate that
259 * the regs structure really matches how we came in.
260 *
261 * Preconditions:
262 * (%rsp is ready for normal call sequence)
263 * Postconditions (if assertion is true):
264 * -none-
265 *
266 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
267 *
268 * If this is false, it meant that we returned to userland without
269 * updating the segment registers as we were supposed to.
270 *
271 * Note that we must ensure no interrupts or other traps intervene
272 * between entering privileged mode and performing the assertion,
273 * otherwise we may perform a context switch on the thread, which
274 * will end up setting pcb_rupdate to 1 again.
275 *
276 * ASSERT(%cr0 & CR0_TS == 0);
277 * Preconditions:
278 * (%rsp is ready for normal call sequence)
279 * Postconditions (if assertion is true):
280 * (specified register is clobbered)
281 *
282 * Check to make sure that we are returning to user land and that CR0.TS
283 * is not set. This is required as part of the eager FPU (see
284 * uts/intel/ia32/os/fpu.c for more information).
285 */
286
287 #if defined(DEBUG)
288
289 #if !defined(__lint)
290
291 __lwptoregs_msg:
292 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
293
294 __codesel_msg:
295 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
296
297 __no_rupdate_msg:
298 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
299
300 __bad_ts_msg:
301 .string "sysscall_asm_amd64.s:%d CR0.TS set on user return"
302
303 #endif /* !__lint */
304
305 #define ASSERT_LWPTOREGS(lwp, rp) \
306 movq LWP_REGS(lwp), %r11; \
307 cmpq rp, %r11; \
308 je 7f; \
309 leaq __lwptoregs_msg(%rip), %rdi; \
310 movl $__LINE__, %esi; \
311 movq lwp, %rdx; \
312 movq %r11, %rcx; \
313 movq rp, %r8; \
314 xorl %eax, %eax; \
315 call panic; \
316 7:
317
318 #define ASSERT_NO_RUPDATE_PENDING(lwp) \
319 testb $0x1, PCB_RUPDATE(lwp); \
320 je 8f; \
321 movq lwp, %rdx; \
322 leaq __no_rupdate_msg(%rip), %rdi; \
323 movl $__LINE__, %esi; \
324 xorl %eax, %eax; \
325 call panic; \
326 8:
327
328 #define ASSERT_CR0TS_ZERO(reg) \
329 movq %cr0, reg; \
330 testq $CR0_TS, reg; \
331 jz 9f; \
332 leaq __bad_ts_msg(%rip), %rdi; \
333 movl $__LINE__, %esi; \
334 xorl %eax, %eax; \
335 call panic; \
336 9:
337
338 #else
339 #define ASSERT_LWPTOREGS(lwp, rp)
340 #define ASSERT_NO_RUPDATE_PENDING(lwp)
341 #define ASSERT_CR0TS_ZERO(reg)
342 #endif
343
344 /*
345 * Do the traptrace thing and restore any registers we used
346 * in situ. Assumes that %rsp is pointing at the base of
347 * the struct regs, obviously ..
348 */
349 #ifdef TRAPTRACE
350 #define SYSCALL_TRAPTRACE(ttype) \
351 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
352 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
353 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
354 movq REGOFF_RAX(%rsp), %rax; \
355 movq REGOFF_RBX(%rsp), %rbx; \
356 movq REGOFF_RCX(%rsp), %rcx; \
357 movq REGOFF_RDX(%rsp), %rdx; \
358 movl %eax, TTR_SYSNUM(%rdi); \
359 movq REGOFF_RDI(%rsp), %rdi
360
361 #define SYSCALL_TRAPTRACE32(ttype) \
362 SYSCALL_TRAPTRACE(ttype); \
363 /* paranoia: clean the top 32-bits of the registers */ \
364 orl %eax, %eax; \
365 orl %ebx, %ebx; \
366 orl %ecx, %ecx; \
367 orl %edx, %edx; \
368 orl %edi, %edi
369 #else /* TRAPTRACE */
370 #define SYSCALL_TRAPTRACE(ttype)
371 #define SYSCALL_TRAPTRACE32(ttype)
372 #endif /* TRAPTRACE */
373
374 /*
375 * The 64-bit libc syscall wrapper does this:
376 *
377 * fn(<args>)
378 * {
379 * movq %rcx, %r10 -- because syscall smashes %rcx
380 * movl $CODE, %eax
381 * syscall
382 * <error processing>
383 * }
384 *
385 * Thus when we come into the kernel:
386 *
387 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
388 * %rax is the syscall number
389 * %r12-%r15 contain caller state
390 *
391 * The syscall instruction arranges that:
392 *
393 * %rcx contains the return %rip
394 * %r11d contains bottom 32-bits of %rflags
395 * %rflags is masked (as determined by the SFMASK msr)
396 * %cs is set to UCS_SEL (as determined by the STAR msr)
397 * %ss is set to UDS_SEL (as determined by the STAR msr)
398 * %rip is set to sys_syscall (as determined by the LSTAR msr)
399 *
400 * Or in other words, we have no registers available at all.
401 * Only swapgs can save us!
402 *
403 * Under the hypervisor, the swapgs has happened already. However, the
404 * state of the world is very different from that we're familiar with.
405 *
406 * In particular, we have a stack structure like that for interrupt
407 * gates, except that the %cs and %ss registers are modified for reasons
408 * that are not entirely clear. Critically, the %rcx/%r11 values do
409 * *not* reflect the usage of those registers under a 'real' syscall[1];
410 * the stack, therefore, looks like this:
411 *
412 * 0x0(rsp) potentially junk %rcx
413 * 0x8(rsp) potentially junk %r11
414 * 0x10(rsp) user %rip
415 * 0x18(rsp) modified %cs
416 * 0x20(rsp) user %rflags
417 * 0x28(rsp) user %rsp
418 * 0x30(rsp) modified %ss
419 *
420 *
421 * and before continuing on, we must load the %rip into %rcx and the
422 * %rflags into %r11.
423 *
424 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
425 * Sigh.
426 */
427 #if defined(__xpv)
428 #define XPV_SYSCALL_PROD \
429 movq 0x10(%rsp), %rcx; \
430 movq 0x20(%rsp), %r11; \
431 movq 0x28(%rsp), %rsp
432 #else
433 #define XPV_SYSCALL_PROD /* nothing */
434 #endif
435
436 #if defined(__lint)
437
438 /*ARGSUSED*/
439 void
440 sys_syscall()
441 {}
442
443 void
444 _allsyscalls()
445 {}
446
447 size_t _allsyscalls_size;
448
449 #else /* __lint */
450
451 ENTRY_NP2(brand_sys_syscall,_allsyscalls)
452 SWAPGS /* kernel gsbase */
453 XPV_SYSCALL_PROD
454 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
455 jmp noprod_sys_syscall
456
457 ALTENTRY(sys_syscall)
458 SWAPGS /* kernel gsbase */
459 XPV_SYSCALL_PROD
460
461 noprod_sys_syscall:
462 movq %r15, %gs:CPU_RTMP_R15
463 movq %rsp, %gs:CPU_RTMP_RSP
464
465 movq %gs:CPU_THREAD, %r15
466 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
467
468 ASSERT_UPCALL_MASK_IS_SET
469
470 movl $UCS_SEL, REGOFF_CS(%rsp)
471 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
472 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
473 movl $UDS_SEL, REGOFF_SS(%rsp)
474
475 movl %eax, %eax /* wrapper: sysc# -> %eax */
476 movq %rdi, REGOFF_RDI(%rsp)
477 movq %rsi, REGOFF_RSI(%rsp)
478 movq %rdx, REGOFF_RDX(%rsp)
479 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
480 movq %r10, %rcx /* arg[3] for direct calls */
481
482 movq %r8, REGOFF_R8(%rsp)
483 movq %r9, REGOFF_R9(%rsp)
484 movq %rax, REGOFF_RAX(%rsp)
485 movq %rbx, REGOFF_RBX(%rsp)
486
487 movq %rbp, REGOFF_RBP(%rsp)
488 movq %r10, REGOFF_R10(%rsp)
489 movq %gs:CPU_RTMP_RSP, %r11
490 movq %r11, REGOFF_RSP(%rsp)
491 movq %r12, REGOFF_R12(%rsp)
492
493 movq %r13, REGOFF_R13(%rsp)
494 movq %r14, REGOFF_R14(%rsp)
495 movq %gs:CPU_RTMP_R15, %r10
496 movq %r10, REGOFF_R15(%rsp)
497 movq $0, REGOFF_SAVFP(%rsp)
498 movq $0, REGOFF_SAVPC(%rsp)
499
500 /*
501 * Copy these registers here in case we end up stopped with
502 * someone (like, say, /proc) messing with our register state.
503 * We don't -restore- them unless we have to in update_sregs.
504 *
505 * Since userland -can't- change fsbase or gsbase directly,
506 * and capturing them involves two serializing instructions,
507 * we don't bother to capture them here.
508 */
509 xorl %ebx, %ebx
510 movw %ds, %bx
511 movq %rbx, REGOFF_DS(%rsp)
512 movw %es, %bx
513 movq %rbx, REGOFF_ES(%rsp)
514 movw %fs, %bx
515 movq %rbx, REGOFF_FS(%rsp)
516 movw %gs, %bx
517 movq %rbx, REGOFF_GS(%rsp)
518
519 /*
520 * If we're trying to use TRAPTRACE though, I take that back: we're
521 * probably debugging some problem in the SWAPGS logic and want to know
522 * what the incoming gsbase was.
523 *
524 * Since we already did SWAPGS, record the KGSBASE.
525 */
526 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
527 movl $MSR_AMD_KGSBASE, %ecx
528 rdmsr
529 movl %eax, REGOFF_GSBASE(%rsp)
530 movl %edx, REGOFF_GSBASE+4(%rsp)
531 #endif
532
533 /*
534 * Machine state saved in the regs structure on the stack
535 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
536 * %eax is the syscall number
537 * %rsp is the thread's stack, %r15 is curthread
538 * REG_RSP(%rsp) is the user's stack
539 */
540
541 SYSCALL_TRAPTRACE($TT_SYSC64)
542
543 movq %rsp, %rbp
544
545 movq T_LWP(%r15), %r14
546 ASSERT_NO_RUPDATE_PENDING(%r14)
547 ENABLE_INTR_FLAGS
548
549 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
550 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
551
552 ASSERT_LWPTOREGS(%r14, %rsp)
553
554 movb $LWP_SYS, LWP_STATE(%r14)
555 incq LWP_RU_SYSC(%r14)
556 movb $NORMALRETURN, LWP_EOSYS(%r14)
557
558 incq %gs:CPU_STATS_SYS_SYSCALL
559
560 movw %ax, T_SYSNUM(%r15)
561 movzbl T_PRE_SYS(%r15), %ebx
562 ORL_SYSCALLTRACE(%ebx)
563 testl %ebx, %ebx
564 jne _syscall_pre
565
566 _syscall_invoke:
567 movq REGOFF_RDI(%rbp), %rdi
568 movq REGOFF_RSI(%rbp), %rsi
569 movq REGOFF_RDX(%rbp), %rdx
570 movq REGOFF_RCX(%rbp), %rcx
571 movq REGOFF_R8(%rbp), %r8
572 movq REGOFF_R9(%rbp), %r9
573
574 cmpl $NSYSCALL, %eax
575 jae _syscall_ill
576 shll $SYSENT_SIZE_SHIFT, %eax
577 leaq sysent(%rax), %rbx
578
579 movq SY_CALLC(%rbx), %rax
580 INDIRECT_CALL_REG(rax)
581
582 movq %rax, %r12
583 movq %rdx, %r13
584
585 /*
586 * If the handler returns two ints, then we need to split the
587 * 64-bit return value into two 32-bit values.
588 */
589 testw $SE_32RVAL2, SY_FLAGS(%rbx)
590 je 5f
591 movq %r12, %r13
592 shrq $32, %r13 /* upper 32-bits into %edx */
593 movl %r12d, %r12d /* lower 32-bits into %eax */
594 5:
595 /*
596 * Optimistically assume that there's no post-syscall
597 * work to do. (This is to avoid having to call syscall_mstate()
598 * with interrupts disabled)
599 */
600 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
601
602 /*
603 * We must protect ourselves from being descheduled here;
604 * If we were, and we ended up on another cpu, or another
605 * lwp got in ahead of us, it could change the segment
606 * registers without us noticing before we return to userland.
607 */
608 CLI(%r14)
609 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
610 jne _syscall_post
611
612 /*
613 * We need to protect ourselves against non-canonical return values
614 * because Intel doesn't check for them on sysret (AMD does). Canonical
615 * addresses on current amd64 processors only use 48-bits for VAs; an
616 * address is canonical if all upper bits (47-63) are identical. If we
617 * find a non-canonical %rip, we opt to go through the full
618 * _syscall_post path which takes us into an iretq which is not
619 * susceptible to the same problems sysret is.
620 *
621 * We're checking for a canonical address by first doing an arithmetic
622 * shift. This will fill in the remaining bits with the value of bit 63.
623 * If the address were canonical, the register would now have either all
624 * zeroes or all ones in it. Therefore we add one (inducing overflow)
625 * and compare against 1. A canonical address will either be zero or one
626 * at this point, hence the use of ja.
627 *
628 * At this point, r12 and r13 have the return value so we can't use
629 * those registers.
630 */
631 movq REGOFF_RIP(%rsp), %rcx
632 sarq $47, %rcx
633 incq %rcx
634 cmpq $1, %rcx
635 ja _syscall_post
636
637
638 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
639
640 movq %r12, REGOFF_RAX(%rsp)
641 movq %r13, REGOFF_RDX(%rsp)
642
643 /*
644 * Clobber %r11 as we check CR0.TS.
645 */
646 ASSERT_CR0TS_ZERO(%r11)
647
648 /*
649 * Unlike other cases, because we need to restore the user stack pointer
650 * before exiting the kernel we must clear the microarch state before
651 * getting here. This should be safe because it means that the only
652 * values on the bus after this are based on the user's registers and
653 * potentially the addresses where we stored them. Given the constraints
654 * of sysret, that's how it has to be.
655 */
656 call x86_md_clear
657
658 /*
659 * To get back to userland, we need the return %rip in %rcx and
660 * the return %rfl in %r11d. The sysretq instruction also arranges
661 * to fix up %cs and %ss; everything else is our responsibility.
662 */
663 movq REGOFF_RDI(%rsp), %rdi
664 movq REGOFF_RSI(%rsp), %rsi
665 movq REGOFF_RDX(%rsp), %rdx
666 /* %rcx used to restore %rip value */
667
668 movq REGOFF_R8(%rsp), %r8
669 movq REGOFF_R9(%rsp), %r9
670 movq REGOFF_RAX(%rsp), %rax
671 movq REGOFF_RBX(%rsp), %rbx
672
673 movq REGOFF_RBP(%rsp), %rbp
674 movq REGOFF_R10(%rsp), %r10
675 /* %r11 used to restore %rfl value */
676 movq REGOFF_R12(%rsp), %r12
677
678 movq REGOFF_R13(%rsp), %r13
679 movq REGOFF_R14(%rsp), %r14
680 movq REGOFF_R15(%rsp), %r15
681
682 movq REGOFF_RIP(%rsp), %rcx
683 movl REGOFF_RFL(%rsp), %r11d
684
685 #if defined(__xpv)
686 addq $REGOFF_RIP, %rsp
687 #else
688 movq REGOFF_RSP(%rsp), %rsp
689 #endif
690
691 /*
692 * There can be no instructions between the ALTENTRY below and
693 * SYSRET or we could end up breaking brand support. See label usage
694 * in sn1_brand_syscall_callback for an example.
695 */
696 ASSERT_UPCALL_MASK_IS_SET
697 #if defined(__xpv)
698 SYSRETQ
699 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
700
701 /*
702 * We can only get here after executing a brand syscall
703 * interposition callback handler and simply need to
704 * "sysretq" back to userland. On the hypervisor this
705 * involves the iret hypercall which requires us to construct
706 * just enough of the stack needed for the hypercall.
707 * (rip, cs, rflags, rsp, ss).
708 */
709 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
710 movq %gs:CPU_THREAD, %r11
711 movq T_STACK(%r11), %rsp
712
713 movq %rcx, REGOFF_RIP(%rsp)
714 movl $UCS_SEL, REGOFF_CS(%rsp)
715 movq %gs:CPU_RTMP_RSP, %r11
716 movq %r11, REGOFF_RSP(%rsp)
717 pushfq
718 popq %r11 /* hypercall enables ints */
719 movq %r11, REGOFF_RFL(%rsp)
720 movl $UDS_SEL, REGOFF_SS(%rsp)
721 addq $REGOFF_RIP, %rsp
722 /*
723 * XXPV: see comment in SYSRETQ definition for future optimization
724 * we could take.
725 */
726 ASSERT_UPCALL_MASK_IS_SET
727 SYSRETQ
728 #else
729 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
730 jmp tr_sysretq
731 #endif
732 /*NOTREACHED*/
733 SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
734
735 _syscall_pre:
736 call pre_syscall
737 movl %eax, %r12d
738 testl %eax, %eax
739 jne _syscall_post_call
740 /*
741 * Didn't abort, so reload the syscall args and invoke the handler.
742 */
743 movzwl T_SYSNUM(%r15), %eax
744 jmp _syscall_invoke
745
746 _syscall_ill:
747 call nosys
748 movq %rax, %r12
749 movq %rdx, %r13
750 jmp _syscall_post_call
751
752 _syscall_post:
753 STI
754 /*
755 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
756 * so that we can account for the extra work it takes us to finish.
757 */
758 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
759 _syscall_post_call:
760 movq %r12, %rdi
761 movq %r13, %rsi
762 call post_syscall
763 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
764 jmp _sys_rtt
765 SET_SIZE(sys_syscall)
766 SET_SIZE(brand_sys_syscall)
767
768 #endif /* __lint */
769
770 #if defined(__lint)
771
772 /*ARGSUSED*/
773 void
774 sys_syscall32()
775 {}
776
777 #else /* __lint */
778
779 ENTRY_NP(brand_sys_syscall32)
780 SWAPGS /* kernel gsbase */
781 XPV_TRAP_POP
782 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
783 jmp nopop_sys_syscall32
784
785 ALTENTRY(sys_syscall32)
786 SWAPGS /* kernel gsbase */
787 XPV_TRAP_POP
788
789 nopop_sys_syscall32:
790 movl %esp, %r10d
791 movq %gs:CPU_THREAD, %r15
792 movq T_STACK(%r15), %rsp
793 movl %eax, %eax
794
795 movl $U32CS_SEL, REGOFF_CS(%rsp)
796 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
797 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
798 movq %r10, REGOFF_RSP(%rsp)
799 movl $UDS_SEL, REGOFF_SS(%rsp)
800
801 _syscall32_save:
802 movl %edi, REGOFF_RDI(%rsp)
803 movl %esi, REGOFF_RSI(%rsp)
804 movl %ebp, REGOFF_RBP(%rsp)
805 movl %ebx, REGOFF_RBX(%rsp)
806 movl %edx, REGOFF_RDX(%rsp)
807 movl %ecx, REGOFF_RCX(%rsp)
808 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
809 movq $0, REGOFF_SAVFP(%rsp)
810 movq $0, REGOFF_SAVPC(%rsp)
811
812 /*
813 * Copy these registers here in case we end up stopped with
814 * someone (like, say, /proc) messing with our register state.
815 * We don't -restore- them unless we have to in update_sregs.
816 *
817 * Since userland -can't- change fsbase or gsbase directly,
818 * we don't bother to capture them here.
819 */
820 xorl %ebx, %ebx
821 movw %ds, %bx
822 movq %rbx, REGOFF_DS(%rsp)
823 movw %es, %bx
824 movq %rbx, REGOFF_ES(%rsp)
825 movw %fs, %bx
826 movq %rbx, REGOFF_FS(%rsp)
827 movw %gs, %bx
828 movq %rbx, REGOFF_GS(%rsp)
829
830 /*
831 * If we're trying to use TRAPTRACE though, I take that back: we're
832 * probably debugging some problem in the SWAPGS logic and want to know
833 * what the incoming gsbase was.
834 *
835 * Since we already did SWAPGS, record the KGSBASE.
836 */
837 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
838 movl $MSR_AMD_KGSBASE, %ecx
839 rdmsr
840 movl %eax, REGOFF_GSBASE(%rsp)
841 movl %edx, REGOFF_GSBASE+4(%rsp)
842 #endif
843
844 /*
845 * Application state saved in the regs structure on the stack
846 * %eax is the syscall number
847 * %rsp is the thread's stack, %r15 is curthread
848 * REG_RSP(%rsp) is the user's stack
849 */
850
851 SYSCALL_TRAPTRACE32($TT_SYSC)
852
853 movq %rsp, %rbp
854
855 movq T_LWP(%r15), %r14
856 ASSERT_NO_RUPDATE_PENDING(%r14)
857
858 ENABLE_INTR_FLAGS
859
860 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
861 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
862
863 ASSERT_LWPTOREGS(%r14, %rsp)
864
865 incq %gs:CPU_STATS_SYS_SYSCALL
866
867 /*
868 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
869 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
870 * more succinctly:
871 *
872 * SA(MAXSYSARGS * sizeof (long)) == 64
873 */
874 #define SYS_DROP 64 /* drop for args */
875 subq $SYS_DROP, %rsp
876 movb $LWP_SYS, LWP_STATE(%r14)
877 movq %r15, %rdi
878 movq %rsp, %rsi
879 call syscall_entry
880
881 /*
882 * Fetch the arguments copied onto the kernel stack and put
883 * them in the right registers to invoke a C-style syscall handler.
884 * %rax contains the handler address.
885 *
886 * Ideas for making all this go faster of course include simply
887 * forcibly fetching 6 arguments from the user stack under lofault
888 * protection, reverting to copyin_args only when watchpoints
889 * are in effect.
890 *
891 * (If we do this, make sure that exec and libthread leave
892 * enough space at the top of the stack to ensure that we'll
893 * never do a fetch from an invalid page.)
894 *
895 * Lots of ideas here, but they won't really help with bringup B-)
896 * Correctness can't wait, performance can wait a little longer ..
897 */
898
899 movq %rax, %rbx
900 movl 0(%rsp), %edi
901 movl 8(%rsp), %esi
902 movl 0x10(%rsp), %edx
903 movl 0x18(%rsp), %ecx
904 movl 0x20(%rsp), %r8d
905 movl 0x28(%rsp), %r9d
906
907 movq SY_CALLC(%rbx), %rax
908 INDIRECT_CALL_REG(rax)
909
910 movq %rbp, %rsp /* pop the args */
911
912 /*
913 * amd64 syscall handlers -always- return a 64-bit value in %rax.
914 * On the 32-bit kernel, they always return that value in %eax:%edx
915 * as required by the 32-bit ABI.
916 *
917 * Simulate the same behaviour by unconditionally splitting the
918 * return value in the same way.
919 */
920 movq %rax, %r13
921 shrq $32, %r13 /* upper 32-bits into %edx */
922 movl %eax, %r12d /* lower 32-bits into %eax */
923
924 /*
925 * Optimistically assume that there's no post-syscall
926 * work to do. (This is to avoid having to call syscall_mstate()
927 * with interrupts disabled)
928 */
929 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
930
931 /*
932 * We must protect ourselves from being descheduled here;
933 * If we were, and we ended up on another cpu, or another
934 * lwp got in ahead of us, it could change the segment
935 * registers without us noticing before we return to userland.
936 */
937 CLI(%r14)
938 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
939 jne _full_syscall_postsys32
940 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
941
942 /*
943 * Clobber %r11 as we check CR0.TS.
944 */
945 ASSERT_CR0TS_ZERO(%r11)
946
947 /*
948 * Unlike other cases, because we need to restore the user stack pointer
949 * before exiting the kernel we must clear the microarch state before
950 * getting here. This should be safe because it means that the only
951 * values on the bus after this are based on the user's registers and
952 * potentially the addresses where we stored them. Given the constraints
953 * of sysret, that's how it has to be.
954 */
955 call x86_md_clear
956
957 /*
958 * To get back to userland, we need to put the return %rip in %rcx and
959 * the return %rfl in %r11d. The sysret instruction also arranges
960 * to fix up %cs and %ss; everything else is our responsibility.
961 */
962
963 movl %r12d, %eax /* %eax: rval1 */
964 movl REGOFF_RBX(%rsp), %ebx
965 /* %ecx used for return pointer */
966 movl %r13d, %edx /* %edx: rval2 */
967 movl REGOFF_RBP(%rsp), %ebp
968 movl REGOFF_RSI(%rsp), %esi
969 movl REGOFF_RDI(%rsp), %edi
970
971 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
972 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
973 movl REGOFF_RSP(%rsp), %esp
974
975 ASSERT_UPCALL_MASK_IS_SET
976 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
977 jmp tr_sysretl
978 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
979 /*NOTREACHED*/
980
981 _full_syscall_postsys32:
982 STI
983 /*
984 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
985 * so that we can account for the extra work it takes us to finish.
986 */
987 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
988 movq %r15, %rdi
989 movq %r12, %rsi /* rval1 - %eax */
990 movq %r13, %rdx /* rval2 - %edx */
991 call syscall_exit
992 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
993 jmp _sys_rtt
994 SET_SIZE(sys_syscall32)
995 SET_SIZE(brand_sys_syscall32)
996
997 #endif /* __lint */
998
999 /*
1000 * System call handler via the sysenter instruction
1001 * Used only for 32-bit system calls on the 64-bit kernel.
1002 *
1003 * The caller in userland has arranged that:
1004 *
1005 * - %eax contains the syscall number
1006 * - %ecx contains the user %esp
1007 * - %edx contains the return %eip
1008 * - the user stack contains the args to the syscall
1009 *
1010 * Hardware and (privileged) initialization code have arranged that by
1011 * the time the sysenter instructions completes:
1012 *
1013 * - %rip is pointing to sys_sysenter (below).
1014 * - %cs and %ss are set to kernel text and stack (data) selectors.
1015 * - %rsp is pointing at the lwp's stack
1016 * - interrupts have been disabled.
1017 *
1018 * Note that we are unable to return both "rvals" to userland with
1019 * this call, as %edx is used by the sysexit instruction.
1020 *
1021 * One final complication in this routine is its interaction with
1022 * single-stepping in a debugger. For most of the system call mechanisms, the
1023 * CPU automatically clears the single-step flag before we enter the kernel.
1024 * The sysenter mechanism does not clear the flag, so a user single-stepping
1025 * through a libc routine may suddenly find themself single-stepping through the
1026 * kernel. To detect this, kmdb and trap() both compare the trap %pc to the
1027 * [brand_]sys_enter addresses on each single-step trap. If it finds that we
1028 * have single-stepped to a sysenter entry point, it explicitly clears the flag
1029 * and executes the sys_sysenter routine.
1030 *
1031 * One final complication in this final complication is the fact that we have
1032 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
1033 * If we enter at brand_sys_sysenter and start single-stepping through the
1034 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
1035 * kmdb cannot distinguish between that valid single-step and the undesirable
1036 * one mentioned above. To avoid this situation, we simply add a jump over the
1037 * instruction at sys_sysenter to make it impossible to single-step to it.
1038 */
1039 #if defined(__lint)
1040
1041 void
1042 sys_sysenter()
1043 {}
1044
1045 #else /* __lint */
1046
1047 ENTRY_NP(brand_sys_sysenter)
1048 SWAPGS /* kernel gsbase */
1049 ALTENTRY(_brand_sys_sysenter_post_swapgs)
1050
1051 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
1052 /*
1053 * Jump over sys_sysenter to allow single-stepping as described
1054 * above.
1055 */
1056 jmp _sys_sysenter_post_swapgs
1057
1058 ALTENTRY(sys_sysenter)
1059 SWAPGS /* kernel gsbase */
1060 ALTENTRY(_sys_sysenter_post_swapgs)
1061
1062 movq %gs:CPU_THREAD, %r15
1063
1064 movl $U32CS_SEL, REGOFF_CS(%rsp)
1065 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
1066 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
1067 /*
1068 * NOTE: none of the instructions that run before we get here should
1069 * clobber bits in (R)FLAGS! This includes the kpti trampoline.
1070 */
1071 pushfq
1072 popq %r10
1073 movl $UDS_SEL, REGOFF_SS(%rsp)
1074
1075 /*
1076 * Set the interrupt flag before storing the flags to the
1077 * flags image on the stack so we can return to user with
1078 * interrupts enabled if we return via sys_rtt_syscall32
1079 */
1080 orq $PS_IE, %r10
1081 movq %r10, REGOFF_RFL(%rsp)
1082
1083 movl %edi, REGOFF_RDI(%rsp)
1084 movl %esi, REGOFF_RSI(%rsp)
1085 movl %ebp, REGOFF_RBP(%rsp)
1086 movl %ebx, REGOFF_RBX(%rsp)
1087 movl %edx, REGOFF_RDX(%rsp)
1088 movl %ecx, REGOFF_RCX(%rsp)
1089 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
1090 movq $0, REGOFF_SAVFP(%rsp)
1091 movq $0, REGOFF_SAVPC(%rsp)
1092
1093 /*
1094 * Copy these registers here in case we end up stopped with
1095 * someone (like, say, /proc) messing with our register state.
1096 * We don't -restore- them unless we have to in update_sregs.
1097 *
1098 * Since userland -can't- change fsbase or gsbase directly,
1099 * we don't bother to capture them here.
1100 */
1101 xorl %ebx, %ebx
1102 movw %ds, %bx
1103 movq %rbx, REGOFF_DS(%rsp)
1104 movw %es, %bx
1105 movq %rbx, REGOFF_ES(%rsp)
1106 movw %fs, %bx
1107 movq %rbx, REGOFF_FS(%rsp)
1108 movw %gs, %bx
1109 movq %rbx, REGOFF_GS(%rsp)
1110
1111 /*
1112 * If we're trying to use TRAPTRACE though, I take that back: we're
1113 * probably debugging some problem in the SWAPGS logic and want to know
1114 * what the incoming gsbase was.
1115 *
1116 * Since we already did SWAPGS, record the KGSBASE.
1117 */
1118 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
1119 movl $MSR_AMD_KGSBASE, %ecx
1120 rdmsr
1121 movl %eax, REGOFF_GSBASE(%rsp)
1122 movl %edx, REGOFF_GSBASE+4(%rsp)
1123 #endif
1124
1125 /*
1126 * Application state saved in the regs structure on the stack
1127 * %eax is the syscall number
1128 * %rsp is the thread's stack, %r15 is curthread
1129 * REG_RSP(%rsp) is the user's stack
1130 */
1131
1132 SYSCALL_TRAPTRACE($TT_SYSENTER)
1133
1134 movq %rsp, %rbp
1135
1136 movq T_LWP(%r15), %r14
1137 ASSERT_NO_RUPDATE_PENDING(%r14)
1138
1139 ENABLE_INTR_FLAGS
1140
1141 /*
1142 * Catch 64-bit process trying to issue sysenter instruction
1143 * on Nocona based systems.
1144 */
1145 movq LWP_PROCP(%r14), %rax
1146 cmpq $DATAMODEL_ILP32, P_MODEL(%rax)
1147 je 7f
1148
1149 /*
1150 * For a non-32-bit process, simulate a #ud, since that's what
1151 * native hardware does. The traptrace entry (above) will
1152 * let you know what really happened.
1153 */
1154 movq $T_ILLINST, REGOFF_TRAPNO(%rsp)
1155 movq REGOFF_CS(%rsp), %rdi
1156 movq %rdi, REGOFF_ERR(%rsp)
1157 movq %rsp, %rdi
1158 movq REGOFF_RIP(%rsp), %rsi
1159 movl %gs:CPU_ID, %edx
1160 call trap
1161 jmp _sys_rtt
1162 7:
1163
1164 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1165 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */
1166
1167 ASSERT_LWPTOREGS(%r14, %rsp)
1168
1169 incq %gs:CPU_STATS_SYS_SYSCALL
1170
1171 /*
1172 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1173 * placed into 64-bit (long) arg slots, plus one 64-bit
1174 * (long) arg count, maintaining 16 byte alignment.
1175 */
1176 subq $SYS_DROP, %rsp
1177 movb $LWP_SYS, LWP_STATE(%r14)
1178 movq %r15, %rdi
1179 movq %rsp, %rsi
1180 call syscall_entry
1181
1182 /*
1183 * Fetch the arguments copied onto the kernel stack and put
1184 * them in the right registers to invoke a C-style syscall handler.
1185 * %rax contains the handler address.
1186 */
1187 movq %rax, %rbx
1188 movl 0(%rsp), %edi
1189 movl 8(%rsp), %esi
1190 movl 0x10(%rsp), %edx
1191 movl 0x18(%rsp), %ecx
1192 movl 0x20(%rsp), %r8d
1193 movl 0x28(%rsp), %r9d
1194
1195 movq SY_CALLC(%rbx), %rax
1196 INDIRECT_CALL_REG(rax)
1197
1198 movq %rbp, %rsp /* pop the args */
1199
1200 /*
1201 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1202 * On the 32-bit kernel, the always return that value in %eax:%edx
1203 * as required by the 32-bit ABI.
1204 *
1205 * Simulate the same behaviour by unconditionally splitting the
1206 * return value in the same way.
1207 */
1208 movq %rax, %r13
1209 shrq $32, %r13 /* upper 32-bits into %edx */
1210 movl %eax, %r12d /* lower 32-bits into %eax */
1211
1212 /*
1213 * Optimistically assume that there's no post-syscall
1214 * work to do. (This is to avoid having to call syscall_mstate()
1215 * with interrupts disabled)
1216 */
1217 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1218
1219 /*
1220 * We must protect ourselves from being descheduled here;
1221 * If we were, and we ended up on another cpu, or another
1222 * lwp got int ahead of us, it could change the segment
1223 * registers without us noticing before we return to userland.
1224 *
1225 * This cli is undone in the tr_sysexit trampoline code.
1226 */
1227 cli
1228 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1229 jne _full_syscall_postsys32
1230 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1231
1232 /*
1233 * To get back to userland, load up the 32-bit registers and
1234 * sysexit back where we came from.
1235 */
1236
1237 /*
1238 * Interrupts will be turned on by the 'sti' executed just before
1239 * sysexit. The following ensures that restoring the user's rflags
1240 * doesn't enable interrupts too soon.
1241 */
1242 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1243
1244 /*
1245 * Clobber %r11 as we check CR0.TS.
1246 */
1247 ASSERT_CR0TS_ZERO(%r11)
1248
1249 /*
1250 * (There's no point in loading up %edx because the sysexit
1251 * mechanism smashes it.)
1252 */
1253 movl %r12d, %eax
1254 movl REGOFF_RBX(%rsp), %ebx
1255 movl REGOFF_RBP(%rsp), %ebp
1256 movl REGOFF_RSI(%rsp), %esi
1257 movl REGOFF_RDI(%rsp), %edi
1258
1259 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */
1260 pushq REGOFF_RFL(%rsp)
1261 popfq
1262 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */
1263 ALTENTRY(sys_sysenter_swapgs_sysexit)
1264 call x86_md_clear
1265 jmp tr_sysexit
1266 SET_SIZE(sys_sysenter_swapgs_sysexit)
1267 SET_SIZE(sys_sysenter)
1268 SET_SIZE(_sys_sysenter_post_swapgs)
1269 SET_SIZE(brand_sys_sysenter)
1270
1271 #endif /* __lint */
1272
1273 /*
1274 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1275 * the generic i386 libc to do system calls. We do a small amount of setup
1276 * before jumping into the existing sys_syscall32 path.
1277 */
1278 #if defined(__lint)
1279
1280 /*ARGSUSED*/
1281 void
1282 sys_syscall_int()
1283 {}
1284
1285 #else /* __lint */
1286
1287 ENTRY_NP(brand_sys_syscall_int)
1288 SWAPGS /* kernel gsbase */
1289 XPV_TRAP_POP
1290 call smap_enable
1291 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1292 jmp nopop_syscall_int
1293
1294 ALTENTRY(sys_syscall_int)
1295 SWAPGS /* kernel gsbase */
1296 XPV_TRAP_POP
1297 call smap_enable
1298
1299 nopop_syscall_int:
1300 movq %gs:CPU_THREAD, %r15
1301 movq T_STACK(%r15), %rsp
1302 movl %eax, %eax
1303 /*
1304 * Set t_post_sys on this thread to force ourselves out via the slow
1305 * path. It might be possible at some later date to optimize this out
1306 * and use a faster return mechanism.
1307 */
1308 movb $1, T_POST_SYS(%r15)
1309 CLEAN_CS
1310 jmp _syscall32_save
1311 /*
1312 * There should be no instructions between this label and SWAPGS/IRET
1313 * or we could end up breaking branded zone support. See the usage of
1314 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1315 * for examples.
1316 *
1317 * We want to swapgs to maintain the invariant that all entries into
1318 * tr_iret_user are done on the user gsbase.
1319 */
1320 ALTENTRY(sys_sysint_swapgs_iret)
1321 call x86_md_clear
1322 SWAPGS
1323 jmp tr_iret_user
1324 /*NOTREACHED*/
1325 SET_SIZE(sys_sysint_swapgs_iret)
1326 SET_SIZE(sys_syscall_int)
1327 SET_SIZE(brand_sys_syscall_int)
1328
1329 #endif /* __lint */
1330
1331 /*
1332 * Legacy 32-bit applications and old libc implementations do lcalls;
1333 * we should never get here because the LDT entry containing the syscall
1334 * segment descriptor has the "segment present" bit cleared, which means
1335 * we end up processing those system calls in trap() via a not-present trap.
1336 *
1337 * We do it this way because a call gate unhelpfully does -nothing- to the
1338 * interrupt flag bit, so an interrupt can run us just after the lcall
1339 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and
1340 * INTR_POP paths would have to be slightly more complex to dance around
1341 * this problem, and end up depending explicitly on the first
1342 * instruction of this handler being either swapgs or cli.
1343 */
1344
1345 #if defined(__lint)
1346
1347 /*ARGSUSED*/
1348 void
1349 sys_lcall32()
1350 {}
1351
1352 #else /* __lint */
1353
1354 ENTRY_NP(sys_lcall32)
1355 SWAPGS /* kernel gsbase */
1356 pushq $0
1357 pushq %rbp
1358 movq %rsp, %rbp
1359 leaq __lcall_panic_str(%rip), %rdi
1360 xorl %eax, %eax
1361 call panic
1362 SET_SIZE(sys_lcall32)
1363
1364 __lcall_panic_str:
1365 .string "sys_lcall32: shouldn't be here!"
1366
1367 /*
1368 * Declare a uintptr_t which covers the entire pc range of syscall
1369 * handlers for the stack walkers that need this.
1370 */
1371 .align CPTRSIZE
1372 .globl _allsyscalls_size
1373 .type _allsyscalls_size, @object
1374 _allsyscalls_size:
1375 .NWORD . - _allsyscalls
1376 SET_SIZE(_allsyscalls_size)
1377
1378 #endif /* __lint */
1379
1380 /*
1381 * These are the thread context handlers for lwps using sysenter/sysexit.
1382 */
1383
1384 #if defined(__lint)
1385
1386 /*ARGSUSED*/
1387 void
1388 sep_save(void *ksp)
1389 {}
1390
1391 /*ARGSUSED*/
1392 void
1393 sep_restore(void *ksp)
1394 {}
1395
1396 #else /* __lint */
1397
1398 /*
1399 * setting this value to zero as we switch away causes the
1400 * stack-pointer-on-sysenter to be NULL, ensuring that we
1401 * don't silently corrupt another (preempted) thread stack
1402 * when running an lwp that (somehow) didn't get sep_restore'd
1403 */
1404 ENTRY_NP(sep_save)
1405 xorl %edx, %edx
1406 xorl %eax, %eax
1407 movl $MSR_INTC_SEP_ESP, %ecx
1408 wrmsr
1409 ret
1410 SET_SIZE(sep_save)
1411
1412 /*
1413 * Update the kernel stack pointer as we resume onto this cpu.
1414 */
1415 ENTRY_NP(sep_restore)
1416 movq %rdi, %rdx
1417 shrq $32, %rdx
1418 movl %edi, %eax
1419 movl $MSR_INTC_SEP_ESP, %ecx
1420 wrmsr
1421 ret
1422 SET_SIZE(sep_restore)
1423
1424 #endif /* __lint */