1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2019 Joyent, Inc.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 #include <sys/asm_linkage.h>
28 #include <sys/asm_misc.h>
29 #include <sys/regset.h>
30 #include <sys/privregs.h>
31 #include <sys/psw.h>
32 #include <sys/machbrand.h>
33
34 #if defined(__lint)
35
36 #include <sys/types.h>
37 #include <sys/thread.h>
38 #include <sys/systm.h>
39
40 #else /* __lint */
41
42 #include <sys/segments.h>
43 #include <sys/pcb.h>
44 #include <sys/trap.h>
45 #include <sys/ftrace.h>
46 #include <sys/traptrace.h>
47 #include <sys/clock.h>
48 #include <sys/model.h>
49 #include <sys/panic.h>
50
51 #if defined(__xpv)
52 #include <sys/hypervisor.h>
53 #endif
54
55 #include "assym.h"
56
57 #endif /* __lint */
58
59 /*
60 * We implement five flavours of system call entry points
61 *
62 * - syscall/sysretq (amd64 generic)
63 * - syscall/sysretl (i386 plus SYSC bit)
64 * - sysenter/sysexit (i386 plus SEP bit)
65 * - int/iret (i386 generic)
66 * - lcall/iret (i386 generic)
67 *
68 * The current libc included in Solaris uses int/iret as the base unoptimized
69 * kernel entry method. Older libc implementations and legacy binaries may use
70 * the lcall call gate, so it must continue to be supported.
71 *
72 * System calls that use an lcall call gate are processed in trap() via a
73 * segment-not-present trap, i.e. lcalls are extremely slow(!).
74 *
75 * The basic pattern used in the 32-bit SYSC handler at this point in time is
76 * to have the bare minimum of assembler, and get to the C handlers as
77 * quickly as possible.
78 *
79 * The 64-bit handler is much closer to the sparcv9 handler; that's
80 * because of passing arguments in registers. The 32-bit world still
81 * passes arguments on the stack -- that makes that handler substantially
82 * more complex.
83 *
84 * The two handlers share a few code fragments which are broken
85 * out into preprocessor macros below.
86 *
87 * XX64 come back and speed all this up later. The 32-bit stuff looks
88 * especially easy to speed up the argument copying part ..
89 *
90 *
91 * Notes about segment register usage (c.f. the 32-bit kernel)
92 *
93 * In the 32-bit kernel, segment registers are dutifully saved and
94 * restored on all mode transitions because the kernel uses them directly.
95 * When the processor is running in 64-bit mode, segment registers are
96 * largely ignored.
97 *
98 * %cs and %ss
99 * controlled by the hardware mechanisms that make mode transitions
100 *
101 * The remaining segment registers have to either be pointing at a valid
102 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
103 *
104 * %ds and %es
105 * always ignored
106 *
107 * %fs and %gs
108 * fsbase and gsbase are used to control the place they really point at.
109 * The kernel only depends on %gs, and controls its own gsbase via swapgs
110 *
111 * Note that loading segment registers is still costly because the GDT
112 * lookup still happens (this is because the hardware can't know that we're
113 * not setting up these segment registers for a 32-bit program). Thus we
114 * avoid doing this in the syscall path, and defer them to lwp context switch
115 * handlers, so the register values remain virtualized to the lwp.
116 */
117
118 #if defined(SYSCALLTRACE)
119 #define ORL_SYSCALLTRACE(r32) \
120 orl syscalltrace(%rip), r32
121 #else
122 #define ORL_SYSCALLTRACE(r32)
123 #endif
124
125 /*
126 * In the 32-bit kernel, we do absolutely nothing before getting into the
127 * brand callback checks. In 64-bit land, we do swapgs and then come here.
128 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
129 * are still unused.
130 *
131 * Check if a brand_mach_ops callback is defined for the specified callback_id
132 * type. If so invoke it with the kernel's %gs value loaded and the following
133 * data on the stack:
134 *
135 * stack: --------------------------------------
136 * 32 | callback pointer |
137 * | 24 | user (or interrupt) stack pointer |
138 * | 16 | lwp pointer |
139 * v 8 | userland return address |
140 * 0 | callback wrapper return addr |
141 * --------------------------------------
142 *
143 * Since we're pushing the userland return address onto the kernel stack
144 * we need to get that address without accessing the user's stack (since we
145 * can't trust that data). There are different ways to get the userland
146 * return address depending on how the syscall trap was made:
147 *
148 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
149 * b) For sys_sysenter the return address is in %rdx.
150 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
151 * the stack pointer points at the state saved when we took the interrupt:
152 * ------------------------
153 * | | user's %ss |
154 * | | user's %esp |
155 * | | EFLAGS register |
156 * v | user's %cs |
157 * | user's %eip |
158 * ------------------------
159 *
160 * The 2nd parameter to the BRAND_CALLBACK macro is either the
161 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
162 * used to generate the proper code to get the userland return address for
163 * each syscall entry point.
164 *
165 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
166 * is available as a scratch register within the callback. If the callback
167 * returns within the kernel then this macro will restore %r15. If the
168 * callback is going to return directly to userland then it should restore
169 * %r15 before returning to userland.
170 */
171 #define BRAND_URET_FROM_REG(rip_reg) \
172 pushq rip_reg /* push the return address */
173
174 /*
175 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
176 * is currently pointing at the user return address (%eip).
177 */
178 #define BRAND_URET_FROM_INTR_STACK() \
179 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
180 pushq (%r15) /* push the return address */
181
182 #define BRAND_CALLBACK(callback_id, push_userland_ret) \
183 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
184 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
185 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
186 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
187 subq $16, %rsp /* save space for 2 pointers */ ;\
188 pushq %r14 /* save %r14 */ ;\
189 movq %gs:CPU_RTMP_RSP, %r14 ;\
190 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
191 popq %r14 /* restore %r14 */ ;\
192 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
193 pushq %r15 /* push the lwp pointer */ ;\
194 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
195 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
196 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
197 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
198 cmpq $0, %r15 ;\
199 je 1f ;\
200 movq %r15, 16(%rsp) /* save the callback pointer */ ;\
201 push_userland_ret /* push the return address */ ;\
202 call *24(%rsp) /* call callback */ ;\
203 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
204 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
205
206 #define MSTATE_TRANSITION(from, to) \
207 movl $from, %edi; \
208 movl $to, %esi; \
209 call syscall_mstate
210
211 /*
212 * Check to see if a simple (direct) return is possible i.e.
213 *
214 * if (t->t_post_sys_ast | syscalltrace |
215 * lwp->lwp_pcb.pcb_rupdate == 1)
216 * do full version ;
217 *
218 * Preconditions:
219 * - t is curthread
220 * Postconditions:
221 * - condition code NE is set if post-sys is too complex
222 * - rtmp is zeroed if it isn't (we rely on this!)
223 * - ltmp is smashed
224 */
225 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
226 movq T_LWP(t), ltmp; \
227 movzbl PCB_RUPDATE(ltmp), rtmp; \
228 ORL_SYSCALLTRACE(rtmp); \
229 orl T_POST_SYS_AST(t), rtmp; \
230 cmpl $0, rtmp
231
232 /*
233 * Fix up the lwp, thread, and eflags for a successful return
234 *
235 * Preconditions:
236 * - zwreg contains zero
237 */
238 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
239 movb $LWP_USER, LWP_STATE(lwp); \
240 movw zwreg, T_SYSNUM(t); \
241 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
242
243 /*
244 * ASSERT(lwptoregs(lwp) == rp);
245 *
246 * This may seem obvious, but very odd things happen if this
247 * assertion is false
248 *
249 * Preconditions:
250 * (%rsp is ready for normal call sequence)
251 * Postconditions (if assertion is true):
252 * %r11 is smashed
253 *
254 * ASSERT(rp->r_cs == descnum)
255 *
256 * The code selector is written into the regs structure when the
257 * lwp stack is created. We use this ASSERT to validate that
258 * the regs structure really matches how we came in.
259 *
260 * Preconditions:
261 * (%rsp is ready for normal call sequence)
262 * Postconditions (if assertion is true):
263 * -none-
264 *
265 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
266 *
267 * If this is false, it meant that we returned to userland without
268 * updating the segment registers as we were supposed to.
269 *
270 * Note that we must ensure no interrupts or other traps intervene
271 * between entering privileged mode and performing the assertion,
272 * otherwise we may perform a context switch on the thread, which
273 * will end up setting pcb_rupdate to 1 again.
274 *
275 * ASSERT(%cr0 & CR0_TS == 0);
276 * Preconditions:
277 * (%rsp is ready for normal call sequence)
278 * Postconditions (if assertion is true):
279 * (specified register is clobbered)
280 *
281 * Check to make sure that we are returning to user land and that CR0.TS
282 * is not set. This is required as part of the eager FPU (see
283 * uts/intel/ia32/os/fpu.c for more information).
284 */
285
286 #if defined(DEBUG)
287
288 #if !defined(__lint)
289
290 __lwptoregs_msg:
291 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
292
293 __codesel_msg:
294 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
295
296 __no_rupdate_msg:
297 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
298
299 __bad_ts_msg:
300 .string "sysscall_asm_amd64.s:%d CR0.TS set on user return"
301
302 #endif /* !__lint */
303
304 #define ASSERT_LWPTOREGS(lwp, rp) \
305 movq LWP_REGS(lwp), %r11; \
306 cmpq rp, %r11; \
307 je 7f; \
308 leaq __lwptoregs_msg(%rip), %rdi; \
309 movl $__LINE__, %esi; \
310 movq lwp, %rdx; \
311 movq %r11, %rcx; \
312 movq rp, %r8; \
313 xorl %eax, %eax; \
314 call panic; \
315 7:
316
317 #define ASSERT_NO_RUPDATE_PENDING(lwp) \
318 testb $0x1, PCB_RUPDATE(lwp); \
319 je 8f; \
320 movq lwp, %rdx; \
321 leaq __no_rupdate_msg(%rip), %rdi; \
322 movl $__LINE__, %esi; \
323 xorl %eax, %eax; \
324 call panic; \
325 8:
326
327 #define ASSERT_CR0TS_ZERO(reg) \
328 movq %cr0, reg; \
329 testq $CR0_TS, reg; \
330 jz 9f; \
331 leaq __bad_ts_msg(%rip), %rdi; \
332 movl $__LINE__, %esi; \
333 xorl %eax, %eax; \
334 call panic; \
335 9:
336
337 #else
338 #define ASSERT_LWPTOREGS(lwp, rp)
339 #define ASSERT_NO_RUPDATE_PENDING(lwp)
340 #define ASSERT_CR0TS_ZERO(reg)
341 #endif
342
343 /*
344 * Do the traptrace thing and restore any registers we used
345 * in situ. Assumes that %rsp is pointing at the base of
346 * the struct regs, obviously ..
347 */
348 #ifdef TRAPTRACE
349 #define SYSCALL_TRAPTRACE(ttype) \
350 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
351 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
352 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
353 movq REGOFF_RAX(%rsp), %rax; \
354 movq REGOFF_RBX(%rsp), %rbx; \
355 movq REGOFF_RCX(%rsp), %rcx; \
356 movq REGOFF_RDX(%rsp), %rdx; \
357 movl %eax, TTR_SYSNUM(%rdi); \
358 movq REGOFF_RDI(%rsp), %rdi
359
360 #define SYSCALL_TRAPTRACE32(ttype) \
361 SYSCALL_TRAPTRACE(ttype); \
362 /* paranoia: clean the top 32-bits of the registers */ \
363 orl %eax, %eax; \
364 orl %ebx, %ebx; \
365 orl %ecx, %ecx; \
366 orl %edx, %edx; \
367 orl %edi, %edi
368 #else /* TRAPTRACE */
369 #define SYSCALL_TRAPTRACE(ttype)
370 #define SYSCALL_TRAPTRACE32(ttype)
371 #endif /* TRAPTRACE */
372
373 /*
374 * The 64-bit libc syscall wrapper does this:
375 *
376 * fn(<args>)
377 * {
378 * movq %rcx, %r10 -- because syscall smashes %rcx
379 * movl $CODE, %eax
380 * syscall
381 * <error processing>
382 * }
383 *
384 * Thus when we come into the kernel:
385 *
386 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
387 * %rax is the syscall number
388 * %r12-%r15 contain caller state
389 *
390 * The syscall instruction arranges that:
391 *
392 * %rcx contains the return %rip
393 * %r11d contains bottom 32-bits of %rflags
394 * %rflags is masked (as determined by the SFMASK msr)
395 * %cs is set to UCS_SEL (as determined by the STAR msr)
396 * %ss is set to UDS_SEL (as determined by the STAR msr)
397 * %rip is set to sys_syscall (as determined by the LSTAR msr)
398 *
399 * Or in other words, we have no registers available at all.
400 * Only swapgs can save us!
401 *
402 * Under the hypervisor, the swapgs has happened already. However, the
403 * state of the world is very different from that we're familiar with.
404 *
405 * In particular, we have a stack structure like that for interrupt
406 * gates, except that the %cs and %ss registers are modified for reasons
407 * that are not entirely clear. Critically, the %rcx/%r11 values do
408 * *not* reflect the usage of those registers under a 'real' syscall[1];
409 * the stack, therefore, looks like this:
410 *
411 * 0x0(rsp) potentially junk %rcx
412 * 0x8(rsp) potentially junk %r11
413 * 0x10(rsp) user %rip
414 * 0x18(rsp) modified %cs
415 * 0x20(rsp) user %rflags
416 * 0x28(rsp) user %rsp
417 * 0x30(rsp) modified %ss
418 *
419 *
420 * and before continuing on, we must load the %rip into %rcx and the
421 * %rflags into %r11.
422 *
423 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
424 * Sigh.
425 */
426 #if defined(__xpv)
427 #define XPV_SYSCALL_PROD \
428 movq 0x10(%rsp), %rcx; \
429 movq 0x20(%rsp), %r11; \
430 movq 0x28(%rsp), %rsp
431 #else
432 #define XPV_SYSCALL_PROD /* nothing */
433 #endif
434
435 #if defined(__lint)
436
437 /*ARGSUSED*/
438 void
439 sys_syscall()
440 {}
441
442 void
443 _allsyscalls()
444 {}
445
446 size_t _allsyscalls_size;
447
448 #else /* __lint */
449
450 ENTRY_NP2(brand_sys_syscall,_allsyscalls)
451 SWAPGS /* kernel gsbase */
452 XPV_SYSCALL_PROD
453 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
454 jmp noprod_sys_syscall
455
456 ALTENTRY(sys_syscall)
457 SWAPGS /* kernel gsbase */
458 XPV_SYSCALL_PROD
459
460 noprod_sys_syscall:
461 movq %r15, %gs:CPU_RTMP_R15
462 movq %rsp, %gs:CPU_RTMP_RSP
463
464 movq %gs:CPU_THREAD, %r15
465 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
466
467 ASSERT_UPCALL_MASK_IS_SET
468
469 movl $UCS_SEL, REGOFF_CS(%rsp)
470 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
471 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
472 movl $UDS_SEL, REGOFF_SS(%rsp)
473
474 movl %eax, %eax /* wrapper: sysc# -> %eax */
475 movq %rdi, REGOFF_RDI(%rsp)
476 movq %rsi, REGOFF_RSI(%rsp)
477 movq %rdx, REGOFF_RDX(%rsp)
478 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
479 movq %r10, %rcx /* arg[3] for direct calls */
480
481 movq %r8, REGOFF_R8(%rsp)
482 movq %r9, REGOFF_R9(%rsp)
483 movq %rax, REGOFF_RAX(%rsp)
484 movq %rbx, REGOFF_RBX(%rsp)
485
486 movq %rbp, REGOFF_RBP(%rsp)
487 movq %r10, REGOFF_R10(%rsp)
488 movq %gs:CPU_RTMP_RSP, %r11
489 movq %r11, REGOFF_RSP(%rsp)
490 movq %r12, REGOFF_R12(%rsp)
491
492 movq %r13, REGOFF_R13(%rsp)
493 movq %r14, REGOFF_R14(%rsp)
494 movq %gs:CPU_RTMP_R15, %r10
495 movq %r10, REGOFF_R15(%rsp)
496 movq $0, REGOFF_SAVFP(%rsp)
497 movq $0, REGOFF_SAVPC(%rsp)
498
499 /*
500 * Copy these registers here in case we end up stopped with
501 * someone (like, say, /proc) messing with our register state.
502 * We don't -restore- them unless we have to in update_sregs.
503 *
504 * Since userland -can't- change fsbase or gsbase directly,
505 * and capturing them involves two serializing instructions,
506 * we don't bother to capture them here.
507 */
508 xorl %ebx, %ebx
509 movw %ds, %bx
510 movq %rbx, REGOFF_DS(%rsp)
511 movw %es, %bx
512 movq %rbx, REGOFF_ES(%rsp)
513 movw %fs, %bx
514 movq %rbx, REGOFF_FS(%rsp)
515 movw %gs, %bx
516 movq %rbx, REGOFF_GS(%rsp)
517
518 /*
519 * If we're trying to use TRAPTRACE though, I take that back: we're
520 * probably debugging some problem in the SWAPGS logic and want to know
521 * what the incoming gsbase was.
522 *
523 * Since we already did SWAPGS, record the KGSBASE.
524 */
525 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
526 movl $MSR_AMD_KGSBASE, %ecx
527 rdmsr
528 movl %eax, REGOFF_GSBASE(%rsp)
529 movl %edx, REGOFF_GSBASE+4(%rsp)
530 #endif
531
532 /*
533 * Machine state saved in the regs structure on the stack
534 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
535 * %eax is the syscall number
536 * %rsp is the thread's stack, %r15 is curthread
537 * REG_RSP(%rsp) is the user's stack
538 */
539
540 SYSCALL_TRAPTRACE($TT_SYSC64)
541
542 movq %rsp, %rbp
543
544 movq T_LWP(%r15), %r14
545 ASSERT_NO_RUPDATE_PENDING(%r14)
546 ENABLE_INTR_FLAGS
547
548 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
549 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
550
551 ASSERT_LWPTOREGS(%r14, %rsp)
552
553 movb $LWP_SYS, LWP_STATE(%r14)
554 incq LWP_RU_SYSC(%r14)
555 movb $NORMALRETURN, LWP_EOSYS(%r14)
556
557 incq %gs:CPU_STATS_SYS_SYSCALL
558
559 movw %ax, T_SYSNUM(%r15)
560 movzbl T_PRE_SYS(%r15), %ebx
561 ORL_SYSCALLTRACE(%ebx)
562 testl %ebx, %ebx
563 jne _syscall_pre
564
565 _syscall_invoke:
566 movq REGOFF_RDI(%rbp), %rdi
567 movq REGOFF_RSI(%rbp), %rsi
568 movq REGOFF_RDX(%rbp), %rdx
569 movq REGOFF_RCX(%rbp), %rcx
570 movq REGOFF_R8(%rbp), %r8
571 movq REGOFF_R9(%rbp), %r9
572
573 cmpl $NSYSCALL, %eax
574 jae _syscall_ill
575 shll $SYSENT_SIZE_SHIFT, %eax
576 leaq sysent(%rax), %rbx
577
578 call *SY_CALLC(%rbx)
579
580 movq %rax, %r12
581 movq %rdx, %r13
582
583 /*
584 * If the handler returns two ints, then we need to split the
585 * 64-bit return value into two 32-bit values.
586 */
587 testw $SE_32RVAL2, SY_FLAGS(%rbx)
588 je 5f
589 movq %r12, %r13
590 shrq $32, %r13 /* upper 32-bits into %edx */
591 movl %r12d, %r12d /* lower 32-bits into %eax */
592 5:
593 /*
594 * Optimistically assume that there's no post-syscall
595 * work to do. (This is to avoid having to call syscall_mstate()
596 * with interrupts disabled)
597 */
598 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
599
600 /*
601 * We must protect ourselves from being descheduled here;
602 * If we were, and we ended up on another cpu, or another
603 * lwp got in ahead of us, it could change the segment
604 * registers without us noticing before we return to userland.
605 */
606 CLI(%r14)
607 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
608 jne _syscall_post
609
610 /*
611 * We need to protect ourselves against non-canonical return values
612 * because Intel doesn't check for them on sysret (AMD does). Canonical
613 * addresses on current amd64 processors only use 48-bits for VAs; an
614 * address is canonical if all upper bits (47-63) are identical. If we
615 * find a non-canonical %rip, we opt to go through the full
616 * _syscall_post path which takes us into an iretq which is not
617 * susceptible to the same problems sysret is.
618 *
619 * We're checking for a canonical address by first doing an arithmetic
620 * shift. This will fill in the remaining bits with the value of bit 63.
621 * If the address were canonical, the register would now have either all
622 * zeroes or all ones in it. Therefore we add one (inducing overflow)
623 * and compare against 1. A canonical address will either be zero or one
624 * at this point, hence the use of ja.
625 *
626 * At this point, r12 and r13 have the return value so we can't use
627 * those registers.
628 */
629 movq REGOFF_RIP(%rsp), %rcx
630 sarq $47, %rcx
631 incq %rcx
632 cmpq $1, %rcx
633 ja _syscall_post
634
635
636 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
637
638 movq %r12, REGOFF_RAX(%rsp)
639 movq %r13, REGOFF_RDX(%rsp)
640
641 /*
642 * Clobber %r11 as we check CR0.TS.
643 */
644 ASSERT_CR0TS_ZERO(%r11)
645
646 /*
647 * Unlike other cases, because we need to restore the user stack pointer
648 * before exiting the kernel we must clear the microarch state before
649 * getting here. This should be safe because it means that the only
650 * values on the bus after this are based on the user's registers and
651 * potentially the addresses where we stored them. Given the constraints
652 * of sysret, that's how it has to be.
653 */
654 call *x86_md_clear
655
656 /*
657 * To get back to userland, we need the return %rip in %rcx and
658 * the return %rfl in %r11d. The sysretq instruction also arranges
659 * to fix up %cs and %ss; everything else is our responsibility.
660 */
661 movq REGOFF_RDI(%rsp), %rdi
662 movq REGOFF_RSI(%rsp), %rsi
663 movq REGOFF_RDX(%rsp), %rdx
664 /* %rcx used to restore %rip value */
665
666 movq REGOFF_R8(%rsp), %r8
667 movq REGOFF_R9(%rsp), %r9
668 movq REGOFF_RAX(%rsp), %rax
669 movq REGOFF_RBX(%rsp), %rbx
670
671 movq REGOFF_RBP(%rsp), %rbp
672 movq REGOFF_R10(%rsp), %r10
673 /* %r11 used to restore %rfl value */
674 movq REGOFF_R12(%rsp), %r12
675
676 movq REGOFF_R13(%rsp), %r13
677 movq REGOFF_R14(%rsp), %r14
678 movq REGOFF_R15(%rsp), %r15
679
680 movq REGOFF_RIP(%rsp), %rcx
681 movl REGOFF_RFL(%rsp), %r11d
682
683 #if defined(__xpv)
684 addq $REGOFF_RIP, %rsp
685 #else
686 movq REGOFF_RSP(%rsp), %rsp
687 #endif
688
689 /*
690 * There can be no instructions between the ALTENTRY below and
691 * SYSRET or we could end up breaking brand support. See label usage
692 * in sn1_brand_syscall_callback for an example.
693 */
694 ASSERT_UPCALL_MASK_IS_SET
695 #if defined(__xpv)
696 SYSRETQ
697 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
698
699 /*
700 * We can only get here after executing a brand syscall
701 * interposition callback handler and simply need to
702 * "sysretq" back to userland. On the hypervisor this
703 * involves the iret hypercall which requires us to construct
704 * just enough of the stack needed for the hypercall.
705 * (rip, cs, rflags, rsp, ss).
706 */
707 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
708 movq %gs:CPU_THREAD, %r11
709 movq T_STACK(%r11), %rsp
710
711 movq %rcx, REGOFF_RIP(%rsp)
712 movl $UCS_SEL, REGOFF_CS(%rsp)
713 movq %gs:CPU_RTMP_RSP, %r11
714 movq %r11, REGOFF_RSP(%rsp)
715 pushfq
716 popq %r11 /* hypercall enables ints */
717 movq %r11, REGOFF_RFL(%rsp)
718 movl $UDS_SEL, REGOFF_SS(%rsp)
719 addq $REGOFF_RIP, %rsp
720 /*
721 * XXPV: see comment in SYSRETQ definition for future optimization
722 * we could take.
723 */
724 ASSERT_UPCALL_MASK_IS_SET
725 SYSRETQ
726 #else
727 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
728 jmp tr_sysretq
729 #endif
730 /*NOTREACHED*/
731 SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
732
733 _syscall_pre:
734 call pre_syscall
735 movl %eax, %r12d
736 testl %eax, %eax
737 jne _syscall_post_call
738 /*
739 * Didn't abort, so reload the syscall args and invoke the handler.
740 */
741 movzwl T_SYSNUM(%r15), %eax
742 jmp _syscall_invoke
743
744 _syscall_ill:
745 call nosys
746 movq %rax, %r12
747 movq %rdx, %r13
748 jmp _syscall_post_call
749
750 _syscall_post:
751 STI
752 /*
753 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
754 * so that we can account for the extra work it takes us to finish.
755 */
756 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
757 _syscall_post_call:
758 movq %r12, %rdi
759 movq %r13, %rsi
760 call post_syscall
761 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
762 jmp _sys_rtt
763 SET_SIZE(sys_syscall)
764 SET_SIZE(brand_sys_syscall)
765
766 #endif /* __lint */
767
768 #if defined(__lint)
769
770 /*ARGSUSED*/
771 void
772 sys_syscall32()
773 {}
774
775 #else /* __lint */
776
777 ENTRY_NP(brand_sys_syscall32)
778 SWAPGS /* kernel gsbase */
779 XPV_TRAP_POP
780 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
781 jmp nopop_sys_syscall32
782
783 ALTENTRY(sys_syscall32)
784 SWAPGS /* kernel gsbase */
785 XPV_TRAP_POP
786
787 nopop_sys_syscall32:
788 movl %esp, %r10d
789 movq %gs:CPU_THREAD, %r15
790 movq T_STACK(%r15), %rsp
791 movl %eax, %eax
792
793 movl $U32CS_SEL, REGOFF_CS(%rsp)
794 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
795 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
796 movq %r10, REGOFF_RSP(%rsp)
797 movl $UDS_SEL, REGOFF_SS(%rsp)
798
799 _syscall32_save:
800 movl %edi, REGOFF_RDI(%rsp)
801 movl %esi, REGOFF_RSI(%rsp)
802 movl %ebp, REGOFF_RBP(%rsp)
803 movl %ebx, REGOFF_RBX(%rsp)
804 movl %edx, REGOFF_RDX(%rsp)
805 movl %ecx, REGOFF_RCX(%rsp)
806 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
807 movq $0, REGOFF_SAVFP(%rsp)
808 movq $0, REGOFF_SAVPC(%rsp)
809
810 /*
811 * Copy these registers here in case we end up stopped with
812 * someone (like, say, /proc) messing with our register state.
813 * We don't -restore- them unless we have to in update_sregs.
814 *
815 * Since userland -can't- change fsbase or gsbase directly,
816 * we don't bother to capture them here.
817 */
818 xorl %ebx, %ebx
819 movw %ds, %bx
820 movq %rbx, REGOFF_DS(%rsp)
821 movw %es, %bx
822 movq %rbx, REGOFF_ES(%rsp)
823 movw %fs, %bx
824 movq %rbx, REGOFF_FS(%rsp)
825 movw %gs, %bx
826 movq %rbx, REGOFF_GS(%rsp)
827
828 /*
829 * If we're trying to use TRAPTRACE though, I take that back: we're
830 * probably debugging some problem in the SWAPGS logic and want to know
831 * what the incoming gsbase was.
832 *
833 * Since we already did SWAPGS, record the KGSBASE.
834 */
835 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
836 movl $MSR_AMD_KGSBASE, %ecx
837 rdmsr
838 movl %eax, REGOFF_GSBASE(%rsp)
839 movl %edx, REGOFF_GSBASE+4(%rsp)
840 #endif
841
842 /*
843 * Application state saved in the regs structure on the stack
844 * %eax is the syscall number
845 * %rsp is the thread's stack, %r15 is curthread
846 * REG_RSP(%rsp) is the user's stack
847 */
848
849 SYSCALL_TRAPTRACE32($TT_SYSC)
850
851 movq %rsp, %rbp
852
853 movq T_LWP(%r15), %r14
854 ASSERT_NO_RUPDATE_PENDING(%r14)
855
856 ENABLE_INTR_FLAGS
857
858 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
859 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
860
861 ASSERT_LWPTOREGS(%r14, %rsp)
862
863 incq %gs:CPU_STATS_SYS_SYSCALL
864
865 /*
866 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
867 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
868 * more succinctly:
869 *
870 * SA(MAXSYSARGS * sizeof (long)) == 64
871 */
872 #define SYS_DROP 64 /* drop for args */
873 subq $SYS_DROP, %rsp
874 movb $LWP_SYS, LWP_STATE(%r14)
875 movq %r15, %rdi
876 movq %rsp, %rsi
877 call syscall_entry
878
879 /*
880 * Fetch the arguments copied onto the kernel stack and put
881 * them in the right registers to invoke a C-style syscall handler.
882 * %rax contains the handler address.
883 *
884 * Ideas for making all this go faster of course include simply
885 * forcibly fetching 6 arguments from the user stack under lofault
886 * protection, reverting to copyin_args only when watchpoints
887 * are in effect.
888 *
889 * (If we do this, make sure that exec and libthread leave
890 * enough space at the top of the stack to ensure that we'll
891 * never do a fetch from an invalid page.)
892 *
893 * Lots of ideas here, but they won't really help with bringup B-)
894 * Correctness can't wait, performance can wait a little longer ..
895 */
896
897 movq %rax, %rbx
898 movl 0(%rsp), %edi
899 movl 8(%rsp), %esi
900 movl 0x10(%rsp), %edx
901 movl 0x18(%rsp), %ecx
902 movl 0x20(%rsp), %r8d
903 movl 0x28(%rsp), %r9d
904
905 call *SY_CALLC(%rbx)
906
907 movq %rbp, %rsp /* pop the args */
908
909 /*
910 * amd64 syscall handlers -always- return a 64-bit value in %rax.
911 * On the 32-bit kernel, they always return that value in %eax:%edx
912 * as required by the 32-bit ABI.
913 *
914 * Simulate the same behaviour by unconditionally splitting the
915 * return value in the same way.
916 */
917 movq %rax, %r13
918 shrq $32, %r13 /* upper 32-bits into %edx */
919 movl %eax, %r12d /* lower 32-bits into %eax */
920
921 /*
922 * Optimistically assume that there's no post-syscall
923 * work to do. (This is to avoid having to call syscall_mstate()
924 * with interrupts disabled)
925 */
926 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
927
928 /*
929 * We must protect ourselves from being descheduled here;
930 * If we were, and we ended up on another cpu, or another
931 * lwp got in ahead of us, it could change the segment
932 * registers without us noticing before we return to userland.
933 */
934 CLI(%r14)
935 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
936 jne _full_syscall_postsys32
937 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
938
939 /*
940 * Clobber %r11 as we check CR0.TS.
941 */
942 ASSERT_CR0TS_ZERO(%r11)
943
944 /*
945 * Unlike other cases, because we need to restore the user stack pointer
946 * before exiting the kernel we must clear the microarch state before
947 * getting here. This should be safe because it means that the only
948 * values on the bus after this are based on the user's registers and
949 * potentially the addresses where we stored them. Given the constraints
950 * of sysret, that's how it has to be.
951 */
952 call *x86_md_clear
953
954 /*
955 * To get back to userland, we need to put the return %rip in %rcx and
956 * the return %rfl in %r11d. The sysret instruction also arranges
957 * to fix up %cs and %ss; everything else is our responsibility.
958 */
959
960 movl %r12d, %eax /* %eax: rval1 */
961 movl REGOFF_RBX(%rsp), %ebx
962 /* %ecx used for return pointer */
963 movl %r13d, %edx /* %edx: rval2 */
964 movl REGOFF_RBP(%rsp), %ebp
965 movl REGOFF_RSI(%rsp), %esi
966 movl REGOFF_RDI(%rsp), %edi
967
968 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
969 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
970 movl REGOFF_RSP(%rsp), %esp
971
972 ASSERT_UPCALL_MASK_IS_SET
973 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
974 jmp tr_sysretl
975 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
976 /*NOTREACHED*/
977
978 _full_syscall_postsys32:
979 STI
980 /*
981 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
982 * so that we can account for the extra work it takes us to finish.
983 */
984 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
985 movq %r15, %rdi
986 movq %r12, %rsi /* rval1 - %eax */
987 movq %r13, %rdx /* rval2 - %edx */
988 call syscall_exit
989 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
990 jmp _sys_rtt
991 SET_SIZE(sys_syscall32)
992 SET_SIZE(brand_sys_syscall32)
993
994 #endif /* __lint */
995
996 /*
997 * System call handler via the sysenter instruction
998 * Used only for 32-bit system calls on the 64-bit kernel.
999 *
1000 * The caller in userland has arranged that:
1001 *
1002 * - %eax contains the syscall number
1003 * - %ecx contains the user %esp
1004 * - %edx contains the return %eip
1005 * - the user stack contains the args to the syscall
1006 *
1007 * Hardware and (privileged) initialization code have arranged that by
1008 * the time the sysenter instructions completes:
1009 *
1010 * - %rip is pointing to sys_sysenter (below).
1011 * - %cs and %ss are set to kernel text and stack (data) selectors.
1012 * - %rsp is pointing at the lwp's stack
1013 * - interrupts have been disabled.
1014 *
1015 * Note that we are unable to return both "rvals" to userland with
1016 * this call, as %edx is used by the sysexit instruction.
1017 *
1018 * One final complication in this routine is its interaction with
1019 * single-stepping in a debugger. For most of the system call mechanisms, the
1020 * CPU automatically clears the single-step flag before we enter the kernel.
1021 * The sysenter mechanism does not clear the flag, so a user single-stepping
1022 * through a libc routine may suddenly find themself single-stepping through the
1023 * kernel. To detect this, kmdb and trap() both compare the trap %pc to the
1024 * [brand_]sys_enter addresses on each single-step trap. If it finds that we
1025 * have single-stepped to a sysenter entry point, it explicitly clears the flag
1026 * and executes the sys_sysenter routine.
1027 *
1028 * One final complication in this final complication is the fact that we have
1029 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
1030 * If we enter at brand_sys_sysenter and start single-stepping through the
1031 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
1032 * kmdb cannot distinguish between that valid single-step and the undesirable
1033 * one mentioned above. To avoid this situation, we simply add a jump over the
1034 * instruction at sys_sysenter to make it impossible to single-step to it.
1035 */
1036 #if defined(__lint)
1037
1038 void
1039 sys_sysenter()
1040 {}
1041
1042 #else /* __lint */
1043
1044 ENTRY_NP(brand_sys_sysenter)
1045 SWAPGS /* kernel gsbase */
1046 ALTENTRY(_brand_sys_sysenter_post_swapgs)
1047
1048 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
1049 /*
1050 * Jump over sys_sysenter to allow single-stepping as described
1051 * above.
1052 */
1053 jmp _sys_sysenter_post_swapgs
1054
1055 ALTENTRY(sys_sysenter)
1056 SWAPGS /* kernel gsbase */
1057 ALTENTRY(_sys_sysenter_post_swapgs)
1058
1059 movq %gs:CPU_THREAD, %r15
1060
1061 movl $U32CS_SEL, REGOFF_CS(%rsp)
1062 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
1063 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
1064 /*
1065 * NOTE: none of the instructions that run before we get here should
1066 * clobber bits in (R)FLAGS! This includes the kpti trampoline.
1067 */
1068 pushfq
1069 popq %r10
1070 movl $UDS_SEL, REGOFF_SS(%rsp)
1071
1072 /*
1073 * Set the interrupt flag before storing the flags to the
1074 * flags image on the stack so we can return to user with
1075 * interrupts enabled if we return via sys_rtt_syscall32
1076 */
1077 orq $PS_IE, %r10
1078 movq %r10, REGOFF_RFL(%rsp)
1079
1080 movl %edi, REGOFF_RDI(%rsp)
1081 movl %esi, REGOFF_RSI(%rsp)
1082 movl %ebp, REGOFF_RBP(%rsp)
1083 movl %ebx, REGOFF_RBX(%rsp)
1084 movl %edx, REGOFF_RDX(%rsp)
1085 movl %ecx, REGOFF_RCX(%rsp)
1086 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
1087 movq $0, REGOFF_SAVFP(%rsp)
1088 movq $0, REGOFF_SAVPC(%rsp)
1089
1090 /*
1091 * Copy these registers here in case we end up stopped with
1092 * someone (like, say, /proc) messing with our register state.
1093 * We don't -restore- them unless we have to in update_sregs.
1094 *
1095 * Since userland -can't- change fsbase or gsbase directly,
1096 * we don't bother to capture them here.
1097 */
1098 xorl %ebx, %ebx
1099 movw %ds, %bx
1100 movq %rbx, REGOFF_DS(%rsp)
1101 movw %es, %bx
1102 movq %rbx, REGOFF_ES(%rsp)
1103 movw %fs, %bx
1104 movq %rbx, REGOFF_FS(%rsp)
1105 movw %gs, %bx
1106 movq %rbx, REGOFF_GS(%rsp)
1107
1108 /*
1109 * If we're trying to use TRAPTRACE though, I take that back: we're
1110 * probably debugging some problem in the SWAPGS logic and want to know
1111 * what the incoming gsbase was.
1112 *
1113 * Since we already did SWAPGS, record the KGSBASE.
1114 */
1115 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
1116 movl $MSR_AMD_KGSBASE, %ecx
1117 rdmsr
1118 movl %eax, REGOFF_GSBASE(%rsp)
1119 movl %edx, REGOFF_GSBASE+4(%rsp)
1120 #endif
1121
1122 /*
1123 * Application state saved in the regs structure on the stack
1124 * %eax is the syscall number
1125 * %rsp is the thread's stack, %r15 is curthread
1126 * REG_RSP(%rsp) is the user's stack
1127 */
1128
1129 SYSCALL_TRAPTRACE($TT_SYSENTER)
1130
1131 movq %rsp, %rbp
1132
1133 movq T_LWP(%r15), %r14
1134 ASSERT_NO_RUPDATE_PENDING(%r14)
1135
1136 ENABLE_INTR_FLAGS
1137
1138 /*
1139 * Catch 64-bit process trying to issue sysenter instruction
1140 * on Nocona based systems.
1141 */
1142 movq LWP_PROCP(%r14), %rax
1143 cmpq $DATAMODEL_ILP32, P_MODEL(%rax)
1144 je 7f
1145
1146 /*
1147 * For a non-32-bit process, simulate a #ud, since that's what
1148 * native hardware does. The traptrace entry (above) will
1149 * let you know what really happened.
1150 */
1151 movq $T_ILLINST, REGOFF_TRAPNO(%rsp)
1152 movq REGOFF_CS(%rsp), %rdi
1153 movq %rdi, REGOFF_ERR(%rsp)
1154 movq %rsp, %rdi
1155 movq REGOFF_RIP(%rsp), %rsi
1156 movl %gs:CPU_ID, %edx
1157 call trap
1158 jmp _sys_rtt
1159 7:
1160
1161 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1162 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */
1163
1164 ASSERT_LWPTOREGS(%r14, %rsp)
1165
1166 incq %gs:CPU_STATS_SYS_SYSCALL
1167
1168 /*
1169 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1170 * placed into 64-bit (long) arg slots, plus one 64-bit
1171 * (long) arg count, maintaining 16 byte alignment.
1172 */
1173 subq $SYS_DROP, %rsp
1174 movb $LWP_SYS, LWP_STATE(%r14)
1175 movq %r15, %rdi
1176 movq %rsp, %rsi
1177 call syscall_entry
1178
1179 /*
1180 * Fetch the arguments copied onto the kernel stack and put
1181 * them in the right registers to invoke a C-style syscall handler.
1182 * %rax contains the handler address.
1183 */
1184 movq %rax, %rbx
1185 movl 0(%rsp), %edi
1186 movl 8(%rsp), %esi
1187 movl 0x10(%rsp), %edx
1188 movl 0x18(%rsp), %ecx
1189 movl 0x20(%rsp), %r8d
1190 movl 0x28(%rsp), %r9d
1191
1192 call *SY_CALLC(%rbx)
1193
1194 movq %rbp, %rsp /* pop the args */
1195
1196 /*
1197 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1198 * On the 32-bit kernel, the always return that value in %eax:%edx
1199 * as required by the 32-bit ABI.
1200 *
1201 * Simulate the same behaviour by unconditionally splitting the
1202 * return value in the same way.
1203 */
1204 movq %rax, %r13
1205 shrq $32, %r13 /* upper 32-bits into %edx */
1206 movl %eax, %r12d /* lower 32-bits into %eax */
1207
1208 /*
1209 * Optimistically assume that there's no post-syscall
1210 * work to do. (This is to avoid having to call syscall_mstate()
1211 * with interrupts disabled)
1212 */
1213 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1214
1215 /*
1216 * We must protect ourselves from being descheduled here;
1217 * If we were, and we ended up on another cpu, or another
1218 * lwp got int ahead of us, it could change the segment
1219 * registers without us noticing before we return to userland.
1220 *
1221 * This cli is undone in the tr_sysexit trampoline code.
1222 */
1223 cli
1224 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1225 jne _full_syscall_postsys32
1226 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1227
1228 /*
1229 * To get back to userland, load up the 32-bit registers and
1230 * sysexit back where we came from.
1231 */
1232
1233 /*
1234 * Interrupts will be turned on by the 'sti' executed just before
1235 * sysexit. The following ensures that restoring the user's rflags
1236 * doesn't enable interrupts too soon.
1237 */
1238 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1239
1240 /*
1241 * Clobber %r11 as we check CR0.TS.
1242 */
1243 ASSERT_CR0TS_ZERO(%r11)
1244
1245 /*
1246 * (There's no point in loading up %edx because the sysexit
1247 * mechanism smashes it.)
1248 */
1249 movl %r12d, %eax
1250 movl REGOFF_RBX(%rsp), %ebx
1251 movl REGOFF_RBP(%rsp), %ebp
1252 movl REGOFF_RSI(%rsp), %esi
1253 movl REGOFF_RDI(%rsp), %edi
1254
1255 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */
1256 pushq REGOFF_RFL(%rsp)
1257 popfq
1258 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */
1259 ALTENTRY(sys_sysenter_swapgs_sysexit)
1260 call *x86_md_clear
1261 jmp tr_sysexit
1262 SET_SIZE(sys_sysenter_swapgs_sysexit)
1263 SET_SIZE(sys_sysenter)
1264 SET_SIZE(_sys_sysenter_post_swapgs)
1265 SET_SIZE(brand_sys_sysenter)
1266
1267 #endif /* __lint */
1268
1269 /*
1270 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1271 * the generic i386 libc to do system calls. We do a small amount of setup
1272 * before jumping into the existing sys_syscall32 path.
1273 */
1274 #if defined(__lint)
1275
1276 /*ARGSUSED*/
1277 void
1278 sys_syscall_int()
1279 {}
1280
1281 #else /* __lint */
1282
1283 ENTRY_NP(brand_sys_syscall_int)
1284 SWAPGS /* kernel gsbase */
1285 XPV_TRAP_POP
1286 call smap_enable
1287 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1288 jmp nopop_syscall_int
1289
1290 ALTENTRY(sys_syscall_int)
1291 SWAPGS /* kernel gsbase */
1292 XPV_TRAP_POP
1293 call smap_enable
1294
1295 nopop_syscall_int:
1296 movq %gs:CPU_THREAD, %r15
1297 movq T_STACK(%r15), %rsp
1298 movl %eax, %eax
1299 /*
1300 * Set t_post_sys on this thread to force ourselves out via the slow
1301 * path. It might be possible at some later date to optimize this out
1302 * and use a faster return mechanism.
1303 */
1304 movb $1, T_POST_SYS(%r15)
1305 CLEAN_CS
1306 jmp _syscall32_save
1307 /*
1308 * There should be no instructions between this label and SWAPGS/IRET
1309 * or we could end up breaking branded zone support. See the usage of
1310 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1311 * for examples.
1312 *
1313 * We want to swapgs to maintain the invariant that all entries into
1314 * tr_iret_user are done on the user gsbase.
1315 */
1316 ALTENTRY(sys_sysint_swapgs_iret)
1317 call *x86_md_clear
1318 SWAPGS
1319 jmp tr_iret_user
1320 /*NOTREACHED*/
1321 SET_SIZE(sys_sysint_swapgs_iret)
1322 SET_SIZE(sys_syscall_int)
1323 SET_SIZE(brand_sys_syscall_int)
1324
1325 #endif /* __lint */
1326
1327 /*
1328 * Legacy 32-bit applications and old libc implementations do lcalls;
1329 * we should never get here because the LDT entry containing the syscall
1330 * segment descriptor has the "segment present" bit cleared, which means
1331 * we end up processing those system calls in trap() via a not-present trap.
1332 *
1333 * We do it this way because a call gate unhelpfully does -nothing- to the
1334 * interrupt flag bit, so an interrupt can run us just after the lcall
1335 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and
1336 * INTR_POP paths would have to be slightly more complex to dance around
1337 * this problem, and end up depending explicitly on the first
1338 * instruction of this handler being either swapgs or cli.
1339 */
1340
1341 #if defined(__lint)
1342
1343 /*ARGSUSED*/
1344 void
1345 sys_lcall32()
1346 {}
1347
1348 #else /* __lint */
1349
1350 ENTRY_NP(sys_lcall32)
1351 SWAPGS /* kernel gsbase */
1352 pushq $0
1353 pushq %rbp
1354 movq %rsp, %rbp
1355 leaq __lcall_panic_str(%rip), %rdi
1356 xorl %eax, %eax
1357 call panic
1358 SET_SIZE(sys_lcall32)
1359
1360 __lcall_panic_str:
1361 .string "sys_lcall32: shouldn't be here!"
1362
1363 /*
1364 * Declare a uintptr_t which covers the entire pc range of syscall
1365 * handlers for the stack walkers that need this.
1366 */
1367 .align CPTRSIZE
1368 .globl _allsyscalls_size
1369 .type _allsyscalls_size, @object
1370 _allsyscalls_size:
1371 .NWORD . - _allsyscalls
1372 SET_SIZE(_allsyscalls_size)
1373
1374 #endif /* __lint */
1375
1376 /*
1377 * These are the thread context handlers for lwps using sysenter/sysexit.
1378 */
1379
1380 #if defined(__lint)
1381
1382 /*ARGSUSED*/
1383 void
1384 sep_save(void *ksp)
1385 {}
1386
1387 /*ARGSUSED*/
1388 void
1389 sep_restore(void *ksp)
1390 {}
1391
1392 #else /* __lint */
1393
1394 /*
1395 * setting this value to zero as we switch away causes the
1396 * stack-pointer-on-sysenter to be NULL, ensuring that we
1397 * don't silently corrupt another (preempted) thread stack
1398 * when running an lwp that (somehow) didn't get sep_restore'd
1399 */
1400 ENTRY_NP(sep_save)
1401 xorl %edx, %edx
1402 xorl %eax, %eax
1403 movl $MSR_INTC_SEP_ESP, %ecx
1404 wrmsr
1405 ret
1406 SET_SIZE(sep_save)
1407
1408 /*
1409 * Update the kernel stack pointer as we resume onto this cpu.
1410 */
1411 ENTRY_NP(sep_restore)
1412 movq %rdi, %rdx
1413 shrq $32, %rdx
1414 movl %edi, %eax
1415 movl $MSR_INTC_SEP_ESP, %ecx
1416 wrmsr
1417 ret
1418 SET_SIZE(sep_restore)
1419
1420 #endif /* __lint */