3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2016 by Delphix. All rights reserved.
24 */
25
26 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
28 /* All Rights Reserved */
29
30 /* Copyright (c) 1987, 1988 Microsoft Corporation */
31 /* All Rights Reserved */
32
33 #include <sys/asm_linkage.h>
34 #include <sys/asm_misc.h>
35 #include <sys/regset.h>
36 #include <sys/psw.h>
37 #include <sys/x86_archext.h>
38 #include <sys/machbrand.h>
39 #include <sys/privregs.h>
40
41 #if defined(__lint)
42
43 #include <sys/types.h>
44 #include <sys/thread.h>
45 #include <sys/systm.h>
46
47 #else /* __lint */
48
49 #include <sys/segments.h>
50 #include <sys/pcb.h>
51 #include <sys/trap.h>
52 #include <sys/ftrace.h>
53 #include <sys/traptrace.h>
54 #include <sys/clock.h>
55 #include <sys/panic.h>
56 #include "assym.h"
57
58 #endif /* __lint */
59
60 /*
61 * We implement two flavours of system call entry points
62 *
63 * - {int,lcall}/iret (i386)
64 * - sysenter/sysexit (Pentium II and beyond)
65 *
66 * The basic pattern used in the handlers is to check to see if we can
67 * do fast (simple) version of the system call; if we can't we use various
68 * C routines that handle corner cases and debugging.
69 *
70 * To reduce the amount of assembler replication, yet keep the system call
71 * implementations vaguely comprehensible, the common code in the body
72 * of the handlers is broken up into a set of preprocessor definitions
73 * below.
74 */
75
76 /*
77 * When we have SYSCALLTRACE defined, we sneak an extra
78 * predicate into a couple of tests.
79 */
80 #if defined(SYSCALLTRACE)
81 #define ORL_SYSCALLTRACE(r32) \
82 orl syscalltrace, r32
83 #else
84 #define ORL_SYSCALLTRACE(r32)
85 #endif
86
87 /*
88 * This check is false whenever we want to go fast i.e.
89 *
90 * if (code >= NSYSCALL ||
91 * t->t_pre_sys || (t->t_proc_flag & TP_WATCHPT) != 0)
92 * do full version
93 * #ifdef SYSCALLTRACE
94 * if (syscalltrace)
95 * do full version
96 * #endif
97 *
98 * Preconditions:
99 * - t curthread
100 * - code contains the syscall number
101 * Postconditions:
102 * - %ecx and %edi are smashed
103 * - condition code flag ZF is cleared if pre-sys is too complex
104 */
105 #define CHECK_PRESYS_NE(t, code) \
106 movzbl T_PRE_SYS(t), %edi; \
107 movzwl T_PROC_FLAG(t), %ecx; \
108 andl $TP_WATCHPT, %ecx; \
109 orl %ecx, %edi; \
110 cmpl $NSYSCALL, code; \
111 setae %cl; \
112 movzbl %cl, %ecx; \
113 orl %ecx, %edi; \
114 ORL_SYSCALLTRACE(%edi)
115
116 /*
117 * Check if a brand_mach_ops callback is defined for the specified callback_id
118 * type. If so invoke it with the user's %gs value loaded and the following
119 * data on the stack:
120 * --------------------------------------
121 * | user's %ss |
122 * | | user's %esp |
123 * | | EFLAGS register |
124 * | | user's %cs |
125 * | | user's %eip (user return address) |
126 * | | 'scratch space' |
127 * | | user's %ebx |
128 * | | user's %gs selector |
129 * v | lwp pointer |
130 * | callback wrapper return addr |
131 * --------------------------------------
132 *
133 * If the brand code returns, we assume that we are meant to execute the
134 * normal system call path.
135 *
136 * The interface to the brand callbacks on the 32-bit kernel assumes %ebx
137 * is available as a scratch register within the callback. If the callback
138 * returns within the kernel then this macro will restore %ebx. If the
139 * callback is going to return directly to userland then it should restore
140 * %ebx before returning to userland.
141 */
142 #define BRAND_CALLBACK(callback_id) \
143 subl $4, %esp /* save some scratch space */ ;\
144 pushl %ebx /* save %ebx to use for scratch */ ;\
145 pushl %gs /* save the user %gs */ ;\
146 movl $KGS_SEL, %ebx ;\
147 movw %bx, %gs /* switch to the kernel's %gs */ ;\
148 movl %gs:CPU_THREAD, %ebx /* load the thread pointer */ ;\
149 movl T_LWP(%ebx), %ebx /* load the lwp pointer */ ;\
150 pushl %ebx /* push the lwp pointer */ ;\
151 movl LWP_PROCP(%ebx), %ebx /* load the proc pointer */ ;\
152 movl P_BRAND(%ebx), %ebx /* load the brand pointer */ ;\
153 movl B_MACHOPS(%ebx), %ebx /* load the machops pointer */ ;\
154 movl _CONST(_MUL(callback_id, CPTRSIZE))(%ebx), %ebx ;\
155 cmpl $0, %ebx ;\
156 je 1f ;\
157 movl %ebx, 12(%esp) /* save callback to scratch */ ;\
158 movl 4(%esp), %ebx /* grab the user %gs */ ;\
159 movw %bx, %gs /* restore the user %gs */ ;\
160 call *12(%esp) /* call callback in scratch */ ;\
161 1: movl 4(%esp), %ebx /* restore user %gs (re-do if */ ;\
162 movw %bx, %gs /* branch due to no callback) */ ;\
163 movl 8(%esp), %ebx /* restore user's %ebx */ ;\
164 addl $16, %esp /* restore stack ptr */
165
166 #define MSTATE_TRANSITION(from, to) \
167 pushl $to; \
168 pushl $from; \
169 call syscall_mstate; \
170 addl $0x8, %esp
171
172 /*
173 * aka CPU_STATS_ADDQ(CPU, sys.syscall, 1)
174 * This must be called with interrupts or preemption disabled.
175 */
176 #define CPU_STATS_SYS_SYSCALL_INC \
177 addl $1, %gs:CPU_STATS_SYS_SYSCALL; \
178 adcl $0, %gs:CPU_STATS_SYS_SYSCALL+4;
179
180 #if !defined(__lint)
181
182 /*
183 * ASSERT(lwptoregs(lwp) == rp);
184 *
185 * this may seem obvious, but very odd things happen if this
186 * assertion is false
187 *
188 * Preconditions:
189 * -none-
190 * Postconditions (if assertion is true):
191 * %esi and %edi are smashed
192 */
193 #if defined(DEBUG)
194
195 __lwptoregs_msg:
196 .string "syscall_asm.s:%d lwptoregs(%p) [%p] != rp [%p]"
197
198 #define ASSERT_LWPTOREGS(t, rp) \
199 movl T_LWP(t), %esi; \
200 movl LWP_REGS(%esi), %edi; \
201 cmpl rp, %edi; \
202 je 7f; \
203 pushl rp; \
204 pushl %edi; \
205 pushl %esi; \
206 pushl $__LINE__; \
207 pushl $__lwptoregs_msg; \
208 call panic; \
209 7:
210 #else
211 #define ASSERT_LWPTOREGS(t, rp)
212 #endif
213
214 #endif /* __lint */
215
216 /*
217 * This is an assembler version of this fragment:
218 *
219 * lwp->lwp_state = LWP_SYS;
220 * lwp->lwp_ru.sysc++;
221 * lwp->lwp_eosys = NORMALRETURN;
222 * lwp->lwp_ap = argp;
223 *
224 * Preconditions:
225 * -none-
226 * Postconditions:
227 * -none-
228 */
229 #define SET_LWP(lwp, argp) \
230 movb $LWP_SYS, LWP_STATE(lwp); \
231 addl $1, LWP_RU_SYSC(lwp); \
232 adcl $0, LWP_RU_SYSC+4(lwp); \
233 movb $NORMALRETURN, LWP_EOSYS(lwp); \
234 movl argp, LWP_AP(lwp)
235
236 /*
237 * Set up the thread, lwp, find the handler, and copy
238 * in the arguments from userland to the kernel stack.
239 *
240 * Preconditions:
241 * - %eax contains the syscall number
242 * Postconditions:
243 * - %eax contains a pointer to the sysent structure
244 * - %ecx is zeroed
245 * - %esi, %edi are smashed
246 * - %esp is SYS_DROPped ready for the syscall
247 */
248 #define SIMPLE_SYSCALL_PRESYS(t, faultlabel) \
249 movl T_LWP(t), %esi; \
250 movw %ax, T_SYSNUM(t); \
251 subl $SYS_DROP, %esp; \
252 shll $SYSENT_SIZE_SHIFT, %eax; \
253 SET_LWP(%esi, %esp); \
254 leal sysent(%eax), %eax; \
255 movzbl SY_NARG(%eax), %ecx; \
256 testl %ecx, %ecx; \
257 jz 4f; \
258 movl %esp, %edi; \
259 movl SYS_DROP + REGOFF_UESP(%esp), %esi; \
260 movl $faultlabel, T_LOFAULT(t); \
261 addl $4, %esi; \
262 rep; \
263 smovl; \
264 movl %ecx, T_LOFAULT(t); \
265 4:
266
267 /*
268 * Check to see if a simple return is possible i.e.
269 *
270 * if ((t->t_post_sys_ast | syscalltrace) != 0)
271 * do full version;
272 *
273 * Preconditions:
274 * - t is curthread
275 * Postconditions:
276 * - condition code NE is set if post-sys is too complex
277 * - rtmp is zeroed if it isn't (we rely on this!)
278 */
279 #define CHECK_POSTSYS_NE(t, rtmp) \
280 xorl rtmp, rtmp; \
281 ORL_SYSCALLTRACE(rtmp); \
282 orl T_POST_SYS_AST(t), rtmp; \
283 cmpl $0, rtmp
284
285 /*
286 * Fix up the lwp, thread, and eflags for a successful return
287 *
288 * Preconditions:
289 * - zwreg contains zero
290 * Postconditions:
291 * - %esp has been unSYS_DROPped
292 * - %esi is smashed (points to lwp)
293 */
294 #define SIMPLE_SYSCALL_POSTSYS(t, zwreg) \
295 movl T_LWP(t), %esi; \
296 addl $SYS_DROP, %esp; \
297 movw zwreg, T_SYSNUM(t); \
298 movb $LWP_USER, LWP_STATE(%esi); \
299 andb $_CONST(0xffff - PS_C), REGOFF_EFL(%esp)
300
301 /*
302 * System call handler. This is the destination of both the call
303 * gate (lcall 0x27) _and_ the interrupt gate (int 0x91). For our purposes,
304 * there are two significant differences between an interrupt gate and a call
305 * gate:
306 *
307 * 1) An interrupt gate runs the handler with interrupts disabled, whereas a
308 * call gate runs the handler with whatever EFLAGS settings were in effect at
309 * the time of the call.
310 *
311 * 2) An interrupt gate pushes the contents of the EFLAGS register at the time
312 * of the interrupt onto the stack, whereas a call gate does not.
313 *
314 * Because we use the following code sequence to handle system calls made from
315 * _both_ a call gate _and_ an interrupt gate, these two differences must be
316 * respected. In regards to number 1) above, the handler must ensure that a sane
317 * EFLAGS snapshot is stored on the stack so that when the kernel returns back
318 * to the user via iret (which returns to user with the EFLAGS value saved on
319 * the stack), interrupts are re-enabled.
320 *
321 * In regards to number 2) above, the handler must always put a current snapshot
322 * of EFLAGS onto the stack in the appropriate place. If we came in via an
323 * interrupt gate, we will be clobbering the EFLAGS value that was pushed by
324 * the interrupt gate. This is OK, as the only bit that was changed by the
325 * hardware was the IE (interrupt enable) bit, which for an interrupt gate is
326 * now off. If we were to do nothing, the stack would contain an EFLAGS with
327 * IE off, resulting in us eventually returning back to the user with interrupts
328 * disabled. The solution is to turn on the IE bit in the EFLAGS value saved on
329 * the stack.
330 *
331 * Another subtlety which deserves mention is the difference between the two
332 * descriptors. The call gate descriptor is set to instruct the hardware to copy
333 * one parameter from the user stack to the kernel stack, whereas the interrupt
334 * gate descriptor doesn't use the parameter passing mechanism at all. The
335 * kernel doesn't actually use the parameter that is copied by the hardware; the
336 * only reason it does this is so that there is a space on the stack large
337 * enough to hold an EFLAGS register value, which happens to be in the correct
338 * place for use by iret when we go back to userland. How convenient.
339 *
340 * Stack frame description in syscall() and callees.
341 *
342 * |------------|
343 * | regs | +(8*4)+4 registers
344 * |------------|
345 * | 8 args | <- %esp MAXSYSARGS (currently 8) arguments
346 * |------------|
347 *
348 */
349 #define SYS_DROP _CONST(_MUL(MAXSYSARGS, 4))
350
351 #if defined(__lint)
352
353 /*ARGSUSED*/
354 void
355 sys_call()
356 {}
357
358 void
359 _allsyscalls()
360 {}
361
362 size_t _allsyscalls_size;
363
364 #else /* __lint */
365
366 ENTRY_NP2(brand_sys_call, _allsyscalls)
367 BRAND_CALLBACK(BRAND_CB_SYSCALL)
368
369 ALTENTRY(sys_call)
370 / on entry eax = system call number
371
372 / set up the stack to look as in reg.h
373 subl $8, %esp / pad the stack with ERRCODE and TRAPNO
374
375 SYSCALL_PUSH
376
377 #ifdef TRAPTRACE
378 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_SYSCALL) / Uses labels "8" and "9"
379 TRACE_REGS(%edi, %esp, %ebx, %ecx) / Uses label "9"
380 pushl %eax
381 TRACE_STAMP(%edi) / Clobbers %eax, %edx, uses "9"
382 popl %eax
383 movl %eax, TTR_SYSNUM(%edi)
384 #endif
385
386 _watch_do_syscall:
387 movl %esp, %ebp
388
389 / Interrupts may be enabled here, so we must make sure this thread
390 / doesn't migrate off the CPU while it updates the CPU stats.
391 /
392 / XXX This is only true if we got here via call gate thru the LDT for
393 / old style syscalls. Perhaps this preempt++-- will go away soon?
394 movl %gs:CPU_THREAD, %ebx
395 addb $1, T_PREEMPT(%ebx)
396 CPU_STATS_SYS_SYSCALL_INC
397 subb $1, T_PREEMPT(%ebx)
398
399 ENABLE_INTR_FLAGS
400
401 pushl %eax / preserve across mstate call
402 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
403 popl %eax
404
405 movl %gs:CPU_THREAD, %ebx
406
407 ASSERT_LWPTOREGS(%ebx, %esp)
408
409 CHECK_PRESYS_NE(%ebx, %eax)
410 jne _full_syscall_presys
411 SIMPLE_SYSCALL_PRESYS(%ebx, _syscall_fault)
412
413 _syslcall_call:
414 call *SY_CALLC(%eax)
415
416 _syslcall_done:
417 CHECK_POSTSYS_NE(%ebx, %ecx)
418 jne _full_syscall_postsys
419 SIMPLE_SYSCALL_POSTSYS(%ebx, %cx)
420 movl %eax, REGOFF_EAX(%esp)
421 movl %edx, REGOFF_EDX(%esp)
422
423 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
424
425 /
426 / get back via iret
427 /
428 CLI(%edx)
429 jmp sys_rtt_syscall
430
431 _full_syscall_presys:
432 movl T_LWP(%ebx), %esi
433 subl $SYS_DROP, %esp
434 movb $LWP_SYS, LWP_STATE(%esi)
435 pushl %esp
436 pushl %ebx
437 call syscall_entry
438 addl $8, %esp
439 jmp _syslcall_call
440
441 _full_syscall_postsys:
442 addl $SYS_DROP, %esp
443 pushl %edx
444 pushl %eax
445 pushl %ebx
446 call syscall_exit
447 addl $12, %esp
448 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
449 jmp _sys_rtt
450
451 _syscall_fault:
452 push $0xe / EFAULT
453 call set_errno
454 addl $4, %esp
455 xorl %eax, %eax / fake syscall_err()
456 xorl %edx, %edx
457 jmp _syslcall_done
458 SET_SIZE(sys_call)
459 SET_SIZE(brand_sys_call)
460
461 #endif /* __lint */
462
463 /*
464 * System call handler via the sysenter instruction
465 *
466 * Here's how syscall entry usually works (see sys_call for details).
467 *
468 * There, the caller (lcall or int) in userland has arranged that:
469 *
470 * - %eax contains the syscall number
471 * - the user stack contains the args to the syscall
472 *
473 * Normally the lcall instruction into the call gate causes the processor
474 * to push %ss, %esp, <top-of-stack>, %cs, %eip onto the kernel stack.
475 * The sys_call handler then leaves space for r_trapno and r_err, and
476 * pusha's {%eax, %ecx, %edx, %ebx, %esp, %ebp, %esi, %edi}, followed
477 * by %ds, %es, %fs and %gs to capture a 'struct regs' on the stack.
478 * Then the kernel sets %ds, %es and %gs to kernel selectors, and finally
479 * extracts %efl and puts it into r_efl (which happens to live at the offset
480 * that <top-of-stack> was copied into). Note that the value in r_efl has
481 * the IF (interrupt enable) flag turned on. (The int instruction into the
482 * interrupt gate does essentially the same thing, only instead of
483 * <top-of-stack> we get eflags - see comment above.)
484 *
485 * In the sysenter case, things are a lot more primitive.
486 *
487 * The caller in userland has arranged that:
488 *
489 * - %eax contains the syscall number
490 * - %ecx contains the user %esp
491 * - %edx contains the return %eip
492 * - the user stack contains the args to the syscall
493 *
494 * e.g.
495 * <args on the stack>
496 * mov $SYS_callnum, %eax
497 * mov $1f, %edx / return %eip
498 * mov %esp, %ecx / return %esp
499 * sysenter
500 * 1:
501 *
502 * Hardware and (privileged) initialization code have arranged that by
503 * the time the sysenter instructions completes:
504 *
505 * - %eip is pointing to sys_sysenter (below).
506 * - %cs and %ss are set to kernel text and stack (data) selectors.
507 * - %esp is pointing at the lwp's stack
508 * - Interrupts have been disabled.
509 *
510 * The task for the sysenter handler is:
511 *
512 * - recreate the same regs structure on the stack and the same
513 * kernel state as if we'd come in on an lcall
514 * - do the normal work of a syscall
515 * - execute the system call epilogue, use sysexit to return to userland.
516 *
517 * Note that we are unable to return both "rvals" to userland with this
518 * call, as %edx is used by the sysexit instruction.
519 *
520 * One final complication in this routine is its interaction with
521 * single-stepping in a debugger. For most of the system call mechanisms,
522 * the CPU automatically clears the single-step flag before we enter the
523 * kernel. The sysenter mechanism does not clear the flag, so a user
524 * single-stepping through a libc routine may suddenly find themself
525 * single-stepping through the kernel. To detect this, kmdb compares the
526 * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
527 * If it finds that we have single-stepped to a sysenter entry point, it
528 * explicitly clears the flag and executes the sys_sysenter routine.
529 *
530 * One final complication in this final complication is the fact that we
531 * have two different entry points for sysenter: brand_sys_sysenter and
532 * sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping
533 * through the kernel with kmdb, we will eventually hit the instruction at
534 * sys_sysenter. kmdb cannot distinguish between that valid single-step
535 * and the undesirable one mentioned above. To avoid this situation, we
536 * simply add a jump over the instruction at sys_sysenter to make it
537 * impossible to single-step to it.
538 */
539 #if defined(__lint)
540
541 void
542 sys_sysenter()
543 {}
544
545 #else /* __lint */
546
547 ENTRY_NP(brand_sys_sysenter)
548 pushl %edx
549 BRAND_CALLBACK(BRAND_CB_SYSENTER)
550 popl %edx
551 /*
552 * Jump over sys_sysenter to allow single-stepping as described
553 * above.
554 */
555 ja 1f
556
557 ALTENTRY(sys_sysenter)
558 nop
559 1:
560 /
561 / do what the call gate would've done to the stack ..
562 /
563 pushl $UDS_SEL / (really %ss, but it's the same ..)
564 pushl %ecx / userland makes this a copy of %esp
565 pushfl
566 orl $PS_IE, (%esp) / turn interrupts on when we return to user
567 pushl $UCS_SEL
568 pushl %edx / userland makes this a copy of %eip
569 /
570 / done. finish building the stack frame
571 /
572 subl $8, %esp / leave space for ERR and TRAPNO
573
574 SYSENTER_PUSH
575
576 #ifdef TRAPTRACE
577 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_SYSENTER) / uses labels 8 and 9
578 TRACE_REGS(%edi, %esp, %ebx, %ecx) / uses label 9
579 pushl %eax
580 TRACE_STAMP(%edi) / clobbers %eax, %edx, uses label 9
581 popl %eax
582 movl %eax, TTR_SYSNUM(%edi)
583 #endif
584 movl %esp, %ebp
585
586 CPU_STATS_SYS_SYSCALL_INC
587
588 ENABLE_INTR_FLAGS
589
590 pushl %eax / preserve across mstate call
591 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
592 popl %eax
593
594 movl %gs:CPU_THREAD, %ebx
595
596 ASSERT_LWPTOREGS(%ebx, %esp)
597
598 CHECK_PRESYS_NE(%ebx, %eax)
599 jne _full_syscall_presys
600 SIMPLE_SYSCALL_PRESYS(%ebx, _syscall_fault)
601
602 _sysenter_call:
603 call *SY_CALLC(%eax)
604
605 _sysenter_done:
606 CHECK_POSTSYS_NE(%ebx, %ecx)
607 jne _full_syscall_postsys
608 SIMPLE_SYSCALL_POSTSYS(%ebx, %cx)
609 /
610 / sysexit uses %edx to restore %eip, so we can't use it
611 / to return a value, sigh.
612 /
613 movl %eax, REGOFF_EAX(%esp)
614 / movl %edx, REGOFF_EDX(%esp)
615
616 / Interrupts will be turned on by the 'sti' executed just before
617 / sysexit. The following ensures that restoring the user's EFLAGS
618 / doesn't enable interrupts too soon.
619 andl $_BITNOT(PS_IE), REGOFF_EFL(%esp)
620
621 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
622
623 cli
624
625 SYSCALL_POP
626
627 popl %edx / sysexit: %edx -> %eip
628 addl $4, %esp / get CS off the stack
629 popfl / EFL
630 popl %ecx / sysexit: %ecx -> %esp
631 sti
632 sysexit
633 SET_SIZE(sys_sysenter)
634 SET_SIZE(brand_sys_sysenter)
635
636 /*
637 * Declare a uintptr_t which covers the entire pc range of syscall
638 * handlers for the stack walkers that need this.
639 */
640 .align CPTRSIZE
641 .globl _allsyscalls_size
642 .type _allsyscalls_size, @object
643 _allsyscalls_size:
644 .NWORD . - _allsyscalls
645 SET_SIZE(_allsyscalls_size)
646
647 #endif /* __lint */
648
649 /*
650 * These are the thread context handlers for lwps using sysenter/sysexit.
651 */
652
653 #if defined(__lint)
654
655 /*ARGSUSED*/
656 void
657 sep_save(void *ksp)
658 {}
659
660 /*ARGSUSED*/
661 void
662 sep_restore(void *ksp)
663 {}
664
665 #else /* __lint */
666
667 /*
668 * setting this value to zero as we switch away causes the
669 * stack-pointer-on-sysenter to be NULL, ensuring that we
670 * don't silently corrupt another (preempted) thread stack
671 * when running an lwp that (somehow) didn't get sep_restore'd
672 */
673 ENTRY_NP(sep_save)
674 xorl %edx, %edx
675 xorl %eax, %eax
676 movl $MSR_INTC_SEP_ESP, %ecx
677 wrmsr
678 ret
679 SET_SIZE(sep_save)
680
681 /*
682 * Update the kernel stack pointer as we resume onto this cpu.
683 */
684 ENTRY_NP(sep_restore)
685 movl 4(%esp), %eax /* per-lwp kernel sp */
686 xorl %edx, %edx
687 movl $MSR_INTC_SEP_ESP, %ecx
688 wrmsr
689 ret
690 SET_SIZE(sep_restore)
691
692 #endif /* __lint */
693
694 /*
695 * Call syscall(). Called from trap() on watchpoint at lcall 0,7
696 */
697
698 #if defined(__lint)
699
700 void
701 watch_syscall(void)
702 {}
703
704 #else /* __lint */
705
706 ENTRY_NP(watch_syscall)
707 CLI(%eax)
708 movl %gs:CPU_THREAD, %ebx
709 movl T_STACK(%ebx), %esp / switch to the thread stack
710 movl REGOFF_EAX(%esp), %eax / recover original syscall#
711 jmp _watch_do_syscall
712 SET_SIZE(watch_syscall)
713
714 #endif /* __lint */
|
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2019 Joyent, Inc.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27 #include <sys/asm_linkage.h>
28 #include <sys/asm_misc.h>
29 #include <sys/regset.h>
30 #include <sys/privregs.h>
31 #include <sys/psw.h>
32 #include <sys/machbrand.h>
33
34 #include <sys/segments.h>
35 #include <sys/pcb.h>
36 #include <sys/trap.h>
37 #include <sys/ftrace.h>
38 #include <sys/traptrace.h>
39 #include <sys/clock.h>
40 #include <sys/model.h>
41 #include <sys/panic.h>
42
43 #if defined(__xpv)
44 #include <sys/hypervisor.h>
45 #endif
46
47 #include "assym.h"
48
49 /*
50 * We implement five flavours of system call entry points
51 *
52 * - syscall/sysretq (amd64 generic)
53 * - syscall/sysretl (i386 plus SYSC bit)
54 * - sysenter/sysexit (i386 plus SEP bit)
55 * - int/iret (i386 generic)
56 * - lcall/iret (i386 generic)
57 *
58 * The current libc included in Solaris uses int/iret as the base unoptimized
59 * kernel entry method. Older libc implementations and legacy binaries may use
60 * the lcall call gate, so it must continue to be supported.
61 *
62 * System calls that use an lcall call gate are processed in trap() via a
63 * segment-not-present trap, i.e. lcalls are extremely slow(!).
64 *
65 * The basic pattern used in the 32-bit SYSC handler at this point in time is
66 * to have the bare minimum of assembler, and get to the C handlers as
67 * quickly as possible.
68 *
69 * The 64-bit handler is much closer to the sparcv9 handler; that's
70 * because of passing arguments in registers. The 32-bit world still
71 * passes arguments on the stack -- that makes that handler substantially
72 * more complex.
73 *
74 * The two handlers share a few code fragments which are broken
75 * out into preprocessor macros below.
76 *
77 * XX64 come back and speed all this up later. The 32-bit stuff looks
78 * especially easy to speed up the argument copying part ..
79 *
80 *
81 * Notes about segment register usage (c.f. the 32-bit kernel)
82 *
83 * In the 32-bit kernel, segment registers are dutifully saved and
84 * restored on all mode transitions because the kernel uses them directly.
85 * When the processor is running in 64-bit mode, segment registers are
86 * largely ignored.
87 *
88 * %cs and %ss
89 * controlled by the hardware mechanisms that make mode transitions
90 *
91 * The remaining segment registers have to either be pointing at a valid
92 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
93 *
94 * %ds and %es
95 * always ignored
96 *
97 * %fs and %gs
98 * fsbase and gsbase are used to control the place they really point at.
99 * The kernel only depends on %gs, and controls its own gsbase via swapgs
100 *
101 * Note that loading segment registers is still costly because the GDT
102 * lookup still happens (this is because the hardware can't know that we're
103 * not setting up these segment registers for a 32-bit program). Thus we
104 * avoid doing this in the syscall path, and defer them to lwp context switch
105 * handlers, so the register values remain virtualized to the lwp.
106 */
107
108 #if defined(SYSCALLTRACE)
109 #define ORL_SYSCALLTRACE(r32) \
110 orl syscalltrace(%rip), r32
111 #else
112 #define ORL_SYSCALLTRACE(r32)
113 #endif
114
115 /*
116 * In the 32-bit kernel, we do absolutely nothing before getting into the
117 * brand callback checks. In 64-bit land, we do swapgs and then come here.
118 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
119 * are still unused.
120 *
121 * Check if a brand_mach_ops callback is defined for the specified callback_id
122 * type. If so invoke it with the kernel's %gs value loaded and the following
123 * data on the stack:
124 *
125 * stack: --------------------------------------
126 * 32 | callback pointer |
127 * | 24 | user (or interrupt) stack pointer |
128 * | 16 | lwp pointer |
129 * v 8 | userland return address |
130 * 0 | callback wrapper return addr |
131 * --------------------------------------
132 *
133 * Since we're pushing the userland return address onto the kernel stack
134 * we need to get that address without accessing the user's stack (since we
135 * can't trust that data). There are different ways to get the userland
136 * return address depending on how the syscall trap was made:
137 *
138 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
139 * b) For sys_sysenter the return address is in %rdx.
140 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
141 * the stack pointer points at the state saved when we took the interrupt:
142 * ------------------------
143 * | | user's %ss |
144 * | | user's %esp |
145 * | | EFLAGS register |
146 * v | user's %cs |
147 * | user's %eip |
148 * ------------------------
149 *
150 * The 2nd parameter to the BRAND_CALLBACK macro is either the
151 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are
152 * used to generate the proper code to get the userland return address for
153 * each syscall entry point.
154 *
155 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
156 * is available as a scratch register within the callback. If the callback
157 * returns within the kernel then this macro will restore %r15. If the
158 * callback is going to return directly to userland then it should restore
159 * %r15 before returning to userland.
160 */
161 #define BRAND_URET_FROM_REG(rip_reg) \
162 pushq rip_reg /* push the return address */
163
164 /*
165 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
166 * is currently pointing at the user return address (%eip).
167 */
168 #define BRAND_URET_FROM_INTR_STACK() \
169 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\
170 pushq (%r15) /* push the return address */
171
172 #define BRAND_CALLBACK(callback_id, push_userland_ret) \
173 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\
174 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\
175 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\
176 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\
177 subq $16, %rsp /* save space for 2 pointers */ ;\
178 pushq %r14 /* save %r14 */ ;\
179 movq %gs:CPU_RTMP_RSP, %r14 ;\
180 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\
181 popq %r14 /* restore %r14 */ ;\
182 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\
183 pushq %r15 /* push the lwp pointer */ ;\
184 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\
185 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\
186 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\
187 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\
188 cmpq $0, %r15 ;\
189 je 1f ;\
190 movq %r15, 16(%rsp) /* save the callback pointer */ ;\
191 push_userland_ret /* push the return address */ ;\
192 movq 24(%rsp), %r15 /* load callback pointer */ ;\
193 INDIRECT_CALL_REG(r15) /* call callback */ ;\
194 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\
195 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */
196
197 #define MSTATE_TRANSITION(from, to) \
198 movl $from, %edi; \
199 movl $to, %esi; \
200 call syscall_mstate
201
202 /*
203 * Check to see if a simple (direct) return is possible i.e.
204 *
205 * if (t->t_post_sys_ast | syscalltrace |
206 * lwp->lwp_pcb.pcb_rupdate == 1)
207 * do full version ;
208 *
209 * Preconditions:
210 * - t is curthread
211 * Postconditions:
212 * - condition code NE is set if post-sys is too complex
213 * - rtmp is zeroed if it isn't (we rely on this!)
214 * - ltmp is smashed
215 */
216 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \
217 movq T_LWP(t), ltmp; \
218 movzbl PCB_RUPDATE(ltmp), rtmp; \
219 ORL_SYSCALLTRACE(rtmp); \
220 orl T_POST_SYS_AST(t), rtmp; \
221 cmpl $0, rtmp
222
223 /*
224 * Fix up the lwp, thread, and eflags for a successful return
225 *
226 * Preconditions:
227 * - zwreg contains zero
228 */
229 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \
230 movb $LWP_USER, LWP_STATE(lwp); \
231 movw zwreg, T_SYSNUM(t); \
232 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
233
234 /*
235 * ASSERT(lwptoregs(lwp) == rp);
236 *
237 * This may seem obvious, but very odd things happen if this
238 * assertion is false
239 *
240 * Preconditions:
241 * (%rsp is ready for normal call sequence)
242 * Postconditions (if assertion is true):
243 * %r11 is smashed
244 *
245 * ASSERT(rp->r_cs == descnum)
246 *
247 * The code selector is written into the regs structure when the
248 * lwp stack is created. We use this ASSERT to validate that
249 * the regs structure really matches how we came in.
250 *
251 * Preconditions:
252 * (%rsp is ready for normal call sequence)
253 * Postconditions (if assertion is true):
254 * -none-
255 *
256 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
257 *
258 * If this is false, it meant that we returned to userland without
259 * updating the segment registers as we were supposed to.
260 *
261 * Note that we must ensure no interrupts or other traps intervene
262 * between entering privileged mode and performing the assertion,
263 * otherwise we may perform a context switch on the thread, which
264 * will end up setting pcb_rupdate to 1 again.
265 *
266 * ASSERT(%cr0 & CR0_TS == 0);
267 * Preconditions:
268 * (%rsp is ready for normal call sequence)
269 * Postconditions (if assertion is true):
270 * (specified register is clobbered)
271 *
272 * Check to make sure that we are returning to user land and that CR0.TS
273 * is not set. This is required as part of the eager FPU (see
274 * uts/intel/ia32/os/fpu.c for more information).
275 */
276
277 #if defined(DEBUG)
278
279 __lwptoregs_msg:
280 .string "syscall_asm.s:%d lwptoregs(%p) [%p] != rp [%p]"
281
282 __codesel_msg:
283 .string "syscall_asm.s:%d rp->r_cs [%ld] != %ld"
284
285 __no_rupdate_msg:
286 .string "syscall_asm.s:%d lwp %p, pcb_rupdate != 0"
287
288 __bad_ts_msg:
289 .string "syscall_asm.s:%d CR0.TS set on user return"
290
291 #define ASSERT_LWPTOREGS(lwp, rp) \
292 movq LWP_REGS(lwp), %r11; \
293 cmpq rp, %r11; \
294 je 7f; \
295 leaq __lwptoregs_msg(%rip), %rdi; \
296 movl $__LINE__, %esi; \
297 movq lwp, %rdx; \
298 movq %r11, %rcx; \
299 movq rp, %r8; \
300 xorl %eax, %eax; \
301 call panic; \
302 7:
303
304 #define ASSERT_NO_RUPDATE_PENDING(lwp) \
305 testb $0x1, PCB_RUPDATE(lwp); \
306 je 8f; \
307 movq lwp, %rdx; \
308 leaq __no_rupdate_msg(%rip), %rdi; \
309 movl $__LINE__, %esi; \
310 xorl %eax, %eax; \
311 call panic; \
312 8:
313
314 #define ASSERT_CR0TS_ZERO(reg) \
315 movq %cr0, reg; \
316 testq $CR0_TS, reg; \
317 jz 9f; \
318 leaq __bad_ts_msg(%rip), %rdi; \
319 movl $__LINE__, %esi; \
320 xorl %eax, %eax; \
321 call panic; \
322 9:
323
324 #else
325 #define ASSERT_LWPTOREGS(lwp, rp)
326 #define ASSERT_NO_RUPDATE_PENDING(lwp)
327 #define ASSERT_CR0TS_ZERO(reg)
328 #endif
329
330 /*
331 * Do the traptrace thing and restore any registers we used
332 * in situ. Assumes that %rsp is pointing at the base of
333 * the struct regs, obviously ..
334 */
335 #ifdef TRAPTRACE
336 #define SYSCALL_TRAPTRACE(ttype) \
337 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \
338 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \
339 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \
340 movq REGOFF_RAX(%rsp), %rax; \
341 movq REGOFF_RBX(%rsp), %rbx; \
342 movq REGOFF_RCX(%rsp), %rcx; \
343 movq REGOFF_RDX(%rsp), %rdx; \
344 movl %eax, TTR_SYSNUM(%rdi); \
345 movq REGOFF_RDI(%rsp), %rdi
346
347 #define SYSCALL_TRAPTRACE32(ttype) \
348 SYSCALL_TRAPTRACE(ttype); \
349 /* paranoia: clean the top 32-bits of the registers */ \
350 orl %eax, %eax; \
351 orl %ebx, %ebx; \
352 orl %ecx, %ecx; \
353 orl %edx, %edx; \
354 orl %edi, %edi
355 #else /* TRAPTRACE */
356 #define SYSCALL_TRAPTRACE(ttype)
357 #define SYSCALL_TRAPTRACE32(ttype)
358 #endif /* TRAPTRACE */
359
360 /*
361 * The 64-bit libc syscall wrapper does this:
362 *
363 * fn(<args>)
364 * {
365 * movq %rcx, %r10 -- because syscall smashes %rcx
366 * movl $CODE, %eax
367 * syscall
368 * <error processing>
369 * }
370 *
371 * Thus when we come into the kernel:
372 *
373 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
374 * %rax is the syscall number
375 * %r12-%r15 contain caller state
376 *
377 * The syscall instruction arranges that:
378 *
379 * %rcx contains the return %rip
380 * %r11d contains bottom 32-bits of %rflags
381 * %rflags is masked (as determined by the SFMASK msr)
382 * %cs is set to UCS_SEL (as determined by the STAR msr)
383 * %ss is set to UDS_SEL (as determined by the STAR msr)
384 * %rip is set to sys_syscall (as determined by the LSTAR msr)
385 *
386 * Or in other words, we have no registers available at all.
387 * Only swapgs can save us!
388 *
389 * Under the hypervisor, the swapgs has happened already. However, the
390 * state of the world is very different from that we're familiar with.
391 *
392 * In particular, we have a stack structure like that for interrupt
393 * gates, except that the %cs and %ss registers are modified for reasons
394 * that are not entirely clear. Critically, the %rcx/%r11 values do
395 * *not* reflect the usage of those registers under a 'real' syscall[1];
396 * the stack, therefore, looks like this:
397 *
398 * 0x0(rsp) potentially junk %rcx
399 * 0x8(rsp) potentially junk %r11
400 * 0x10(rsp) user %rip
401 * 0x18(rsp) modified %cs
402 * 0x20(rsp) user %rflags
403 * 0x28(rsp) user %rsp
404 * 0x30(rsp) modified %ss
405 *
406 *
407 * and before continuing on, we must load the %rip into %rcx and the
408 * %rflags into %r11.
409 *
410 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
411 * Sigh.
412 */
413 #if defined(__xpv)
414 #define XPV_SYSCALL_PROD \
415 movq 0x10(%rsp), %rcx; \
416 movq 0x20(%rsp), %r11; \
417 movq 0x28(%rsp), %rsp
418 #else
419 #define XPV_SYSCALL_PROD /* nothing */
420 #endif
421
422 ENTRY_NP2(brand_sys_syscall,_allsyscalls)
423 SWAPGS /* kernel gsbase */
424 XPV_SYSCALL_PROD
425 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
426 jmp noprod_sys_syscall
427
428 ALTENTRY(sys_syscall)
429 SWAPGS /* kernel gsbase */
430 XPV_SYSCALL_PROD
431
432 noprod_sys_syscall:
433 movq %r15, %gs:CPU_RTMP_R15
434 movq %rsp, %gs:CPU_RTMP_RSP
435
436 movq %gs:CPU_THREAD, %r15
437 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */
438
439 ASSERT_UPCALL_MASK_IS_SET
440
441 movl $UCS_SEL, REGOFF_CS(%rsp)
442 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
443 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
444 movl $UDS_SEL, REGOFF_SS(%rsp)
445
446 movl %eax, %eax /* wrapper: sysc# -> %eax */
447 movq %rdi, REGOFF_RDI(%rsp)
448 movq %rsi, REGOFF_RSI(%rsp)
449 movq %rdx, REGOFF_RDX(%rsp)
450 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */
451 movq %r10, %rcx /* arg[3] for direct calls */
452
453 movq %r8, REGOFF_R8(%rsp)
454 movq %r9, REGOFF_R9(%rsp)
455 movq %rax, REGOFF_RAX(%rsp)
456 movq %rbx, REGOFF_RBX(%rsp)
457
458 movq %rbp, REGOFF_RBP(%rsp)
459 movq %r10, REGOFF_R10(%rsp)
460 movq %gs:CPU_RTMP_RSP, %r11
461 movq %r11, REGOFF_RSP(%rsp)
462 movq %r12, REGOFF_R12(%rsp)
463
464 movq %r13, REGOFF_R13(%rsp)
465 movq %r14, REGOFF_R14(%rsp)
466 movq %gs:CPU_RTMP_R15, %r10
467 movq %r10, REGOFF_R15(%rsp)
468 movq $0, REGOFF_SAVFP(%rsp)
469 movq $0, REGOFF_SAVPC(%rsp)
470
471 /*
472 * Copy these registers here in case we end up stopped with
473 * someone (like, say, /proc) messing with our register state.
474 * We don't -restore- them unless we have to in update_sregs.
475 *
476 * Since userland -can't- change fsbase or gsbase directly,
477 * and capturing them involves two serializing instructions,
478 * we don't bother to capture them here.
479 */
480 xorl %ebx, %ebx
481 movw %ds, %bx
482 movq %rbx, REGOFF_DS(%rsp)
483 movw %es, %bx
484 movq %rbx, REGOFF_ES(%rsp)
485 movw %fs, %bx
486 movq %rbx, REGOFF_FS(%rsp)
487 movw %gs, %bx
488 movq %rbx, REGOFF_GS(%rsp)
489
490 /*
491 * If we're trying to use TRAPTRACE though, I take that back: we're
492 * probably debugging some problem in the SWAPGS logic and want to know
493 * what the incoming gsbase was.
494 *
495 * Since we already did SWAPGS, record the KGSBASE.
496 */
497 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
498 movl $MSR_AMD_KGSBASE, %ecx
499 rdmsr
500 movl %eax, REGOFF_GSBASE(%rsp)
501 movl %edx, REGOFF_GSBASE+4(%rsp)
502 #endif
503
504 /*
505 * Machine state saved in the regs structure on the stack
506 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
507 * %eax is the syscall number
508 * %rsp is the thread's stack, %r15 is curthread
509 * REG_RSP(%rsp) is the user's stack
510 */
511
512 SYSCALL_TRAPTRACE($TT_SYSC64)
513
514 movq %rsp, %rbp
515
516 movq T_LWP(%r15), %r14
517 ASSERT_NO_RUPDATE_PENDING(%r14)
518 ENABLE_INTR_FLAGS
519
520 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
521 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
522
523 ASSERT_LWPTOREGS(%r14, %rsp)
524
525 movb $LWP_SYS, LWP_STATE(%r14)
526 incq LWP_RU_SYSC(%r14)
527 movb $NORMALRETURN, LWP_EOSYS(%r14)
528
529 incq %gs:CPU_STATS_SYS_SYSCALL
530
531 movw %ax, T_SYSNUM(%r15)
532 movzbl T_PRE_SYS(%r15), %ebx
533 ORL_SYSCALLTRACE(%ebx)
534 testl %ebx, %ebx
535 jne _syscall_pre
536
537 _syscall_invoke:
538 movq REGOFF_RDI(%rbp), %rdi
539 movq REGOFF_RSI(%rbp), %rsi
540 movq REGOFF_RDX(%rbp), %rdx
541 movq REGOFF_RCX(%rbp), %rcx
542 movq REGOFF_R8(%rbp), %r8
543 movq REGOFF_R9(%rbp), %r9
544
545 cmpl $NSYSCALL, %eax
546 jae _syscall_ill
547 shll $SYSENT_SIZE_SHIFT, %eax
548 leaq sysent(%rax), %rbx
549
550 movq SY_CALLC(%rbx), %rax
551 INDIRECT_CALL_REG(rax)
552
553 movq %rax, %r12
554 movq %rdx, %r13
555
556 /*
557 * If the handler returns two ints, then we need to split the
558 * 64-bit return value into two 32-bit values.
559 */
560 testw $SE_32RVAL2, SY_FLAGS(%rbx)
561 je 5f
562 movq %r12, %r13
563 shrq $32, %r13 /* upper 32-bits into %edx */
564 movl %r12d, %r12d /* lower 32-bits into %eax */
565 5:
566 /*
567 * Optimistically assume that there's no post-syscall
568 * work to do. (This is to avoid having to call syscall_mstate()
569 * with interrupts disabled)
570 */
571 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
572
573 /*
574 * We must protect ourselves from being descheduled here;
575 * If we were, and we ended up on another cpu, or another
576 * lwp got in ahead of us, it could change the segment
577 * registers without us noticing before we return to userland.
578 */
579 CLI(%r14)
580 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
581 jne _syscall_post
582
583 /*
584 * We need to protect ourselves against non-canonical return values
585 * because Intel doesn't check for them on sysret (AMD does). Canonical
586 * addresses on current amd64 processors only use 48-bits for VAs; an
587 * address is canonical if all upper bits (47-63) are identical. If we
588 * find a non-canonical %rip, we opt to go through the full
589 * _syscall_post path which takes us into an iretq which is not
590 * susceptible to the same problems sysret is.
591 *
592 * We're checking for a canonical address by first doing an arithmetic
593 * shift. This will fill in the remaining bits with the value of bit 63.
594 * If the address were canonical, the register would now have either all
595 * zeroes or all ones in it. Therefore we add one (inducing overflow)
596 * and compare against 1. A canonical address will either be zero or one
597 * at this point, hence the use of ja.
598 *
599 * At this point, r12 and r13 have the return value so we can't use
600 * those registers.
601 */
602 movq REGOFF_RIP(%rsp), %rcx
603 sarq $47, %rcx
604 incq %rcx
605 cmpq $1, %rcx
606 ja _syscall_post
607
608
609 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
610
611 movq %r12, REGOFF_RAX(%rsp)
612 movq %r13, REGOFF_RDX(%rsp)
613
614 /*
615 * Clobber %r11 as we check CR0.TS.
616 */
617 ASSERT_CR0TS_ZERO(%r11)
618
619 /*
620 * Unlike other cases, because we need to restore the user stack pointer
621 * before exiting the kernel we must clear the microarch state before
622 * getting here. This should be safe because it means that the only
623 * values on the bus after this are based on the user's registers and
624 * potentially the addresses where we stored them. Given the constraints
625 * of sysret, that's how it has to be.
626 */
627 call x86_md_clear
628
629 /*
630 * To get back to userland, we need the return %rip in %rcx and
631 * the return %rfl in %r11d. The sysretq instruction also arranges
632 * to fix up %cs and %ss; everything else is our responsibility.
633 */
634 movq REGOFF_RDI(%rsp), %rdi
635 movq REGOFF_RSI(%rsp), %rsi
636 movq REGOFF_RDX(%rsp), %rdx
637 /* %rcx used to restore %rip value */
638
639 movq REGOFF_R8(%rsp), %r8
640 movq REGOFF_R9(%rsp), %r9
641 movq REGOFF_RAX(%rsp), %rax
642 movq REGOFF_RBX(%rsp), %rbx
643
644 movq REGOFF_RBP(%rsp), %rbp
645 movq REGOFF_R10(%rsp), %r10
646 /* %r11 used to restore %rfl value */
647 movq REGOFF_R12(%rsp), %r12
648
649 movq REGOFF_R13(%rsp), %r13
650 movq REGOFF_R14(%rsp), %r14
651 movq REGOFF_R15(%rsp), %r15
652
653 movq REGOFF_RIP(%rsp), %rcx
654 movl REGOFF_RFL(%rsp), %r11d
655
656 #if defined(__xpv)
657 addq $REGOFF_RIP, %rsp
658 #else
659 movq REGOFF_RSP(%rsp), %rsp
660 #endif
661
662 /*
663 * There can be no instructions between the ALTENTRY below and
664 * SYSRET or we could end up breaking brand support. See label usage
665 * in sn1_brand_syscall_callback for an example.
666 */
667 ASSERT_UPCALL_MASK_IS_SET
668 #if defined(__xpv)
669 SYSRETQ
670 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
671
672 /*
673 * We can only get here after executing a brand syscall
674 * interposition callback handler and simply need to
675 * "sysretq" back to userland. On the hypervisor this
676 * involves the iret hypercall which requires us to construct
677 * just enough of the stack needed for the hypercall.
678 * (rip, cs, rflags, rsp, ss).
679 */
680 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */
681 movq %gs:CPU_THREAD, %r11
682 movq T_STACK(%r11), %rsp
683
684 movq %rcx, REGOFF_RIP(%rsp)
685 movl $UCS_SEL, REGOFF_CS(%rsp)
686 movq %gs:CPU_RTMP_RSP, %r11
687 movq %r11, REGOFF_RSP(%rsp)
688 pushfq
689 popq %r11 /* hypercall enables ints */
690 movq %r11, REGOFF_RFL(%rsp)
691 movl $UDS_SEL, REGOFF_SS(%rsp)
692 addq $REGOFF_RIP, %rsp
693 /*
694 * XXPV: see comment in SYSRETQ definition for future optimization
695 * we could take.
696 */
697 ASSERT_UPCALL_MASK_IS_SET
698 SYSRETQ
699 #else
700 ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
701 jmp tr_sysretq
702 #endif
703 /*NOTREACHED*/
704 SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
705
706 _syscall_pre:
707 call pre_syscall
708 movl %eax, %r12d
709 testl %eax, %eax
710 jne _syscall_post_call
711 /*
712 * Didn't abort, so reload the syscall args and invoke the handler.
713 */
714 movzwl T_SYSNUM(%r15), %eax
715 jmp _syscall_invoke
716
717 _syscall_ill:
718 call nosys
719 movq %rax, %r12
720 movq %rdx, %r13
721 jmp _syscall_post_call
722
723 _syscall_post:
724 STI
725 /*
726 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
727 * so that we can account for the extra work it takes us to finish.
728 */
729 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
730 _syscall_post_call:
731 movq %r12, %rdi
732 movq %r13, %rsi
733 call post_syscall
734 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
735 jmp _sys_rtt
736 SET_SIZE(sys_syscall)
737 SET_SIZE(brand_sys_syscall)
738
739 ENTRY_NP(brand_sys_syscall32)
740 SWAPGS /* kernel gsbase */
741 XPV_TRAP_POP
742 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
743 jmp nopop_sys_syscall32
744
745 ALTENTRY(sys_syscall32)
746 SWAPGS /* kernel gsbase */
747 XPV_TRAP_POP
748
749 nopop_sys_syscall32:
750 movl %esp, %r10d
751 movq %gs:CPU_THREAD, %r15
752 movq T_STACK(%r15), %rsp
753 movl %eax, %eax
754
755 movl $U32CS_SEL, REGOFF_CS(%rsp)
756 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */
757 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */
758 movq %r10, REGOFF_RSP(%rsp)
759 movl $UDS_SEL, REGOFF_SS(%rsp)
760
761 _syscall32_save:
762 movl %edi, REGOFF_RDI(%rsp)
763 movl %esi, REGOFF_RSI(%rsp)
764 movl %ebp, REGOFF_RBP(%rsp)
765 movl %ebx, REGOFF_RBX(%rsp)
766 movl %edx, REGOFF_RDX(%rsp)
767 movl %ecx, REGOFF_RCX(%rsp)
768 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
769 movq $0, REGOFF_SAVFP(%rsp)
770 movq $0, REGOFF_SAVPC(%rsp)
771
772 /*
773 * Copy these registers here in case we end up stopped with
774 * someone (like, say, /proc) messing with our register state.
775 * We don't -restore- them unless we have to in update_sregs.
776 *
777 * Since userland -can't- change fsbase or gsbase directly,
778 * we don't bother to capture them here.
779 */
780 xorl %ebx, %ebx
781 movw %ds, %bx
782 movq %rbx, REGOFF_DS(%rsp)
783 movw %es, %bx
784 movq %rbx, REGOFF_ES(%rsp)
785 movw %fs, %bx
786 movq %rbx, REGOFF_FS(%rsp)
787 movw %gs, %bx
788 movq %rbx, REGOFF_GS(%rsp)
789
790 /*
791 * If we're trying to use TRAPTRACE though, I take that back: we're
792 * probably debugging some problem in the SWAPGS logic and want to know
793 * what the incoming gsbase was.
794 *
795 * Since we already did SWAPGS, record the KGSBASE.
796 */
797 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
798 movl $MSR_AMD_KGSBASE, %ecx
799 rdmsr
800 movl %eax, REGOFF_GSBASE(%rsp)
801 movl %edx, REGOFF_GSBASE+4(%rsp)
802 #endif
803
804 /*
805 * Application state saved in the regs structure on the stack
806 * %eax is the syscall number
807 * %rsp is the thread's stack, %r15 is curthread
808 * REG_RSP(%rsp) is the user's stack
809 */
810
811 SYSCALL_TRAPTRACE32($TT_SYSC)
812
813 movq %rsp, %rbp
814
815 movq T_LWP(%r15), %r14
816 ASSERT_NO_RUPDATE_PENDING(%r14)
817
818 ENABLE_INTR_FLAGS
819
820 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
821 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */
822
823 ASSERT_LWPTOREGS(%r14, %rsp)
824
825 incq %gs:CPU_STATS_SYS_SYSCALL
826
827 /*
828 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
829 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or
830 * more succinctly:
831 *
832 * SA(MAXSYSARGS * sizeof (long)) == 64
833 */
834 #define SYS_DROP 64 /* drop for args */
835 subq $SYS_DROP, %rsp
836 movb $LWP_SYS, LWP_STATE(%r14)
837 movq %r15, %rdi
838 movq %rsp, %rsi
839 call syscall_entry
840
841 /*
842 * Fetch the arguments copied onto the kernel stack and put
843 * them in the right registers to invoke a C-style syscall handler.
844 * %rax contains the handler address.
845 *
846 * Ideas for making all this go faster of course include simply
847 * forcibly fetching 6 arguments from the user stack under lofault
848 * protection, reverting to copyin_args only when watchpoints
849 * are in effect.
850 *
851 * (If we do this, make sure that exec and libthread leave
852 * enough space at the top of the stack to ensure that we'll
853 * never do a fetch from an invalid page.)
854 *
855 * Lots of ideas here, but they won't really help with bringup B-)
856 * Correctness can't wait, performance can wait a little longer ..
857 */
858
859 movq %rax, %rbx
860 movl 0(%rsp), %edi
861 movl 8(%rsp), %esi
862 movl 0x10(%rsp), %edx
863 movl 0x18(%rsp), %ecx
864 movl 0x20(%rsp), %r8d
865 movl 0x28(%rsp), %r9d
866
867 movq SY_CALLC(%rbx), %rax
868 INDIRECT_CALL_REG(rax)
869
870 movq %rbp, %rsp /* pop the args */
871
872 /*
873 * amd64 syscall handlers -always- return a 64-bit value in %rax.
874 * On the 32-bit kernel, they always return that value in %eax:%edx
875 * as required by the 32-bit ABI.
876 *
877 * Simulate the same behaviour by unconditionally splitting the
878 * return value in the same way.
879 */
880 movq %rax, %r13
881 shrq $32, %r13 /* upper 32-bits into %edx */
882 movl %eax, %r12d /* lower 32-bits into %eax */
883
884 /*
885 * Optimistically assume that there's no post-syscall
886 * work to do. (This is to avoid having to call syscall_mstate()
887 * with interrupts disabled)
888 */
889 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
890
891 /*
892 * We must protect ourselves from being descheduled here;
893 * If we were, and we ended up on another cpu, or another
894 * lwp got in ahead of us, it could change the segment
895 * registers without us noticing before we return to userland.
896 */
897 CLI(%r14)
898 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
899 jne _full_syscall_postsys32
900 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
901
902 /*
903 * Clobber %r11 as we check CR0.TS.
904 */
905 ASSERT_CR0TS_ZERO(%r11)
906
907 /*
908 * Unlike other cases, because we need to restore the user stack pointer
909 * before exiting the kernel we must clear the microarch state before
910 * getting here. This should be safe because it means that the only
911 * values on the bus after this are based on the user's registers and
912 * potentially the addresses where we stored them. Given the constraints
913 * of sysret, that's how it has to be.
914 */
915 call x86_md_clear
916
917 /*
918 * To get back to userland, we need to put the return %rip in %rcx and
919 * the return %rfl in %r11d. The sysret instruction also arranges
920 * to fix up %cs and %ss; everything else is our responsibility.
921 */
922
923 movl %r12d, %eax /* %eax: rval1 */
924 movl REGOFF_RBX(%rsp), %ebx
925 /* %ecx used for return pointer */
926 movl %r13d, %edx /* %edx: rval2 */
927 movl REGOFF_RBP(%rsp), %ebp
928 movl REGOFF_RSI(%rsp), %esi
929 movl REGOFF_RDI(%rsp), %edi
930
931 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */
932 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */
933 movl REGOFF_RSP(%rsp), %esp
934
935 ASSERT_UPCALL_MASK_IS_SET
936 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
937 jmp tr_sysretl
938 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
939 /*NOTREACHED*/
940
941 _full_syscall_postsys32:
942 STI
943 /*
944 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
945 * so that we can account for the extra work it takes us to finish.
946 */
947 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
948 movq %r15, %rdi
949 movq %r12, %rsi /* rval1 - %eax */
950 movq %r13, %rdx /* rval2 - %edx */
951 call syscall_exit
952 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
953 jmp _sys_rtt
954 SET_SIZE(sys_syscall32)
955 SET_SIZE(brand_sys_syscall32)
956
957 /*
958 * System call handler via the sysenter instruction
959 * Used only for 32-bit system calls on the 64-bit kernel.
960 *
961 * The caller in userland has arranged that:
962 *
963 * - %eax contains the syscall number
964 * - %ecx contains the user %esp
965 * - %edx contains the return %eip
966 * - the user stack contains the args to the syscall
967 *
968 * Hardware and (privileged) initialization code have arranged that by
969 * the time the sysenter instructions completes:
970 *
971 * - %rip is pointing to sys_sysenter (below).
972 * - %cs and %ss are set to kernel text and stack (data) selectors.
973 * - %rsp is pointing at the lwp's stack
974 * - interrupts have been disabled.
975 *
976 * Note that we are unable to return both "rvals" to userland with
977 * this call, as %edx is used by the sysexit instruction.
978 *
979 * One final complication in this routine is its interaction with
980 * single-stepping in a debugger. For most of the system call mechanisms, the
981 * CPU automatically clears the single-step flag before we enter the kernel.
982 * The sysenter mechanism does not clear the flag, so a user single-stepping
983 * through a libc routine may suddenly find themself single-stepping through the
984 * kernel. To detect this, kmdb and trap() both compare the trap %pc to the
985 * [brand_]sys_enter addresses on each single-step trap. If it finds that we
986 * have single-stepped to a sysenter entry point, it explicitly clears the flag
987 * and executes the sys_sysenter routine.
988 *
989 * One final complication in this final complication is the fact that we have
990 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
991 * If we enter at brand_sys_sysenter and start single-stepping through the
992 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
993 * kmdb cannot distinguish between that valid single-step and the undesirable
994 * one mentioned above. To avoid this situation, we simply add a jump over the
995 * instruction at sys_sysenter to make it impossible to single-step to it.
996 */
997
998 ENTRY_NP(brand_sys_sysenter)
999 SWAPGS /* kernel gsbase */
1000 ALTENTRY(_brand_sys_sysenter_post_swapgs)
1001
1002 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
1003 /*
1004 * Jump over sys_sysenter to allow single-stepping as described
1005 * above.
1006 */
1007 jmp _sys_sysenter_post_swapgs
1008
1009 ALTENTRY(sys_sysenter)
1010 SWAPGS /* kernel gsbase */
1011 ALTENTRY(_sys_sysenter_post_swapgs)
1012
1013 movq %gs:CPU_THREAD, %r15
1014
1015 movl $U32CS_SEL, REGOFF_CS(%rsp)
1016 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */
1017 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */
1018 /*
1019 * NOTE: none of the instructions that run before we get here should
1020 * clobber bits in (R)FLAGS! This includes the kpti trampoline.
1021 */
1022 pushfq
1023 popq %r10
1024 movl $UDS_SEL, REGOFF_SS(%rsp)
1025
1026 /*
1027 * Set the interrupt flag before storing the flags to the
1028 * flags image on the stack so we can return to user with
1029 * interrupts enabled if we return via sys_rtt_syscall32
1030 */
1031 orq $PS_IE, %r10
1032 movq %r10, REGOFF_RFL(%rsp)
1033
1034 movl %edi, REGOFF_RDI(%rsp)
1035 movl %esi, REGOFF_RSI(%rsp)
1036 movl %ebp, REGOFF_RBP(%rsp)
1037 movl %ebx, REGOFF_RBX(%rsp)
1038 movl %edx, REGOFF_RDX(%rsp)
1039 movl %ecx, REGOFF_RCX(%rsp)
1040 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */
1041 movq $0, REGOFF_SAVFP(%rsp)
1042 movq $0, REGOFF_SAVPC(%rsp)
1043
1044 /*
1045 * Copy these registers here in case we end up stopped with
1046 * someone (like, say, /proc) messing with our register state.
1047 * We don't -restore- them unless we have to in update_sregs.
1048 *
1049 * Since userland -can't- change fsbase or gsbase directly,
1050 * we don't bother to capture them here.
1051 */
1052 xorl %ebx, %ebx
1053 movw %ds, %bx
1054 movq %rbx, REGOFF_DS(%rsp)
1055 movw %es, %bx
1056 movq %rbx, REGOFF_ES(%rsp)
1057 movw %fs, %bx
1058 movq %rbx, REGOFF_FS(%rsp)
1059 movw %gs, %bx
1060 movq %rbx, REGOFF_GS(%rsp)
1061
1062 /*
1063 * If we're trying to use TRAPTRACE though, I take that back: we're
1064 * probably debugging some problem in the SWAPGS logic and want to know
1065 * what the incoming gsbase was.
1066 *
1067 * Since we already did SWAPGS, record the KGSBASE.
1068 */
1069 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
1070 movl $MSR_AMD_KGSBASE, %ecx
1071 rdmsr
1072 movl %eax, REGOFF_GSBASE(%rsp)
1073 movl %edx, REGOFF_GSBASE+4(%rsp)
1074 #endif
1075
1076 /*
1077 * Application state saved in the regs structure on the stack
1078 * %eax is the syscall number
1079 * %rsp is the thread's stack, %r15 is curthread
1080 * REG_RSP(%rsp) is the user's stack
1081 */
1082
1083 SYSCALL_TRAPTRACE($TT_SYSENTER)
1084
1085 movq %rsp, %rbp
1086
1087 movq T_LWP(%r15), %r14
1088 ASSERT_NO_RUPDATE_PENDING(%r14)
1089
1090 ENABLE_INTR_FLAGS
1091
1092 /*
1093 * Catch 64-bit process trying to issue sysenter instruction
1094 * on Nocona based systems.
1095 */
1096 movq LWP_PROCP(%r14), %rax
1097 cmpq $DATAMODEL_ILP32, P_MODEL(%rax)
1098 je 7f
1099
1100 /*
1101 * For a non-32-bit process, simulate a #ud, since that's what
1102 * native hardware does. The traptrace entry (above) will
1103 * let you know what really happened.
1104 */
1105 movq $T_ILLINST, REGOFF_TRAPNO(%rsp)
1106 movq REGOFF_CS(%rsp), %rdi
1107 movq %rdi, REGOFF_ERR(%rsp)
1108 movq %rsp, %rdi
1109 movq REGOFF_RIP(%rsp), %rsi
1110 movl %gs:CPU_ID, %edx
1111 call trap
1112 jmp _sys_rtt
1113 7:
1114
1115 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1116 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */
1117
1118 ASSERT_LWPTOREGS(%r14, %rsp)
1119
1120 incq %gs:CPU_STATS_SYS_SYSCALL
1121
1122 /*
1123 * Make some space for MAXSYSARGS (currently 8) 32-bit args
1124 * placed into 64-bit (long) arg slots, plus one 64-bit
1125 * (long) arg count, maintaining 16 byte alignment.
1126 */
1127 subq $SYS_DROP, %rsp
1128 movb $LWP_SYS, LWP_STATE(%r14)
1129 movq %r15, %rdi
1130 movq %rsp, %rsi
1131 call syscall_entry
1132
1133 /*
1134 * Fetch the arguments copied onto the kernel stack and put
1135 * them in the right registers to invoke a C-style syscall handler.
1136 * %rax contains the handler address.
1137 */
1138 movq %rax, %rbx
1139 movl 0(%rsp), %edi
1140 movl 8(%rsp), %esi
1141 movl 0x10(%rsp), %edx
1142 movl 0x18(%rsp), %ecx
1143 movl 0x20(%rsp), %r8d
1144 movl 0x28(%rsp), %r9d
1145
1146 movq SY_CALLC(%rbx), %rax
1147 INDIRECT_CALL_REG(rax)
1148
1149 movq %rbp, %rsp /* pop the args */
1150
1151 /*
1152 * amd64 syscall handlers -always- return a 64-bit value in %rax.
1153 * On the 32-bit kernel, the always return that value in %eax:%edx
1154 * as required by the 32-bit ABI.
1155 *
1156 * Simulate the same behaviour by unconditionally splitting the
1157 * return value in the same way.
1158 */
1159 movq %rax, %r13
1160 shrq $32, %r13 /* upper 32-bits into %edx */
1161 movl %eax, %r12d /* lower 32-bits into %eax */
1162
1163 /*
1164 * Optimistically assume that there's no post-syscall
1165 * work to do. (This is to avoid having to call syscall_mstate()
1166 * with interrupts disabled)
1167 */
1168 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1169
1170 /*
1171 * We must protect ourselves from being descheduled here;
1172 * If we were, and we ended up on another cpu, or another
1173 * lwp got int ahead of us, it could change the segment
1174 * registers without us noticing before we return to userland.
1175 *
1176 * This cli is undone in the tr_sysexit trampoline code.
1177 */
1178 cli
1179 CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1180 jne _full_syscall_postsys32
1181 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1182
1183 /*
1184 * To get back to userland, load up the 32-bit registers and
1185 * sysexit back where we came from.
1186 */
1187
1188 /*
1189 * Interrupts will be turned on by the 'sti' executed just before
1190 * sysexit. The following ensures that restoring the user's rflags
1191 * doesn't enable interrupts too soon.
1192 */
1193 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1194
1195 /*
1196 * Clobber %r11 as we check CR0.TS.
1197 */
1198 ASSERT_CR0TS_ZERO(%r11)
1199
1200 /*
1201 * (There's no point in loading up %edx because the sysexit
1202 * mechanism smashes it.)
1203 */
1204 movl %r12d, %eax
1205 movl REGOFF_RBX(%rsp), %ebx
1206 movl REGOFF_RBP(%rsp), %ebp
1207 movl REGOFF_RSI(%rsp), %esi
1208 movl REGOFF_RDI(%rsp), %edi
1209
1210 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */
1211 pushq REGOFF_RFL(%rsp)
1212 popfq
1213 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */
1214 ALTENTRY(sys_sysenter_swapgs_sysexit)
1215 call x86_md_clear
1216 jmp tr_sysexit
1217 SET_SIZE(sys_sysenter_swapgs_sysexit)
1218 SET_SIZE(sys_sysenter)
1219 SET_SIZE(_sys_sysenter_post_swapgs)
1220 SET_SIZE(brand_sys_sysenter)
1221
1222 /*
1223 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1224 * the generic i386 libc to do system calls. We do a small amount of setup
1225 * before jumping into the existing sys_syscall32 path.
1226 */
1227
1228 ENTRY_NP(brand_sys_syscall_int)
1229 SWAPGS /* kernel gsbase */
1230 XPV_TRAP_POP
1231 call smap_enable
1232 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1233 jmp nopop_syscall_int
1234
1235 ALTENTRY(sys_syscall_int)
1236 SWAPGS /* kernel gsbase */
1237 XPV_TRAP_POP
1238 call smap_enable
1239
1240 nopop_syscall_int:
1241 movq %gs:CPU_THREAD, %r15
1242 movq T_STACK(%r15), %rsp
1243 movl %eax, %eax
1244 /*
1245 * Set t_post_sys on this thread to force ourselves out via the slow
1246 * path. It might be possible at some later date to optimize this out
1247 * and use a faster return mechanism.
1248 */
1249 movb $1, T_POST_SYS(%r15)
1250 CLEAN_CS
1251 jmp _syscall32_save
1252 /*
1253 * There should be no instructions between this label and SWAPGS/IRET
1254 * or we could end up breaking branded zone support. See the usage of
1255 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1256 * for examples.
1257 *
1258 * We want to swapgs to maintain the invariant that all entries into
1259 * tr_iret_user are done on the user gsbase.
1260 */
1261 ALTENTRY(sys_sysint_swapgs_iret)
1262 call x86_md_clear
1263 SWAPGS
1264 jmp tr_iret_user
1265 /*NOTREACHED*/
1266 SET_SIZE(sys_sysint_swapgs_iret)
1267 SET_SIZE(sys_syscall_int)
1268 SET_SIZE(brand_sys_syscall_int)
1269
1270 /*
1271 * Legacy 32-bit applications and old libc implementations do lcalls;
1272 * we should never get here because the LDT entry containing the syscall
1273 * segment descriptor has the "segment present" bit cleared, which means
1274 * we end up processing those system calls in trap() via a not-present trap.
1275 *
1276 * We do it this way because a call gate unhelpfully does -nothing- to the
1277 * interrupt flag bit, so an interrupt can run us just after the lcall
1278 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and
1279 * INTR_POP paths would have to be slightly more complex to dance around
1280 * this problem, and end up depending explicitly on the first
1281 * instruction of this handler being either swapgs or cli.
1282 */
1283
1284 ENTRY_NP(sys_lcall32)
1285 SWAPGS /* kernel gsbase */
1286 pushq $0
1287 pushq %rbp
1288 movq %rsp, %rbp
1289 leaq __lcall_panic_str(%rip), %rdi
1290 xorl %eax, %eax
1291 call panic
1292 SET_SIZE(sys_lcall32)
1293
1294 __lcall_panic_str:
1295 .string "sys_lcall32: shouldn't be here!"
1296
1297 /*
1298 * Declare a uintptr_t which covers the entire pc range of syscall
1299 * handlers for the stack walkers that need this.
1300 */
1301 .align CPTRSIZE
1302 .globl _allsyscalls_size
1303 .type _allsyscalls_size, @object
1304 _allsyscalls_size:
1305 .NWORD . - _allsyscalls
1306 SET_SIZE(_allsyscalls_size)
1307
1308 /*
1309 * These are the thread context handlers for lwps using sysenter/sysexit.
1310 */
1311
1312 /*
1313 * setting this value to zero as we switch away causes the
1314 * stack-pointer-on-sysenter to be NULL, ensuring that we
1315 * don't silently corrupt another (preempted) thread stack
1316 * when running an lwp that (somehow) didn't get sep_restore'd
1317 */
1318 ENTRY_NP(sep_save)
1319 xorl %edx, %edx
1320 xorl %eax, %eax
1321 movl $MSR_INTC_SEP_ESP, %ecx
1322 wrmsr
1323 ret
1324 SET_SIZE(sep_save)
1325
1326 /*
1327 * Update the kernel stack pointer as we resume onto this cpu.
1328 */
1329 ENTRY_NP(sep_restore)
1330 movq %rdi, %rdx
1331 shrq $32, %rdx
1332 movl %edi, %eax
1333 movl $MSR_INTC_SEP_ESP, %ecx
1334 wrmsr
1335 ret
1336 SET_SIZE(sep_restore)
1337
|