8956 Implement KPTI Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com> Reviewed by: Robert Mustacchi <rm@joyent.com>
1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2018 Joyent, Inc. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 */ 26 27 #include <sys/asm_linkage.h> 28 #include <sys/asm_misc.h> 29 #include <sys/regset.h> 30 #include <sys/privregs.h> 31 #include <sys/psw.h> 32 #include <sys/machbrand.h> 33 34 #if defined(__lint) 35 36 #include <sys/types.h> 37 #include <sys/thread.h> 38 #include <sys/systm.h> 39 40 #else /* __lint */ 41 42 #include <sys/segments.h> 43 #include <sys/pcb.h> 44 #include <sys/trap.h> 45 #include <sys/ftrace.h> 46 #include <sys/traptrace.h> 47 #include <sys/clock.h> 48 #include <sys/model.h> 49 #include <sys/panic.h> 50 51 #if defined(__xpv) 52 #include <sys/hypervisor.h> 53 #endif 54 55 #include "assym.h" 56 57 #endif /* __lint */ 58 59 /* 60 * We implement five flavours of system call entry points 61 * 62 * - syscall/sysretq (amd64 generic) 63 * - syscall/sysretl (i386 plus SYSC bit) 64 * - sysenter/sysexit (i386 plus SEP bit) 65 * - int/iret (i386 generic) 66 * - lcall/iret (i386 generic) 67 * 68 * The current libc included in Solaris uses int/iret as the base unoptimized 69 * kernel entry method. Older libc implementations and legacy binaries may use 70 * the lcall call gate, so it must continue to be supported. 71 * 72 * System calls that use an lcall call gate are processed in trap() via a 73 * segment-not-present trap, i.e. lcalls are extremely slow(!). 74 * 75 * The basic pattern used in the 32-bit SYSC handler at this point in time is 76 * to have the bare minimum of assembler, and get to the C handlers as 77 * quickly as possible. 78 * 79 * The 64-bit handler is much closer to the sparcv9 handler; that's 80 * because of passing arguments in registers. The 32-bit world still 81 * passes arguments on the stack -- that makes that handler substantially 82 * more complex. 83 * 84 * The two handlers share a few code fragments which are broken 85 * out into preprocessor macros below. 86 * 87 * XX64 come back and speed all this up later. The 32-bit stuff looks 88 * especially easy to speed up the argument copying part .. 89 * 90 * 91 * Notes about segment register usage (c.f. the 32-bit kernel) 92 * 93 * In the 32-bit kernel, segment registers are dutifully saved and 94 * restored on all mode transitions because the kernel uses them directly. 95 * When the processor is running in 64-bit mode, segment registers are 96 * largely ignored. 97 * 98 * %cs and %ss 99 * controlled by the hardware mechanisms that make mode transitions 100 * 101 * The remaining segment registers have to either be pointing at a valid 102 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors 103 * 104 * %ds and %es 105 * always ignored 106 * 107 * %fs and %gs 108 * fsbase and gsbase are used to control the place they really point at. 109 * The kernel only depends on %gs, and controls its own gsbase via swapgs 110 * 111 * Note that loading segment registers is still costly because the GDT 112 * lookup still happens (this is because the hardware can't know that we're 113 * not setting up these segment registers for a 32-bit program). Thus we 114 * avoid doing this in the syscall path, and defer them to lwp context switch 115 * handlers, so the register values remain virtualized to the lwp. 116 */ 117 118 #if defined(SYSCALLTRACE) 119 #define ORL_SYSCALLTRACE(r32) \ 120 orl syscalltrace(%rip), r32 121 #else 122 #define ORL_SYSCALLTRACE(r32) 123 #endif 124 125 /* 126 * In the 32-bit kernel, we do absolutely nothing before getting into the 127 * brand callback checks. In 64-bit land, we do swapgs and then come here. 128 * We assume that the %rsp- and %r15-stashing fields in the CPU structure 129 * are still unused. 130 * 131 * Check if a brand_mach_ops callback is defined for the specified callback_id 132 * type. If so invoke it with the kernel's %gs value loaded and the following 133 * data on the stack: 134 * 135 * stack: -------------------------------------- 136 * 32 | callback pointer | 137 * | 24 | user (or interrupt) stack pointer | 138 * | 16 | lwp pointer | 139 * v 8 | userland return address | 140 * 0 | callback wrapper return addr | 141 * -------------------------------------- 142 * 143 * Since we're pushing the userland return address onto the kernel stack 144 * we need to get that address without accessing the user's stack (since we 145 * can't trust that data). There are different ways to get the userland 146 * return address depending on how the syscall trap was made: 147 * 148 * a) For sys_syscall and sys_syscall32 the return address is in %rcx. 149 * b) For sys_sysenter the return address is in %rdx. 150 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro, 151 * the stack pointer points at the state saved when we took the interrupt: 152 * ------------------------ 153 * | | user's %ss | 154 * | | user's %esp | 155 * | | EFLAGS register | 156 * v | user's %cs | 157 * | user's %eip | 158 * ------------------------ 159 * 160 * The 2nd parameter to the BRAND_CALLBACK macro is either the 161 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are 162 * used to generate the proper code to get the userland return address for 163 * each syscall entry point. 164 * 165 * The interface to the brand callbacks on the 64-bit kernel assumes %r15 166 * is available as a scratch register within the callback. If the callback 167 * returns within the kernel then this macro will restore %r15. If the 168 * callback is going to return directly to userland then it should restore 169 * %r15 before returning to userland. 170 */ 171 #define BRAND_URET_FROM_REG(rip_reg) \ 172 pushq rip_reg /* push the return address */ 173 174 /* 175 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro 176 * is currently pointing at the user return address (%eip). 177 */ 178 #define BRAND_URET_FROM_INTR_STACK() \ 179 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\ 180 pushq (%r15) /* push the return address */ 181 182 #define BRAND_CALLBACK(callback_id, push_userland_ret) \ 183 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\ 184 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\ 185 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\ 186 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\ 187 subq $16, %rsp /* save space for 2 pointers */ ;\ 188 pushq %r14 /* save %r14 */ ;\ 189 movq %gs:CPU_RTMP_RSP, %r14 ;\ 190 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\ 191 popq %r14 /* restore %r14 */ ;\ 192 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\ 193 pushq %r15 /* push the lwp pointer */ ;\ 194 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\ 195 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\ 196 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\ 197 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\ 198 cmpq $0, %r15 ;\ 199 je 1f ;\ 200 movq %r15, 16(%rsp) /* save the callback pointer */ ;\ 201 push_userland_ret /* push the return address */ ;\ 202 call *24(%rsp) /* call callback */ ;\ 203 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\ 204 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */ 205 206 #define MSTATE_TRANSITION(from, to) \ 207 movl $from, %edi; \ 208 movl $to, %esi; \ 209 call syscall_mstate 210 211 /* 212 * Check to see if a simple (direct) return is possible i.e. 213 * 214 * if (t->t_post_sys_ast | syscalltrace | 215 * lwp->lwp_pcb.pcb_rupdate == 1) 216 * do full version ; 217 * 218 * Preconditions: 219 * - t is curthread 220 * Postconditions: 221 * - condition code NE is set if post-sys is too complex 222 * - rtmp is zeroed if it isn't (we rely on this!) 223 * - ltmp is smashed 224 */ 225 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \ 226 movq T_LWP(t), ltmp; \ 227 movzbl PCB_RUPDATE(ltmp), rtmp; \ 228 ORL_SYSCALLTRACE(rtmp); \ 229 orl T_POST_SYS_AST(t), rtmp; \ 230 cmpl $0, rtmp 231 232 /* 233 * Fix up the lwp, thread, and eflags for a successful return 234 * 235 * Preconditions: 236 * - zwreg contains zero 237 */ 238 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \ 239 movb $LWP_USER, LWP_STATE(lwp); \ 240 movw zwreg, T_SYSNUM(t); \ 241 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp) 242 243 /* 244 * ASSERT(lwptoregs(lwp) == rp); 245 * 246 * This may seem obvious, but very odd things happen if this 247 * assertion is false 248 * 249 * Preconditions: 250 * (%rsp is ready for normal call sequence) 251 * Postconditions (if assertion is true): 252 * %r11 is smashed 253 * 254 * ASSERT(rp->r_cs == descnum) 255 * 256 * The code selector is written into the regs structure when the 257 * lwp stack is created. We use this ASSERT to validate that 258 * the regs structure really matches how we came in. 259 * 260 * Preconditions: 261 * (%rsp is ready for normal call sequence) 262 * Postconditions (if assertion is true): 263 * -none- 264 * 265 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0); 266 * 267 * If this is false, it meant that we returned to userland without 268 * updating the segment registers as we were supposed to. 269 * 270 * Note that we must ensure no interrupts or other traps intervene 271 * between entering privileged mode and performing the assertion, 272 * otherwise we may perform a context switch on the thread, which 273 * will end up setting pcb_rupdate to 1 again. 274 */ 275 #if defined(DEBUG) 276 277 #if !defined(__lint) 278 279 __lwptoregs_msg: 280 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]" 281 282 __codesel_msg: 283 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld" 284 285 __no_rupdate_msg: 286 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0" 287 288 #endif /* !__lint */ 289 290 #define ASSERT_LWPTOREGS(lwp, rp) \ 291 movq LWP_REGS(lwp), %r11; \ 292 cmpq rp, %r11; \ 293 je 7f; \ 294 leaq __lwptoregs_msg(%rip), %rdi; \ 295 movl $__LINE__, %esi; \ 296 movq lwp, %rdx; \ 297 movq %r11, %rcx; \ 298 movq rp, %r8; \ 299 xorl %eax, %eax; \ 300 call panic; \ 301 7: 302 303 #define ASSERT_NO_RUPDATE_PENDING(lwp) \ 304 testb $0x1, PCB_RUPDATE(lwp); \ 305 je 8f; \ 306 movq lwp, %rdx; \ 307 leaq __no_rupdate_msg(%rip), %rdi; \ 308 movl $__LINE__, %esi; \ 309 xorl %eax, %eax; \ 310 call panic; \ 311 8: 312 313 #else 314 #define ASSERT_LWPTOREGS(lwp, rp) 315 #define ASSERT_NO_RUPDATE_PENDING(lwp) 316 #endif 317 318 /* 319 * Do the traptrace thing and restore any registers we used 320 * in situ. Assumes that %rsp is pointing at the base of 321 * the struct regs, obviously .. 322 */ 323 #ifdef TRAPTRACE 324 #define SYSCALL_TRAPTRACE(ttype) \ 325 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \ 326 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \ 327 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \ 328 movq REGOFF_RAX(%rsp), %rax; \ 329 movq REGOFF_RBX(%rsp), %rbx; \ 330 movq REGOFF_RCX(%rsp), %rcx; \ 331 movq REGOFF_RDX(%rsp), %rdx; \ 332 movl %eax, TTR_SYSNUM(%rdi); \ 333 movq REGOFF_RDI(%rsp), %rdi 334 335 #define SYSCALL_TRAPTRACE32(ttype) \ 336 SYSCALL_TRAPTRACE(ttype); \ 337 /* paranoia: clean the top 32-bits of the registers */ \ 338 orl %eax, %eax; \ 339 orl %ebx, %ebx; \ 340 orl %ecx, %ecx; \ 341 orl %edx, %edx; \ 342 orl %edi, %edi 343 #else /* TRAPTRACE */ 344 #define SYSCALL_TRAPTRACE(ttype) 345 #define SYSCALL_TRAPTRACE32(ttype) 346 #endif /* TRAPTRACE */ 347 348 /* 349 * The 64-bit libc syscall wrapper does this: 350 * 351 * fn(<args>) 352 * { 353 * movq %rcx, %r10 -- because syscall smashes %rcx 354 * movl $CODE, %eax 355 * syscall 356 * <error processing> 357 * } 358 * 359 * Thus when we come into the kernel: 360 * 361 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args 362 * %rax is the syscall number 363 * %r12-%r15 contain caller state 364 * 365 * The syscall instruction arranges that: 366 * 367 * %rcx contains the return %rip 368 * %r11d contains bottom 32-bits of %rflags 369 * %rflags is masked (as determined by the SFMASK msr) 370 * %cs is set to UCS_SEL (as determined by the STAR msr) 371 * %ss is set to UDS_SEL (as determined by the STAR msr) 372 * %rip is set to sys_syscall (as determined by the LSTAR msr) 373 * 374 * Or in other words, we have no registers available at all. 375 * Only swapgs can save us! 376 * 377 * Under the hypervisor, the swapgs has happened already. However, the 378 * state of the world is very different from that we're familiar with. 379 * 380 * In particular, we have a stack structure like that for interrupt 381 * gates, except that the %cs and %ss registers are modified for reasons 382 * that are not entirely clear. Critically, the %rcx/%r11 values do 383 * *not* reflect the usage of those registers under a 'real' syscall[1]; 384 * the stack, therefore, looks like this: 385 * 386 * 0x0(rsp) potentially junk %rcx 387 * 0x8(rsp) potentially junk %r11 388 * 0x10(rsp) user %rip 389 * 0x18(rsp) modified %cs 390 * 0x20(rsp) user %rflags 391 * 0x28(rsp) user %rsp 392 * 0x30(rsp) modified %ss 393 * 394 * 395 * and before continuing on, we must load the %rip into %rcx and the 396 * %rflags into %r11. 397 * 398 * [1] They used to, and we relied on it, but this was broken in 3.1.1. 399 * Sigh. 400 */ 401 #if defined(__xpv) 402 #define XPV_SYSCALL_PROD \ 403 movq 0x10(%rsp), %rcx; \ 404 movq 0x20(%rsp), %r11; \ 405 movq 0x28(%rsp), %rsp 406 #else 407 #define XPV_SYSCALL_PROD /* nothing */ 408 #endif 409 410 #if defined(__lint) 411 412 /*ARGSUSED*/ 413 void 414 sys_syscall() 415 {} 416 417 void 418 _allsyscalls() 419 {} 420 421 size_t _allsyscalls_size; 422 423 #else /* __lint */ 424 425 ENTRY_NP2(brand_sys_syscall,_allsyscalls) 426 SWAPGS /* kernel gsbase */ 427 XPV_SYSCALL_PROD 428 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx)) 429 jmp noprod_sys_syscall 430 431 ALTENTRY(sys_syscall) 432 SWAPGS /* kernel gsbase */ 433 XPV_SYSCALL_PROD 434 435 noprod_sys_syscall: 436 movq %r15, %gs:CPU_RTMP_R15 437 movq %rsp, %gs:CPU_RTMP_RSP 438 439 movq %gs:CPU_THREAD, %r15 440 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */ 441 442 ASSERT_UPCALL_MASK_IS_SET 443 444 movl $UCS_SEL, REGOFF_CS(%rsp) 445 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 446 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 447 movl $UDS_SEL, REGOFF_SS(%rsp) 448 449 movl %eax, %eax /* wrapper: sysc# -> %eax */ 450 movq %rdi, REGOFF_RDI(%rsp) 451 movq %rsi, REGOFF_RSI(%rsp) 452 movq %rdx, REGOFF_RDX(%rsp) 453 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */ 454 movq %r10, %rcx /* arg[3] for direct calls */ 455 456 movq %r8, REGOFF_R8(%rsp) 457 movq %r9, REGOFF_R9(%rsp) 458 movq %rax, REGOFF_RAX(%rsp) 459 movq %rbx, REGOFF_RBX(%rsp) 460 461 movq %rbp, REGOFF_RBP(%rsp) 462 movq %r10, REGOFF_R10(%rsp) 463 movq %gs:CPU_RTMP_RSP, %r11 464 movq %r11, REGOFF_RSP(%rsp) 465 movq %r12, REGOFF_R12(%rsp) 466 467 movq %r13, REGOFF_R13(%rsp) 468 movq %r14, REGOFF_R14(%rsp) 469 movq %gs:CPU_RTMP_R15, %r10 470 movq %r10, REGOFF_R15(%rsp) 471 movq $0, REGOFF_SAVFP(%rsp) 472 movq $0, REGOFF_SAVPC(%rsp) 473 474 /* 475 * Copy these registers here in case we end up stopped with 476 * someone (like, say, /proc) messing with our register state. 477 * We don't -restore- them unless we have to in update_sregs. 478 * 479 * Since userland -can't- change fsbase or gsbase directly, 480 * and capturing them involves two serializing instructions, 481 * we don't bother to capture them here. 482 */ 483 xorl %ebx, %ebx 484 movw %ds, %bx 485 movq %rbx, REGOFF_DS(%rsp) 486 movw %es, %bx 487 movq %rbx, REGOFF_ES(%rsp) 488 movw %fs, %bx 489 movq %rbx, REGOFF_FS(%rsp) 490 movw %gs, %bx 491 movq %rbx, REGOFF_GS(%rsp) 492 493 /* 494 * If we're trying to use TRAPTRACE though, I take that back: we're 495 * probably debugging some problem in the SWAPGS logic and want to know 496 * what the incoming gsbase was. 497 * 498 * Since we already did SWAPGS, record the KGSBASE. 499 */ 500 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv) 501 movl $MSR_AMD_KGSBASE, %ecx 502 rdmsr 503 movl %eax, REGOFF_GSBASE(%rsp) 504 movl %edx, REGOFF_GSBASE+4(%rsp) 505 #endif 506 507 /* 508 * Machine state saved in the regs structure on the stack 509 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9 510 * %eax is the syscall number 511 * %rsp is the thread's stack, %r15 is curthread 512 * REG_RSP(%rsp) is the user's stack 513 */ 514 515 SYSCALL_TRAPTRACE($TT_SYSC64) 516 517 movq %rsp, %rbp 518 519 movq T_LWP(%r15), %r14 520 ASSERT_NO_RUPDATE_PENDING(%r14) 521 ENABLE_INTR_FLAGS 522 523 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 524 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 525 526 ASSERT_LWPTOREGS(%r14, %rsp) 527 528 movb $LWP_SYS, LWP_STATE(%r14) 529 incq LWP_RU_SYSC(%r14) 530 movb $NORMALRETURN, LWP_EOSYS(%r14) 531 532 incq %gs:CPU_STATS_SYS_SYSCALL 533 534 movw %ax, T_SYSNUM(%r15) 535 movzbl T_PRE_SYS(%r15), %ebx 536 ORL_SYSCALLTRACE(%ebx) 537 testl %ebx, %ebx 538 jne _syscall_pre 539 540 _syscall_invoke: 541 movq REGOFF_RDI(%rbp), %rdi 542 movq REGOFF_RSI(%rbp), %rsi 543 movq REGOFF_RDX(%rbp), %rdx 544 movq REGOFF_RCX(%rbp), %rcx 545 movq REGOFF_R8(%rbp), %r8 546 movq REGOFF_R9(%rbp), %r9 547 548 cmpl $NSYSCALL, %eax 549 jae _syscall_ill 550 shll $SYSENT_SIZE_SHIFT, %eax 551 leaq sysent(%rax), %rbx 552 553 call *SY_CALLC(%rbx) 554 555 movq %rax, %r12 556 movq %rdx, %r13 557 558 /* 559 * If the handler returns two ints, then we need to split the 560 * 64-bit return value into two 32-bit values. 561 */ 562 testw $SE_32RVAL2, SY_FLAGS(%rbx) 563 je 5f 564 movq %r12, %r13 565 shrq $32, %r13 /* upper 32-bits into %edx */ 566 movl %r12d, %r12d /* lower 32-bits into %eax */ 567 5: 568 /* 569 * Optimistically assume that there's no post-syscall 570 * work to do. (This is to avoid having to call syscall_mstate() 571 * with interrupts disabled) 572 */ 573 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 574 575 /* 576 * We must protect ourselves from being descheduled here; 577 * If we were, and we ended up on another cpu, or another 578 * lwp got in ahead of us, it could change the segment 579 * registers without us noticing before we return to userland. 580 */ 581 CLI(%r14) 582 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 583 jne _syscall_post 584 585 /* 586 * We need to protect ourselves against non-canonical return values 587 * because Intel doesn't check for them on sysret (AMD does). Canonical 588 * addresses on current amd64 processors only use 48-bits for VAs; an 589 * address is canonical if all upper bits (47-63) are identical. If we 590 * find a non-canonical %rip, we opt to go through the full 591 * _syscall_post path which takes us into an iretq which is not 592 * susceptible to the same problems sysret is. 593 * 594 * We're checking for a canonical address by first doing an arithmetic 595 * shift. This will fill in the remaining bits with the value of bit 63. 596 * If the address were canonical, the register would now have either all 597 * zeroes or all ones in it. Therefore we add one (inducing overflow) 598 * and compare against 1. A canonical address will either be zero or one 599 * at this point, hence the use of ja. 600 * 601 * At this point, r12 and r13 have the return value so we can't use 602 * those registers. 603 */ 604 movq REGOFF_RIP(%rsp), %rcx 605 sarq $47, %rcx 606 incq %rcx 607 cmpq $1, %rcx 608 ja _syscall_post 609 610 611 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 612 613 movq %r12, REGOFF_RAX(%rsp) 614 movq %r13, REGOFF_RDX(%rsp) 615 616 /* 617 * To get back to userland, we need the return %rip in %rcx and 618 * the return %rfl in %r11d. The sysretq instruction also arranges 619 * to fix up %cs and %ss; everything else is our responsibility. 620 */ 621 movq REGOFF_RDI(%rsp), %rdi 622 movq REGOFF_RSI(%rsp), %rsi 623 movq REGOFF_RDX(%rsp), %rdx 624 /* %rcx used to restore %rip value */ 625 626 movq REGOFF_R8(%rsp), %r8 627 movq REGOFF_R9(%rsp), %r9 628 movq REGOFF_RAX(%rsp), %rax 629 movq REGOFF_RBX(%rsp), %rbx 630 631 movq REGOFF_RBP(%rsp), %rbp 632 movq REGOFF_R10(%rsp), %r10 633 /* %r11 used to restore %rfl value */ 634 movq REGOFF_R12(%rsp), %r12 635 636 movq REGOFF_R13(%rsp), %r13 637 movq REGOFF_R14(%rsp), %r14 638 movq REGOFF_R15(%rsp), %r15 639 640 movq REGOFF_RIP(%rsp), %rcx 641 movl REGOFF_RFL(%rsp), %r11d 642 643 #if defined(__xpv) 644 addq $REGOFF_RIP, %rsp 645 #else 646 movq REGOFF_RSP(%rsp), %rsp 647 #endif 648 649 /* 650 * There can be no instructions between the ALTENTRY below and 651 * SYSRET or we could end up breaking brand support. See label usage 652 * in sn1_brand_syscall_callback for an example. 653 */ 654 ASSERT_UPCALL_MASK_IS_SET 655 #if defined(__xpv) 656 SYSRETQ 657 ALTENTRY(nopop_sys_syscall_swapgs_sysretq) 658 659 /* 660 * We can only get here after executing a brand syscall 661 * interposition callback handler and simply need to 662 * "sysretq" back to userland. On the hypervisor this 663 * involves the iret hypercall which requires us to construct 664 * just enough of the stack needed for the hypercall. 665 * (rip, cs, rflags, rsp, ss). 666 */ 667 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */ 668 movq %gs:CPU_THREAD, %r11 669 movq T_STACK(%r11), %rsp 670 671 movq %rcx, REGOFF_RIP(%rsp) 672 movl $UCS_SEL, REGOFF_CS(%rsp) 673 movq %gs:CPU_RTMP_RSP, %r11 674 movq %r11, REGOFF_RSP(%rsp) 675 pushfq 676 popq %r11 /* hypercall enables ints */ 677 movq %r11, REGOFF_RFL(%rsp) 678 movl $UDS_SEL, REGOFF_SS(%rsp) 679 addq $REGOFF_RIP, %rsp 680 /* 681 * XXPV: see comment in SYSRETQ definition for future optimization 682 * we could take. 683 */ 684 ASSERT_UPCALL_MASK_IS_SET 685 SYSRETQ 686 #else 687 ALTENTRY(nopop_sys_syscall_swapgs_sysretq) 688 jmp tr_sysretq 689 #endif 690 /*NOTREACHED*/ 691 SET_SIZE(nopop_sys_syscall_swapgs_sysretq) 692 693 _syscall_pre: 694 call pre_syscall 695 movl %eax, %r12d 696 testl %eax, %eax 697 jne _syscall_post_call 698 /* 699 * Didn't abort, so reload the syscall args and invoke the handler. 700 */ 701 movzwl T_SYSNUM(%r15), %eax 702 jmp _syscall_invoke 703 704 _syscall_ill: 705 call nosys 706 movq %rax, %r12 707 movq %rdx, %r13 708 jmp _syscall_post_call 709 710 _syscall_post: 711 STI 712 /* 713 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 714 * so that we can account for the extra work it takes us to finish. 715 */ 716 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 717 _syscall_post_call: 718 movq %r12, %rdi 719 movq %r13, %rsi 720 call post_syscall 721 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 722 jmp _sys_rtt 723 SET_SIZE(sys_syscall) 724 SET_SIZE(brand_sys_syscall) 725 726 #endif /* __lint */ 727 728 #if defined(__lint) 729 730 /*ARGSUSED*/ 731 void 732 sys_syscall32() 733 {} 734 735 #else /* __lint */ 736 737 ENTRY_NP(brand_sys_syscall32) 738 SWAPGS /* kernel gsbase */ 739 XPV_TRAP_POP 740 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx)) 741 jmp nopop_sys_syscall32 742 743 ALTENTRY(sys_syscall32) 744 SWAPGS /* kernel gsbase */ 745 XPV_TRAP_POP 746 747 nopop_sys_syscall32: 748 movl %esp, %r10d 749 movq %gs:CPU_THREAD, %r15 750 movq T_STACK(%r15), %rsp 751 movl %eax, %eax 752 753 movl $U32CS_SEL, REGOFF_CS(%rsp) 754 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 755 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 756 movq %r10, REGOFF_RSP(%rsp) 757 movl $UDS_SEL, REGOFF_SS(%rsp) 758 759 _syscall32_save: 760 movl %edi, REGOFF_RDI(%rsp) 761 movl %esi, REGOFF_RSI(%rsp) 762 movl %ebp, REGOFF_RBP(%rsp) 763 movl %ebx, REGOFF_RBX(%rsp) 764 movl %edx, REGOFF_RDX(%rsp) 765 movl %ecx, REGOFF_RCX(%rsp) 766 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 767 movq $0, REGOFF_SAVFP(%rsp) 768 movq $0, REGOFF_SAVPC(%rsp) 769 770 /* 771 * Copy these registers here in case we end up stopped with 772 * someone (like, say, /proc) messing with our register state. 773 * We don't -restore- them unless we have to in update_sregs. 774 * 775 * Since userland -can't- change fsbase or gsbase directly, 776 * we don't bother to capture them here. 777 */ 778 xorl %ebx, %ebx 779 movw %ds, %bx 780 movq %rbx, REGOFF_DS(%rsp) 781 movw %es, %bx 782 movq %rbx, REGOFF_ES(%rsp) 783 movw %fs, %bx 784 movq %rbx, REGOFF_FS(%rsp) 785 movw %gs, %bx 786 movq %rbx, REGOFF_GS(%rsp) 787 788 /* 789 * If we're trying to use TRAPTRACE though, I take that back: we're 790 * probably debugging some problem in the SWAPGS logic and want to know 791 * what the incoming gsbase was. 792 * 793 * Since we already did SWAPGS, record the KGSBASE. 794 */ 795 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv) 796 movl $MSR_AMD_KGSBASE, %ecx 797 rdmsr 798 movl %eax, REGOFF_GSBASE(%rsp) 799 movl %edx, REGOFF_GSBASE+4(%rsp) 800 #endif 801 802 /* 803 * Application state saved in the regs structure on the stack 804 * %eax is the syscall number 805 * %rsp is the thread's stack, %r15 is curthread 806 * REG_RSP(%rsp) is the user's stack 807 */ 808 809 SYSCALL_TRAPTRACE32($TT_SYSC) 810 811 movq %rsp, %rbp 812 813 movq T_LWP(%r15), %r14 814 ASSERT_NO_RUPDATE_PENDING(%r14) 815 816 ENABLE_INTR_FLAGS 817 818 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 819 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 820 821 ASSERT_LWPTOREGS(%r14, %rsp) 822 823 incq %gs:CPU_STATS_SYS_SYSCALL 824 825 /* 826 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed 827 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or 828 * more succinctly: 829 * 830 * SA(MAXSYSARGS * sizeof (long)) == 64 831 */ 832 #define SYS_DROP 64 /* drop for args */ 833 subq $SYS_DROP, %rsp 834 movb $LWP_SYS, LWP_STATE(%r14) 835 movq %r15, %rdi 836 movq %rsp, %rsi 837 call syscall_entry 838 839 /* 840 * Fetch the arguments copied onto the kernel stack and put 841 * them in the right registers to invoke a C-style syscall handler. 842 * %rax contains the handler address. 843 * 844 * Ideas for making all this go faster of course include simply 845 * forcibly fetching 6 arguments from the user stack under lofault 846 * protection, reverting to copyin_args only when watchpoints 847 * are in effect. 848 * 849 * (If we do this, make sure that exec and libthread leave 850 * enough space at the top of the stack to ensure that we'll 851 * never do a fetch from an invalid page.) 852 * 853 * Lots of ideas here, but they won't really help with bringup B-) 854 * Correctness can't wait, performance can wait a little longer .. 855 */ 856 857 movq %rax, %rbx 858 movl 0(%rsp), %edi 859 movl 8(%rsp), %esi 860 movl 0x10(%rsp), %edx 861 movl 0x18(%rsp), %ecx 862 movl 0x20(%rsp), %r8d 863 movl 0x28(%rsp), %r9d 864 865 call *SY_CALLC(%rbx) 866 867 movq %rbp, %rsp /* pop the args */ 868 869 /* 870 * amd64 syscall handlers -always- return a 64-bit value in %rax. 871 * On the 32-bit kernel, they always return that value in %eax:%edx 872 * as required by the 32-bit ABI. 873 * 874 * Simulate the same behaviour by unconditionally splitting the 875 * return value in the same way. 876 */ 877 movq %rax, %r13 878 shrq $32, %r13 /* upper 32-bits into %edx */ 879 movl %eax, %r12d /* lower 32-bits into %eax */ 880 881 /* 882 * Optimistically assume that there's no post-syscall 883 * work to do. (This is to avoid having to call syscall_mstate() 884 * with interrupts disabled) 885 */ 886 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 887 888 /* 889 * We must protect ourselves from being descheduled here; 890 * If we were, and we ended up on another cpu, or another 891 * lwp got in ahead of us, it could change the segment 892 * registers without us noticing before we return to userland. 893 */ 894 CLI(%r14) 895 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 896 jne _full_syscall_postsys32 897 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 898 899 /* 900 * To get back to userland, we need to put the return %rip in %rcx and 901 * the return %rfl in %r11d. The sysret instruction also arranges 902 * to fix up %cs and %ss; everything else is our responsibility. 903 */ 904 905 movl %r12d, %eax /* %eax: rval1 */ 906 movl REGOFF_RBX(%rsp), %ebx 907 /* %ecx used for return pointer */ 908 movl %r13d, %edx /* %edx: rval2 */ 909 movl REGOFF_RBP(%rsp), %ebp 910 movl REGOFF_RSI(%rsp), %esi 911 movl REGOFF_RDI(%rsp), %edi 912 913 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */ 914 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */ 915 movl REGOFF_RSP(%rsp), %esp 916 917 ASSERT_UPCALL_MASK_IS_SET 918 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl) 919 jmp tr_sysretl 920 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl) 921 /*NOTREACHED*/ 922 923 _full_syscall_postsys32: 924 STI 925 /* 926 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 927 * so that we can account for the extra work it takes us to finish. 928 */ 929 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 930 movq %r15, %rdi 931 movq %r12, %rsi /* rval1 - %eax */ 932 movq %r13, %rdx /* rval2 - %edx */ 933 call syscall_exit 934 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 935 jmp _sys_rtt 936 SET_SIZE(sys_syscall32) 937 SET_SIZE(brand_sys_syscall32) 938 939 #endif /* __lint */ 940 941 /* 942 * System call handler via the sysenter instruction 943 * Used only for 32-bit system calls on the 64-bit kernel. 944 * 945 * The caller in userland has arranged that: 946 * 947 * - %eax contains the syscall number 948 * - %ecx contains the user %esp 949 * - %edx contains the return %eip 950 * - the user stack contains the args to the syscall 951 * 952 * Hardware and (privileged) initialization code have arranged that by 953 * the time the sysenter instructions completes: 954 * 955 * - %rip is pointing to sys_sysenter (below). 956 * - %cs and %ss are set to kernel text and stack (data) selectors. 957 * - %rsp is pointing at the lwp's stack 958 * - interrupts have been disabled. 959 * 960 * Note that we are unable to return both "rvals" to userland with 961 * this call, as %edx is used by the sysexit instruction. 962 * 963 * One final complication in this routine is its interaction with 964 * single-stepping in a debugger. For most of the system call mechanisms, the 965 * CPU automatically clears the single-step flag before we enter the kernel. 966 * The sysenter mechanism does not clear the flag, so a user single-stepping 967 * through a libc routine may suddenly find themself single-stepping through the 968 * kernel. To detect this, kmdb and trap() both compare the trap %pc to the 969 * [brand_]sys_enter addresses on each single-step trap. If it finds that we 970 * have single-stepped to a sysenter entry point, it explicitly clears the flag 971 * and executes the sys_sysenter routine. 972 * 973 * One final complication in this final complication is the fact that we have 974 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter. 975 * If we enter at brand_sys_sysenter and start single-stepping through the 976 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter. 977 * kmdb cannot distinguish between that valid single-step and the undesirable 978 * one mentioned above. To avoid this situation, we simply add a jump over the 979 * instruction at sys_sysenter to make it impossible to single-step to it. 980 */ 981 #if defined(__lint) 982 983 void 984 sys_sysenter() 985 {} 986 987 #else /* __lint */ 988 989 ENTRY_NP(brand_sys_sysenter) 990 SWAPGS /* kernel gsbase */ 991 ALTENTRY(_brand_sys_sysenter_post_swapgs) 992 993 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx)) 994 /* 995 * Jump over sys_sysenter to allow single-stepping as described 996 * above. 997 */ 998 jmp _sys_sysenter_post_swapgs 999 1000 ALTENTRY(sys_sysenter) 1001 SWAPGS /* kernel gsbase */ 1002 ALTENTRY(_sys_sysenter_post_swapgs) 1003 1004 movq %gs:CPU_THREAD, %r15 1005 1006 movl $U32CS_SEL, REGOFF_CS(%rsp) 1007 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */ 1008 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */ 1009 /* 1010 * NOTE: none of the instructions that run before we get here should 1011 * clobber bits in (R)FLAGS! This includes the kpti trampoline. 1012 */ 1013 pushfq 1014 popq %r10 1015 movl $UDS_SEL, REGOFF_SS(%rsp) 1016 1017 /* 1018 * Set the interrupt flag before storing the flags to the 1019 * flags image on the stack so we can return to user with 1020 * interrupts enabled if we return via sys_rtt_syscall32 1021 */ 1022 orq $PS_IE, %r10 1023 movq %r10, REGOFF_RFL(%rsp) 1024 1025 movl %edi, REGOFF_RDI(%rsp) 1026 movl %esi, REGOFF_RSI(%rsp) 1027 movl %ebp, REGOFF_RBP(%rsp) 1028 movl %ebx, REGOFF_RBX(%rsp) 1029 movl %edx, REGOFF_RDX(%rsp) 1030 movl %ecx, REGOFF_RCX(%rsp) 1031 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 1032 movq $0, REGOFF_SAVFP(%rsp) 1033 movq $0, REGOFF_SAVPC(%rsp) 1034 1035 /* 1036 * Copy these registers here in case we end up stopped with 1037 * someone (like, say, /proc) messing with our register state. 1038 * We don't -restore- them unless we have to in update_sregs. 1039 * 1040 * Since userland -can't- change fsbase or gsbase directly, 1041 * we don't bother to capture them here. 1042 */ 1043 xorl %ebx, %ebx 1044 movw %ds, %bx 1045 movq %rbx, REGOFF_DS(%rsp) 1046 movw %es, %bx 1047 movq %rbx, REGOFF_ES(%rsp) 1048 movw %fs, %bx 1049 movq %rbx, REGOFF_FS(%rsp) 1050 movw %gs, %bx 1051 movq %rbx, REGOFF_GS(%rsp) 1052 1053 /* 1054 * If we're trying to use TRAPTRACE though, I take that back: we're 1055 * probably debugging some problem in the SWAPGS logic and want to know 1056 * what the incoming gsbase was. 1057 * 1058 * Since we already did SWAPGS, record the KGSBASE. 1059 */ 1060 #if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv) 1061 movl $MSR_AMD_KGSBASE, %ecx 1062 rdmsr 1063 movl %eax, REGOFF_GSBASE(%rsp) 1064 movl %edx, REGOFF_GSBASE+4(%rsp) 1065 #endif 1066 1067 /* 1068 * Application state saved in the regs structure on the stack 1069 * %eax is the syscall number 1070 * %rsp is the thread's stack, %r15 is curthread 1071 * REG_RSP(%rsp) is the user's stack 1072 */ 1073 1074 SYSCALL_TRAPTRACE($TT_SYSENTER) 1075 1076 movq %rsp, %rbp 1077 1078 movq T_LWP(%r15), %r14 1079 ASSERT_NO_RUPDATE_PENDING(%r14) 1080 1081 ENABLE_INTR_FLAGS 1082 1083 /* 1084 * Catch 64-bit process trying to issue sysenter instruction 1085 * on Nocona based systems. 1086 */ 1087 movq LWP_PROCP(%r14), %rax 1088 cmpq $DATAMODEL_ILP32, P_MODEL(%rax) 1089 je 7f 1090 1091 /* 1092 * For a non-32-bit process, simulate a #ud, since that's what 1093 * native hardware does. The traptrace entry (above) will 1094 * let you know what really happened. 1095 */ 1096 movq $T_ILLINST, REGOFF_TRAPNO(%rsp) 1097 movq REGOFF_CS(%rsp), %rdi 1098 movq %rdi, REGOFF_ERR(%rsp) 1099 movq %rsp, %rdi 1100 movq REGOFF_RIP(%rsp), %rsi 1101 movl %gs:CPU_ID, %edx 1102 call trap 1103 jmp _sys_rtt 1104 7: 1105 1106 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 1107 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */ 1108 1109 ASSERT_LWPTOREGS(%r14, %rsp) 1110 1111 incq %gs:CPU_STATS_SYS_SYSCALL 1112 1113 /* 1114 * Make some space for MAXSYSARGS (currently 8) 32-bit args 1115 * placed into 64-bit (long) arg slots, plus one 64-bit 1116 * (long) arg count, maintaining 16 byte alignment. 1117 */ 1118 subq $SYS_DROP, %rsp 1119 movb $LWP_SYS, LWP_STATE(%r14) 1120 movq %r15, %rdi 1121 movq %rsp, %rsi 1122 call syscall_entry 1123 1124 /* 1125 * Fetch the arguments copied onto the kernel stack and put 1126 * them in the right registers to invoke a C-style syscall handler. 1127 * %rax contains the handler address. 1128 */ 1129 movq %rax, %rbx 1130 movl 0(%rsp), %edi 1131 movl 8(%rsp), %esi 1132 movl 0x10(%rsp), %edx 1133 movl 0x18(%rsp), %ecx 1134 movl 0x20(%rsp), %r8d 1135 movl 0x28(%rsp), %r9d 1136 1137 call *SY_CALLC(%rbx) 1138 1139 movq %rbp, %rsp /* pop the args */ 1140 1141 /* 1142 * amd64 syscall handlers -always- return a 64-bit value in %rax. 1143 * On the 32-bit kernel, the always return that value in %eax:%edx 1144 * as required by the 32-bit ABI. 1145 * 1146 * Simulate the same behaviour by unconditionally splitting the 1147 * return value in the same way. 1148 */ 1149 movq %rax, %r13 1150 shrq $32, %r13 /* upper 32-bits into %edx */ 1151 movl %eax, %r12d /* lower 32-bits into %eax */ 1152 1153 /* 1154 * Optimistically assume that there's no post-syscall 1155 * work to do. (This is to avoid having to call syscall_mstate() 1156 * with interrupts disabled) 1157 */ 1158 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 1159 1160 /* 1161 * We must protect ourselves from being descheduled here; 1162 * If we were, and we ended up on another cpu, or another 1163 * lwp got int ahead of us, it could change the segment 1164 * registers without us noticing before we return to userland. 1165 * 1166 * This cli is undone in the tr_sysexit trampoline code. 1167 */ 1168 cli 1169 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 1170 jne _full_syscall_postsys32 1171 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 1172 1173 /* 1174 * To get back to userland, load up the 32-bit registers and 1175 * sysexit back where we came from. 1176 */ 1177 1178 /* 1179 * Interrupts will be turned on by the 'sti' executed just before 1180 * sysexit. The following ensures that restoring the user's rflags 1181 * doesn't enable interrupts too soon. 1182 */ 1183 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp) 1184 1185 /* 1186 * (There's no point in loading up %edx because the sysexit 1187 * mechanism smashes it.) 1188 */ 1189 movl %r12d, %eax 1190 movl REGOFF_RBX(%rsp), %ebx 1191 movl REGOFF_RBP(%rsp), %ebp 1192 movl REGOFF_RSI(%rsp), %esi 1193 movl REGOFF_RDI(%rsp), %edi 1194 1195 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */ 1196 pushq REGOFF_RFL(%rsp) 1197 popfq 1198 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */ 1199 ALTENTRY(sys_sysenter_swapgs_sysexit) 1200 jmp tr_sysexit 1201 SET_SIZE(sys_sysenter_swapgs_sysexit) 1202 SET_SIZE(sys_sysenter) 1203 SET_SIZE(_sys_sysenter_post_swapgs) 1204 SET_SIZE(brand_sys_sysenter) 1205 1206 #endif /* __lint */ 1207 1208 /* 1209 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by 1210 * the generic i386 libc to do system calls. We do a small amount of setup 1211 * before jumping into the existing sys_syscall32 path. 1212 */ 1213 #if defined(__lint) 1214 1215 /*ARGSUSED*/ 1216 void 1217 sys_syscall_int() 1218 {} 1219 1220 #else /* __lint */ 1221 1222 ENTRY_NP(brand_sys_syscall_int) 1223 SWAPGS /* kernel gsbase */ 1224 XPV_TRAP_POP 1225 call smap_enable 1226 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK()) 1227 jmp nopop_syscall_int 1228 1229 ALTENTRY(sys_syscall_int) 1230 SWAPGS /* kernel gsbase */ 1231 XPV_TRAP_POP 1232 call smap_enable 1233 1234 nopop_syscall_int: 1235 movq %gs:CPU_THREAD, %r15 1236 movq T_STACK(%r15), %rsp 1237 movl %eax, %eax 1238 /* 1239 * Set t_post_sys on this thread to force ourselves out via the slow 1240 * path. It might be possible at some later date to optimize this out 1241 * and use a faster return mechanism. 1242 */ 1243 movb $1, T_POST_SYS(%r15) 1244 CLEAN_CS 1245 jmp _syscall32_save 1246 /* 1247 * There should be no instructions between this label and SWAPGS/IRET 1248 * or we could end up breaking branded zone support. See the usage of 1249 * this label in lx_brand_int80_callback and sn1_brand_int91_callback 1250 * for examples. 1251 * 1252 * We want to swapgs to maintain the invariant that all entries into 1253 * tr_iret_user are done on the user gsbase. 1254 */ 1255 ALTENTRY(sys_sysint_swapgs_iret) 1256 SWAPGS 1257 jmp tr_iret_user 1258 /*NOTREACHED*/ 1259 SET_SIZE(sys_sysint_swapgs_iret) 1260 SET_SIZE(sys_syscall_int) 1261 SET_SIZE(brand_sys_syscall_int) 1262 1263 #endif /* __lint */ 1264 1265 /* 1266 * Legacy 32-bit applications and old libc implementations do lcalls; 1267 * we should never get here because the LDT entry containing the syscall 1268 * segment descriptor has the "segment present" bit cleared, which means 1269 * we end up processing those system calls in trap() via a not-present trap. 1270 * 1271 * We do it this way because a call gate unhelpfully does -nothing- to the 1272 * interrupt flag bit, so an interrupt can run us just after the lcall 1273 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and 1274 * INTR_POP paths would have to be slightly more complex to dance around 1275 * this problem, and end up depending explicitly on the first 1276 * instruction of this handler being either swapgs or cli. 1277 */ 1278 1279 #if defined(__lint) 1280 1281 /*ARGSUSED*/ 1282 void 1283 sys_lcall32() 1284 {} 1285 1286 #else /* __lint */ 1287 1288 ENTRY_NP(sys_lcall32) 1289 SWAPGS /* kernel gsbase */ 1290 pushq $0 1291 pushq %rbp 1292 movq %rsp, %rbp 1293 leaq __lcall_panic_str(%rip), %rdi 1294 xorl %eax, %eax 1295 call panic 1296 SET_SIZE(sys_lcall32) 1297 1298 __lcall_panic_str: 1299 .string "sys_lcall32: shouldn't be here!" 1300 1301 /* 1302 * Declare a uintptr_t which covers the entire pc range of syscall 1303 * handlers for the stack walkers that need this. 1304 */ 1305 .align CPTRSIZE 1306 .globl _allsyscalls_size 1307 .type _allsyscalls_size, @object 1308 _allsyscalls_size: 1309 .NWORD . - _allsyscalls 1310 SET_SIZE(_allsyscalls_size) 1311 1312 #endif /* __lint */ 1313 1314 /* 1315 * These are the thread context handlers for lwps using sysenter/sysexit. 1316 */ 1317 1318 #if defined(__lint) 1319 1320 /*ARGSUSED*/ 1321 void 1322 sep_save(void *ksp) 1323 {} 1324 1325 /*ARGSUSED*/ 1326 void 1327 sep_restore(void *ksp) 1328 {} 1329 1330 #else /* __lint */ 1331 1332 /* 1333 * setting this value to zero as we switch away causes the 1334 * stack-pointer-on-sysenter to be NULL, ensuring that we 1335 * don't silently corrupt another (preempted) thread stack 1336 * when running an lwp that (somehow) didn't get sep_restore'd 1337 */ 1338 ENTRY_NP(sep_save) 1339 xorl %edx, %edx 1340 xorl %eax, %eax 1341 movl $MSR_INTC_SEP_ESP, %ecx 1342 wrmsr 1343 ret 1344 SET_SIZE(sep_save) 1345 1346 /* 1347 * Update the kernel stack pointer as we resume onto this cpu. 1348 */ 1349 ENTRY_NP(sep_restore) 1350 movq %rdi, %rdx 1351 shrq $32, %rdx 1352 movl %edi, %eax 1353 movl $MSR_INTC_SEP_ESP, %ecx 1354 wrmsr 1355 ret 1356 SET_SIZE(sep_restore) 1357 1358 #endif /* __lint */ --- EOF ---