1 /* 2 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2013, 2014 by Delphix. All rights reserved. 4 * Copyright 2019 Joyent, Inc. 5 */ 6 7 /* 8 * Copyright (c) 1989, 1990 William F. Jolitz. 9 * Copyright (c) 1990 The Regents of the University of California. 10 * All rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * $FreeBSD: src/sys/amd64/amd64/exception.S,v 1.113 2003/10/15 02:04:52 peter Exp $ 41 */ 42 43 #include <sys/asm_linkage.h> 44 #include <sys/asm_misc.h> 45 #include <sys/trap.h> 46 #include <sys/psw.h> 47 #include <sys/regset.h> 48 #include <sys/privregs.h> 49 #include <sys/dtrace.h> 50 #include <sys/x86_archext.h> 51 #include <sys/traptrace.h> 52 #include <sys/machparam.h> 53 54 #if !defined(__lint) 55 56 #include "assym.h" 57 58 /* 59 * push $0 on stack for traps that do not 60 * generate an error code. This is so the rest 61 * of the kernel can expect a consistent stack 62 * from from any exception. 63 * 64 * Note that for all exceptions for amd64 65 * %r11 and %rcx are on the stack. Just pop 66 * them back into their appropriate registers and let 67 * it get saved as is running native. 68 */ 69 70 #if defined(__xpv) && defined(__amd64) 71 72 #define NPTRAP_NOERR(trapno) \ 73 pushq $0; \ 74 pushq $trapno 75 76 #define TRAP_NOERR(trapno) \ 77 XPV_TRAP_POP; \ 78 NPTRAP_NOERR(trapno) 79 80 /* 81 * error code already pushed by hw 82 * onto stack. 83 */ 84 #define TRAP_ERR(trapno) \ 85 XPV_TRAP_POP; \ 86 pushq $trapno 87 88 #else /* __xpv && __amd64 */ 89 90 #define TRAP_NOERR(trapno) \ 91 push $0; \ 92 push $trapno 93 94 #define NPTRAP_NOERR(trapno) TRAP_NOERR(trapno) 95 96 /* 97 * error code already pushed by hw 98 * onto stack. 99 */ 100 #define TRAP_ERR(trapno) \ 101 push $trapno 102 103 #endif /* __xpv && __amd64 */ 104 105 /* 106 * These are the stacks used on cpu0 for taking double faults, 107 * NMIs and MCEs (the latter two only on amd64 where we have IST). 108 * 109 * We define them here instead of in a C file so that we can page-align 110 * them (gcc won't do that in a .c file). 111 */ 112 .data 113 DGDEF3(dblfault_stack0, DEFAULTSTKSZ, MMU_PAGESIZE) 114 .fill DEFAULTSTKSZ, 1, 0 115 DGDEF3(nmi_stack0, DEFAULTSTKSZ, MMU_PAGESIZE) 116 .fill DEFAULTSTKSZ, 1, 0 117 DGDEF3(mce_stack0, DEFAULTSTKSZ, MMU_PAGESIZE) 118 .fill DEFAULTSTKSZ, 1, 0 119 120 /* 121 * #DE 122 */ 123 ENTRY_NP(div0trap) 124 TRAP_NOERR(T_ZERODIV) /* $0 */ 125 jmp cmntrap 126 SET_SIZE(div0trap) 127 128 /* 129 * #DB 130 * 131 * Fetch %dr6 and clear it, handing off the value to the 132 * cmntrap code in %r15/%esi 133 */ 134 ENTRY_NP(dbgtrap) 135 TRAP_NOERR(T_SGLSTP) /* $1 */ 136 137 #if defined(__amd64) 138 #if !defined(__xpv) /* no sysenter support yet */ 139 /* 140 * If we get here as a result of single-stepping a sysenter 141 * instruction, we suddenly find ourselves taking a #db 142 * in kernel mode -before- we've swapgs'ed. So before we can 143 * take the trap, we do the swapgs here, and fix the return 144 * %rip in trap() so that we return immediately after the 145 * swapgs in the sysenter handler to avoid doing the swapgs again. 146 * 147 * Nobody said that the design of sysenter was particularly 148 * elegant, did they? 149 */ 150 151 pushq %r11 152 153 /* 154 * At this point the stack looks like this: 155 * 156 * (high address) r_ss 157 * r_rsp 158 * r_rfl 159 * r_cs 160 * r_rip <-- %rsp + 24 161 * r_err <-- %rsp + 16 162 * r_trapno <-- %rsp + 8 163 * (low address) %r11 <-- %rsp 164 */ 165 leaq sys_sysenter(%rip), %r11 166 cmpq %r11, 24(%rsp) /* Compare to saved r_rip on the stack */ 167 je 1f 168 leaq brand_sys_sysenter(%rip), %r11 169 cmpq %r11, 24(%rsp) /* Compare to saved r_rip on the stack */ 170 je 1f 171 leaq tr_sys_sysenter(%rip), %r11 172 cmpq %r11, 24(%rsp) 173 je 1f 174 leaq tr_brand_sys_sysenter(%rip), %r11 175 cmpq %r11, 24(%rsp) 176 jne 2f 177 1: SWAPGS 178 2: popq %r11 179 #endif /* !__xpv */ 180 181 INTR_PUSH 182 #if defined(__xpv) 183 movl $6, %edi 184 call kdi_dreg_get 185 movq %rax, %r15 /* %db6 -> %r15 */ 186 movl $6, %edi 187 movl $0, %esi 188 call kdi_dreg_set /* 0 -> %db6 */ 189 #else 190 movq %db6, %r15 191 xorl %eax, %eax 192 movq %rax, %db6 193 #endif 194 195 #elif defined(__i386) 196 197 INTR_PUSH 198 #if defined(__xpv) 199 pushl $6 200 call kdi_dreg_get 201 addl $4, %esp 202 movl %eax, %esi /* %dr6 -> %esi */ 203 pushl $0 204 pushl $6 205 call kdi_dreg_set /* 0 -> %dr6 */ 206 addl $8, %esp 207 #else 208 movl %db6, %esi 209 xorl %eax, %eax 210 movl %eax, %db6 211 #endif 212 #endif /* __i386 */ 213 214 jmp cmntrap_pushed 215 SET_SIZE(dbgtrap) 216 217 #if defined(__amd64) 218 #if !defined(__xpv) 219 220 /* 221 * Macro to set the gsbase or kgsbase to the address of the struct cpu 222 * for this processor. If we came from userland, set kgsbase else 223 * set gsbase. We find the proper cpu struct by looping through 224 * the cpu structs for all processors till we find a match for the gdt 225 * of the trapping processor. The stack is expected to be pointing at 226 * the standard regs pushed by hardware on a trap (plus error code and trapno). 227 * 228 * It's ok for us to clobber gsbase here (and possibly end up with both gsbase 229 * and kgsbase set to the same value) because we're not going back the normal 230 * way out of here (via IRET). Where we're going, we don't need no user %gs. 231 */ 232 #define SET_CPU_GSBASE \ 233 subq $REGOFF_TRAPNO, %rsp; /* save regs */ \ 234 movq %rax, REGOFF_RAX(%rsp); \ 235 movq %rbx, REGOFF_RBX(%rsp); \ 236 movq %rcx, REGOFF_RCX(%rsp); \ 237 movq %rdx, REGOFF_RDX(%rsp); \ 238 movq %rbp, REGOFF_RBP(%rsp); \ 239 movq %rsp, %rbp; \ 240 subq $16, %rsp; /* space for gdt */ \ 241 sgdt 6(%rsp); \ 242 movq 8(%rsp), %rcx; /* %rcx has gdt to match */ \ 243 xorl %ebx, %ebx; /* loop index */ \ 244 leaq cpu(%rip), %rdx; /* cpu pointer array */ \ 245 1: \ 246 movq (%rdx, %rbx, CLONGSIZE), %rax; /* get cpu[i] */ \ 247 cmpq $0x0, %rax; /* cpu[i] == NULL ? */ \ 248 je 2f; /* yes, continue */ \ 249 cmpq %rcx, CPU_GDT(%rax); /* gdt == cpu[i]->cpu_gdt ? */ \ 250 je 3f; /* yes, go set gsbase */ \ 251 2: \ 252 incl %ebx; /* i++ */ \ 253 cmpl $NCPU, %ebx; /* i < NCPU ? */ \ 254 jb 1b; /* yes, loop */ \ 255 /* XXX BIG trouble if we fall thru here. We didn't find a gdt match */ \ 256 3: \ 257 movl $MSR_AMD_KGSBASE, %ecx; \ 258 cmpw $KCS_SEL, REGOFF_CS(%rbp); /* trap from kernel? */ \ 259 jne 4f; /* no, go set KGSBASE */ \ 260 movl $MSR_AMD_GSBASE, %ecx; /* yes, set GSBASE */ \ 261 mfence; /* OPTERON_ERRATUM_88 */ \ 262 4: \ 263 movq %rax, %rdx; /* write base register */ \ 264 shrq $32, %rdx; \ 265 wrmsr; \ 266 movq REGOFF_RDX(%rbp), %rdx; /* restore regs */ \ 267 movq REGOFF_RCX(%rbp), %rcx; \ 268 movq REGOFF_RBX(%rbp), %rbx; \ 269 movq REGOFF_RAX(%rbp), %rax; \ 270 movq %rbp, %rsp; \ 271 movq REGOFF_RBP(%rsp), %rbp; \ 272 addq $REGOFF_TRAPNO, %rsp /* pop stack */ 273 274 #else /* __xpv */ 275 276 #define SET_CPU_GSBASE /* noop on the hypervisor */ 277 278 #endif /* __xpv */ 279 #endif /* __amd64 */ 280 281 282 #if defined(__amd64) 283 284 /* 285 * #NMI 286 * 287 * XXPV: See 6532669. 288 */ 289 ENTRY_NP(nmiint) 290 TRAP_NOERR(T_NMIFLT) /* $2 */ 291 292 SET_CPU_GSBASE 293 294 /* 295 * Save all registers and setup segment registers 296 * with kernel selectors. 297 */ 298 INTR_PUSH 299 INTGATE_INIT_KERNEL_FLAGS 300 301 TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_TRAP) 302 TRACE_REGS(%r12, %rsp, %rax, %rbx) 303 TRACE_STAMP(%r12) 304 305 movq %rsp, %rbp 306 307 movq %rbp, %rdi 308 call av_dispatch_nmivect 309 310 INTR_POP 311 call x86_md_clear 312 jmp tr_iret_auto 313 /*NOTREACHED*/ 314 SET_SIZE(nmiint) 315 316 #elif defined(__i386) 317 318 /* 319 * #NMI 320 */ 321 ENTRY_NP(nmiint) 322 TRAP_NOERR(T_NMIFLT) /* $2 */ 323 324 /* 325 * Save all registers and setup segment registers 326 * with kernel selectors. 327 */ 328 INTR_PUSH 329 INTGATE_INIT_KERNEL_FLAGS 330 331 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) 332 TRACE_REGS(%edi, %esp, %ebx, %ecx) 333 TRACE_STAMP(%edi) 334 335 movl %esp, %ebp 336 337 pushl %ebp 338 call av_dispatch_nmivect 339 addl $4, %esp 340 341 INTR_POP_USER 342 IRET 343 SET_SIZE(nmiint) 344 345 #endif /* __i386 */ 346 347 /* 348 * #BP 349 */ 350 ENTRY_NP(brktrap) 351 352 #if defined(__amd64) 353 XPV_TRAP_POP 354 cmpw $KCS_SEL, 8(%rsp) 355 jne bp_user 356 357 /* 358 * This is a breakpoint in the kernel -- it is very likely that this 359 * is DTrace-induced. To unify DTrace handling, we spoof this as an 360 * invalid opcode (#UD) fault. Note that #BP is a trap, not a fault -- 361 * we must decrement the trapping %rip to make it appear as a fault. 362 * We then push a non-zero error code to indicate that this is coming 363 * from #BP. 364 */ 365 decq (%rsp) 366 push $1 /* error code -- non-zero for #BP */ 367 jmp ud_kernel 368 369 bp_user: 370 #endif /* __amd64 */ 371 372 NPTRAP_NOERR(T_BPTFLT) /* $3 */ 373 jmp dtrace_trap 374 375 SET_SIZE(brktrap) 376 377 /* 378 * #OF 379 */ 380 ENTRY_NP(ovflotrap) 381 TRAP_NOERR(T_OVFLW) /* $4 */ 382 jmp cmntrap 383 SET_SIZE(ovflotrap) 384 385 /* 386 * #BR 387 */ 388 ENTRY_NP(boundstrap) 389 TRAP_NOERR(T_BOUNDFLT) /* $5 */ 390 jmp cmntrap 391 SET_SIZE(boundstrap) 392 393 #if defined(__amd64) 394 395 ENTRY_NP(invoptrap) 396 397 XPV_TRAP_POP 398 399 cmpw $KCS_SEL, 8(%rsp) 400 jne ud_user 401 402 #if defined(__xpv) 403 movb $0, 12(%rsp) /* clear saved upcall_mask from %cs */ 404 #endif 405 push $0 /* error code -- zero for #UD */ 406 ud_kernel: 407 push $0xdddd /* a dummy trap number */ 408 INTR_PUSH 409 movq REGOFF_RIP(%rsp), %rdi 410 movq REGOFF_RSP(%rsp), %rsi 411 movq REGOFF_RAX(%rsp), %rdx 412 pushq (%rsi) 413 movq %rsp, %rsi 414 subq $8, %rsp 415 call dtrace_invop 416 ALTENTRY(dtrace_invop_callsite) 417 addq $16, %rsp 418 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 419 je ud_push 420 cmpl $DTRACE_INVOP_LEAVE, %eax 421 je ud_leave 422 cmpl $DTRACE_INVOP_NOP, %eax 423 je ud_nop 424 cmpl $DTRACE_INVOP_RET, %eax 425 je ud_ret 426 jmp ud_trap 427 428 ud_push: 429 /* 430 * We must emulate a "pushq %rbp". To do this, we pull the stack 431 * down 8 bytes, and then store the base pointer. 432 */ 433 INTR_POP 434 subq $16, %rsp /* make room for %rbp */ 435 pushq %rax /* push temp */ 436 movq 24(%rsp), %rax /* load calling RIP */ 437 addq $1, %rax /* increment over trapping instr */ 438 movq %rax, 8(%rsp) /* store calling RIP */ 439 movq 32(%rsp), %rax /* load calling CS */ 440 movq %rax, 16(%rsp) /* store calling CS */ 441 movq 40(%rsp), %rax /* load calling RFLAGS */ 442 movq %rax, 24(%rsp) /* store calling RFLAGS */ 443 movq 48(%rsp), %rax /* load calling RSP */ 444 subq $8, %rax /* make room for %rbp */ 445 movq %rax, 32(%rsp) /* store calling RSP */ 446 movq 56(%rsp), %rax /* load calling SS */ 447 movq %rax, 40(%rsp) /* store calling SS */ 448 movq 32(%rsp), %rax /* reload calling RSP */ 449 movq %rbp, (%rax) /* store %rbp there */ 450 popq %rax /* pop off temp */ 451 jmp tr_iret_kernel /* return from interrupt */ 452 /*NOTREACHED*/ 453 454 ud_leave: 455 /* 456 * We must emulate a "leave", which is the same as a "movq %rbp, %rsp" 457 * followed by a "popq %rbp". This is quite a bit simpler on amd64 458 * than it is on i386 -- we can exploit the fact that the %rsp is 459 * explicitly saved to effect the pop without having to reshuffle 460 * the other data pushed for the trap. 461 */ 462 INTR_POP 463 pushq %rax /* push temp */ 464 movq 8(%rsp), %rax /* load calling RIP */ 465 addq $1, %rax /* increment over trapping instr */ 466 movq %rax, 8(%rsp) /* store calling RIP */ 467 movq (%rbp), %rax /* get new %rbp */ 468 addq $8, %rbp /* adjust new %rsp */ 469 movq %rbp, 32(%rsp) /* store new %rsp */ 470 movq %rax, %rbp /* set new %rbp */ 471 popq %rax /* pop off temp */ 472 jmp tr_iret_kernel /* return from interrupt */ 473 /*NOTREACHED*/ 474 475 ud_nop: 476 /* 477 * We must emulate a "nop". This is obviously not hard: we need only 478 * advance the %rip by one. 479 */ 480 INTR_POP 481 incq (%rsp) 482 jmp tr_iret_kernel 483 /*NOTREACHED*/ 484 485 ud_ret: 486 INTR_POP 487 pushq %rax /* push temp */ 488 movq 32(%rsp), %rax /* load %rsp */ 489 movq (%rax), %rax /* load calling RIP */ 490 movq %rax, 8(%rsp) /* store calling RIP */ 491 addq $8, 32(%rsp) /* adjust new %rsp */ 492 popq %rax /* pop off temp */ 493 jmp tr_iret_kernel /* return from interrupt */ 494 /*NOTREACHED*/ 495 496 ud_trap: 497 /* 498 * We're going to let the kernel handle this as a normal #UD. If, 499 * however, we came through #BP and are spoofing #UD (in this case, 500 * the stored error value will be non-zero), we need to de-spoof 501 * the trap by incrementing %rip and pushing T_BPTFLT. 502 */ 503 cmpq $0, REGOFF_ERR(%rsp) 504 je ud_ud 505 incq REGOFF_RIP(%rsp) 506 addq $REGOFF_RIP, %rsp 507 NPTRAP_NOERR(T_BPTFLT) /* $3 */ 508 jmp cmntrap 509 510 ud_ud: 511 addq $REGOFF_RIP, %rsp 512 ud_user: 513 NPTRAP_NOERR(T_ILLINST) 514 jmp cmntrap 515 SET_SIZE(invoptrap) 516 517 #elif defined(__i386) 518 519 /* 520 * #UD 521 */ 522 ENTRY_NP(invoptrap) 523 /* 524 * If we are taking an invalid opcode trap while in the kernel, this 525 * is likely an FBT probe point. 526 */ 527 pushl %gs 528 cmpw $KGS_SEL, (%esp) 529 jne 8f 530 531 addl $4, %esp 532 #if defined(__xpv) 533 movb $0, 6(%esp) /* clear saved upcall_mask from %cs */ 534 #endif /* __xpv */ 535 pusha 536 pushl %eax /* push %eax -- may be return value */ 537 pushl %esp /* push stack pointer */ 538 addl $48, (%esp) /* adjust to incoming args */ 539 pushl 40(%esp) /* push calling EIP */ 540 call dtrace_invop 541 ALTENTRY(dtrace_invop_callsite) 542 addl $12, %esp 543 cmpl $DTRACE_INVOP_PUSHL_EBP, %eax 544 je 1f 545 cmpl $DTRACE_INVOP_POPL_EBP, %eax 546 je 2f 547 cmpl $DTRACE_INVOP_LEAVE, %eax 548 je 3f 549 cmpl $DTRACE_INVOP_NOP, %eax 550 je 4f 551 jmp 7f 552 1: 553 /* 554 * We must emulate a "pushl %ebp". To do this, we pull the stack 555 * down 4 bytes, and then store the base pointer. 556 */ 557 popa 558 subl $4, %esp /* make room for %ebp */ 559 pushl %eax /* push temp */ 560 movl 8(%esp), %eax /* load calling EIP */ 561 incl %eax /* increment over LOCK prefix */ 562 movl %eax, 4(%esp) /* store calling EIP */ 563 movl 12(%esp), %eax /* load calling CS */ 564 movl %eax, 8(%esp) /* store calling CS */ 565 movl 16(%esp), %eax /* load calling EFLAGS */ 566 movl %eax, 12(%esp) /* store calling EFLAGS */ 567 movl %ebp, 16(%esp) /* push %ebp */ 568 popl %eax /* pop off temp */ 569 jmp _emul_done 570 2: 571 /* 572 * We must emulate a "popl %ebp". To do this, we do the opposite of 573 * the above: we remove the %ebp from the stack, and squeeze up the 574 * saved state from the trap. 575 */ 576 popa 577 pushl %eax /* push temp */ 578 movl 16(%esp), %ebp /* pop %ebp */ 579 movl 12(%esp), %eax /* load calling EFLAGS */ 580 movl %eax, 16(%esp) /* store calling EFLAGS */ 581 movl 8(%esp), %eax /* load calling CS */ 582 movl %eax, 12(%esp) /* store calling CS */ 583 movl 4(%esp), %eax /* load calling EIP */ 584 incl %eax /* increment over LOCK prefix */ 585 movl %eax, 8(%esp) /* store calling EIP */ 586 popl %eax /* pop off temp */ 587 addl $4, %esp /* adjust stack pointer */ 588 jmp _emul_done 589 3: 590 /* 591 * We must emulate a "leave", which is the same as a "movl %ebp, %esp" 592 * followed by a "popl %ebp". This looks similar to the above, but 593 * requires two temporaries: one for the new base pointer, and one 594 * for the staging register. 595 */ 596 popa 597 pushl %eax /* push temp */ 598 pushl %ebx /* push temp */ 599 movl %ebp, %ebx /* set temp to old %ebp */ 600 movl (%ebx), %ebp /* pop %ebp */ 601 movl 16(%esp), %eax /* load calling EFLAGS */ 602 movl %eax, (%ebx) /* store calling EFLAGS */ 603 movl 12(%esp), %eax /* load calling CS */ 604 movl %eax, -4(%ebx) /* store calling CS */ 605 movl 8(%esp), %eax /* load calling EIP */ 606 incl %eax /* increment over LOCK prefix */ 607 movl %eax, -8(%ebx) /* store calling EIP */ 608 movl %ebx, -4(%esp) /* temporarily store new %esp */ 609 popl %ebx /* pop off temp */ 610 popl %eax /* pop off temp */ 611 movl -12(%esp), %esp /* set stack pointer */ 612 subl $8, %esp /* adjust for three pushes, one pop */ 613 jmp _emul_done 614 4: 615 /* 616 * We must emulate a "nop". This is obviously not hard: we need only 617 * advance the %eip by one. 618 */ 619 popa 620 incl (%esp) 621 _emul_done: 622 IRET /* return from interrupt */ 623 7: 624 popa 625 pushl $0 626 pushl $T_ILLINST /* $6 */ 627 jmp cmntrap 628 8: 629 addl $4, %esp 630 pushl $0 631 pushl $T_ILLINST /* $6 */ 632 jmp cmntrap 633 SET_SIZE(invoptrap) 634 635 #endif /* __i386 */ 636 637 /* 638 * #NM 639 */ 640 641 ENTRY_NP(ndptrap) 642 TRAP_NOERR(T_NOEXTFLT) /* $0 */ 643 SET_CPU_GSBASE 644 jmp cmntrap 645 SET_SIZE(ndptrap) 646 647 #if !defined(__xpv) 648 #if defined(__amd64) 649 650 /* 651 * #DF 652 */ 653 ENTRY_NP(syserrtrap) 654 pushq $T_DBLFLT 655 SET_CPU_GSBASE 656 657 /* 658 * We share this handler with kmdb (if kmdb is loaded). As such, we 659 * may have reached this point after encountering a #df in kmdb. If 660 * that happens, we'll still be on kmdb's IDT. We need to switch back 661 * to this CPU's IDT before proceeding. Furthermore, if we did arrive 662 * here from kmdb, kmdb is probably in a very sickly state, and 663 * shouldn't be entered from the panic flow. We'll suppress that 664 * entry by setting nopanicdebug. 665 */ 666 pushq %rax 667 subq $DESCTBR_SIZE, %rsp 668 sidt (%rsp) 669 movq %gs:CPU_IDT, %rax 670 cmpq %rax, DTR_BASE(%rsp) 671 je 1f 672 673 movq %rax, DTR_BASE(%rsp) 674 movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%rsp) 675 lidt (%rsp) 676 677 movl $1, nopanicdebug 678 679 1: addq $DESCTBR_SIZE, %rsp 680 popq %rax 681 682 DFTRAP_PUSH 683 684 /* 685 * freeze trap trace. 686 */ 687 #ifdef TRAPTRACE 688 leaq trap_trace_freeze(%rip), %r11 689 incl (%r11) 690 #endif 691 692 ENABLE_INTR_FLAGS 693 694 movq %rsp, %rdi /* ®s */ 695 xorl %esi, %esi /* clear address */ 696 xorl %edx, %edx /* cpuid = 0 */ 697 call trap 698 699 SET_SIZE(syserrtrap) 700 701 #elif defined(__i386) 702 703 /* 704 * #DF 705 */ 706 ENTRY_NP(syserrtrap) 707 cli /* disable interrupts */ 708 709 /* 710 * We share this handler with kmdb (if kmdb is loaded). As such, we 711 * may have reached this point after encountering a #df in kmdb. If 712 * that happens, we'll still be on kmdb's IDT. We need to switch back 713 * to this CPU's IDT before proceeding. Furthermore, if we did arrive 714 * here from kmdb, kmdb is probably in a very sickly state, and 715 * shouldn't be entered from the panic flow. We'll suppress that 716 * entry by setting nopanicdebug. 717 */ 718 719 subl $DESCTBR_SIZE, %esp 720 movl %gs:CPU_IDT, %eax 721 sidt (%esp) 722 cmpl DTR_BASE(%esp), %eax 723 je 1f 724 725 movl %eax, DTR_BASE(%esp) 726 movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%esp) 727 lidt (%esp) 728 729 movl $1, nopanicdebug 730 731 1: addl $DESCTBR_SIZE, %esp 732 733 /* 734 * Check the CPL in the TSS to see what mode 735 * (user or kernel) we took the fault in. At this 736 * point we are running in the context of the double 737 * fault task (dftss) but the CPU's task points to 738 * the previous task (ktss) where the process context 739 * has been saved as the result of the task switch. 740 */ 741 movl %gs:CPU_TSS, %eax /* get the TSS */ 742 movl TSS_SS(%eax), %ebx /* save the fault SS */ 743 movl TSS_ESP(%eax), %edx /* save the fault ESP */ 744 testw $CPL_MASK, TSS_CS(%eax) /* user mode ? */ 745 jz make_frame 746 movw TSS_SS0(%eax), %ss /* get on the kernel stack */ 747 movl TSS_ESP0(%eax), %esp 748 749 /* 750 * Clear the NT flag to avoid a task switch when the process 751 * finally pops the EFL off the stack via an iret. Clear 752 * the TF flag since that is what the processor does for 753 * a normal exception. Clear the IE flag so that interrupts 754 * remain disabled. 755 */ 756 movl TSS_EFL(%eax), %ecx 757 andl $_BITNOT(PS_NT|PS_T|PS_IE), %ecx 758 pushl %ecx 759 popfl /* restore the EFL */ 760 movw TSS_LDT(%eax), %cx /* restore the LDT */ 761 lldt %cx 762 763 /* 764 * Restore process segment selectors. 765 */ 766 movw TSS_DS(%eax), %ds 767 movw TSS_ES(%eax), %es 768 movw TSS_FS(%eax), %fs 769 movw TSS_GS(%eax), %gs 770 771 /* 772 * Restore task segment selectors. 773 */ 774 movl $KDS_SEL, TSS_DS(%eax) 775 movl $KDS_SEL, TSS_ES(%eax) 776 movl $KDS_SEL, TSS_SS(%eax) 777 movl $KFS_SEL, TSS_FS(%eax) 778 movl $KGS_SEL, TSS_GS(%eax) 779 780 /* 781 * Clear the TS bit, the busy bits in both task 782 * descriptors, and switch tasks. 783 */ 784 clts 785 leal gdt0, %ecx 786 movl DFTSS_SEL+4(%ecx), %esi 787 andl $_BITNOT(0x200), %esi 788 movl %esi, DFTSS_SEL+4(%ecx) 789 movl KTSS_SEL+4(%ecx), %esi 790 andl $_BITNOT(0x200), %esi 791 movl %esi, KTSS_SEL+4(%ecx) 792 movw $KTSS_SEL, %cx 793 ltr %cx 794 795 /* 796 * Restore part of the process registers. 797 */ 798 movl TSS_EBP(%eax), %ebp 799 movl TSS_ECX(%eax), %ecx 800 movl TSS_ESI(%eax), %esi 801 movl TSS_EDI(%eax), %edi 802 803 make_frame: 804 /* 805 * Make a trap frame. Leave the error code (0) on 806 * the stack since the first word on a trap stack is 807 * unused anyway. 808 */ 809 pushl %ebx / fault SS 810 pushl %edx / fault ESP 811 pushl TSS_EFL(%eax) / fault EFL 812 pushl TSS_CS(%eax) / fault CS 813 pushl TSS_EIP(%eax) / fault EIP 814 pushl $0 / error code 815 pushl $T_DBLFLT / trap number 8 816 movl TSS_EBX(%eax), %ebx / restore EBX 817 movl TSS_EDX(%eax), %edx / restore EDX 818 movl TSS_EAX(%eax), %eax / restore EAX 819 sti / enable interrupts 820 jmp cmntrap 821 SET_SIZE(syserrtrap) 822 823 #endif /* __i386 */ 824 #endif /* !__xpv */ 825 826 /* 827 * #TS 828 */ 829 ENTRY_NP(invtsstrap) 830 TRAP_ERR(T_TSSFLT) /* $10 already have error code on stack */ 831 jmp cmntrap 832 SET_SIZE(invtsstrap) 833 834 /* 835 * #NP 836 */ 837 ENTRY_NP(segnptrap) 838 TRAP_ERR(T_SEGFLT) /* $11 already have error code on stack */ 839 #if defined(__amd64) 840 SET_CPU_GSBASE 841 #endif 842 jmp cmntrap 843 SET_SIZE(segnptrap) 844 845 /* 846 * #SS 847 */ 848 ENTRY_NP(stktrap) 849 TRAP_ERR(T_STKFLT) /* $12 already have error code on stack */ 850 #if defined(__amd64) 851 SET_CPU_GSBASE 852 #endif 853 jmp cmntrap 854 SET_SIZE(stktrap) 855 856 /* 857 * #GP 858 */ 859 ENTRY_NP(gptrap) 860 TRAP_ERR(T_GPFLT) /* $13 already have error code on stack */ 861 #if defined(__amd64) 862 SET_CPU_GSBASE 863 #endif 864 jmp cmntrap 865 SET_SIZE(gptrap) 866 867 /* 868 * #PF 869 */ 870 ENTRY_NP(pftrap) 871 TRAP_ERR(T_PGFLT) /* $14 already have error code on stack */ 872 INTR_PUSH 873 #if defined(__xpv) 874 875 #if defined(__amd64) 876 movq %gs:CPU_VCPU_INFO, %r15 877 movq VCPU_INFO_ARCH_CR2(%r15), %r15 /* vcpu[].arch.cr2 */ 878 #elif defined(__i386) 879 movl %gs:CPU_VCPU_INFO, %esi 880 movl VCPU_INFO_ARCH_CR2(%esi), %esi /* vcpu[].arch.cr2 */ 881 #endif /* __i386 */ 882 883 #else /* __xpv */ 884 885 #if defined(__amd64) 886 movq %cr2, %r15 887 #elif defined(__i386) 888 movl %cr2, %esi 889 #endif /* __i386 */ 890 891 #endif /* __xpv */ 892 jmp cmntrap_pushed 893 SET_SIZE(pftrap) 894 895 #if !defined(__amd64) 896 897 .globl idt0_default_r 898 899 /* 900 * #PF pentium bug workaround 901 */ 902 ENTRY_NP(pentium_pftrap) 903 pushl %eax 904 movl %cr2, %eax 905 andl $MMU_STD_PAGEMASK, %eax 906 907 cmpl %eax, %cs:idt0_default_r+2 /* fixme */ 908 909 je check_for_user_address 910 user_mode: 911 popl %eax 912 pushl $T_PGFLT /* $14 */ 913 jmp cmntrap 914 check_for_user_address: 915 /* 916 * Before we assume that we have an unmapped trap on our hands, 917 * check to see if this is a fault from user mode. If it is, 918 * we'll kick back into the page fault handler. 919 */ 920 movl 4(%esp), %eax /* error code */ 921 andl $PF_ERR_USER, %eax 922 jnz user_mode 923 924 /* 925 * We now know that this is the invalid opcode trap. 926 */ 927 popl %eax 928 addl $4, %esp /* pop error code */ 929 jmp invoptrap 930 SET_SIZE(pentium_pftrap) 931 932 #endif /* !__amd64 */ 933 934 ENTRY_NP(resvtrap) 935 TRAP_NOERR(T_RESVTRAP) /* (reserved) */ 936 jmp cmntrap 937 SET_SIZE(resvtrap) 938 939 /* 940 * #MF 941 */ 942 ENTRY_NP(ndperr) 943 TRAP_NOERR(T_EXTERRFLT) /* $16 */ 944 jmp cmninttrap 945 SET_SIZE(ndperr) 946 947 /* 948 * #AC 949 */ 950 ENTRY_NP(achktrap) 951 TRAP_ERR(T_ALIGNMENT) /* $17 */ 952 jmp cmntrap 953 SET_SIZE(achktrap) 954 955 /* 956 * #MC 957 */ 958 .globl cmi_mca_trap /* see uts/i86pc/os/cmi.c */ 959 960 #if defined(__amd64) 961 962 ENTRY_NP(mcetrap) 963 TRAP_NOERR(T_MCE) /* $18 */ 964 965 SET_CPU_GSBASE 966 967 INTR_PUSH 968 INTGATE_INIT_KERNEL_FLAGS 969 970 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) 971 TRACE_REGS(%rdi, %rsp, %rbx, %rcx) 972 TRACE_STAMP(%rdi) 973 974 movq %rsp, %rbp 975 976 movq %rsp, %rdi /* arg0 = struct regs *rp */ 977 call cmi_mca_trap /* cmi_mca_trap(rp); */ 978 979 jmp _sys_rtt 980 SET_SIZE(mcetrap) 981 982 #else 983 984 ENTRY_NP(mcetrap) 985 TRAP_NOERR(T_MCE) /* $18 */ 986 987 INTR_PUSH 988 INTGATE_INIT_KERNEL_FLAGS 989 990 TRACE_PTR(%edi, %ebx, %ebx, %ecx, $TT_TRAP) 991 TRACE_REGS(%edi, %esp, %ebx, %ecx) 992 TRACE_STAMP(%edi) 993 994 movl %esp, %ebp 995 996 movl %esp, %ecx 997 pushl %ecx /* arg0 = struct regs *rp */ 998 call cmi_mca_trap /* cmi_mca_trap(rp) */ 999 addl $4, %esp /* pop arg0 */ 1000 1001 jmp _sys_rtt 1002 SET_SIZE(mcetrap) 1003 1004 #endif 1005 1006 /* 1007 * #XF 1008 */ 1009 ENTRY_NP(xmtrap) 1010 TRAP_NOERR(T_SIMDFPE) /* $19 */ 1011 jmp cmninttrap 1012 SET_SIZE(xmtrap) 1013 1014 ENTRY_NP(invaltrap) 1015 TRAP_NOERR(T_INVALTRAP) /* very invalid */ 1016 jmp cmntrap 1017 SET_SIZE(invaltrap) 1018 1019 .globl fasttable 1020 1021 #if defined(__amd64) 1022 1023 ENTRY_NP(fasttrap) 1024 cmpl $T_LASTFAST, %eax 1025 ja 1f 1026 orl %eax, %eax /* (zero extend top 32-bits) */ 1027 leaq fasttable(%rip), %r11 1028 leaq (%r11, %rax, CLONGSIZE), %r11 1029 movq (%r11), %r11 1030 INDIRECT_JMP_REG(r11) 1031 1: 1032 /* 1033 * Fast syscall number was illegal. Make it look 1034 * as if the INT failed. Modify %rip to point before the 1035 * INT, push the expected error code and fake a GP fault. 1036 * 1037 * XXX Why make the error code be offset into idt + 1? 1038 * Instead we should push a real (soft?) error code 1039 * on the stack and #gp handler could know about fasttraps? 1040 */ 1041 XPV_TRAP_POP 1042 1043 subq $2, (%rsp) /* XXX int insn 2-bytes */ 1044 pushq $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2) 1045 1046 #if defined(__xpv) 1047 pushq %r11 1048 pushq %rcx 1049 #endif 1050 jmp gptrap 1051 SET_SIZE(fasttrap) 1052 1053 #elif defined(__i386) 1054 1055 ENTRY_NP(fasttrap) 1056 cmpl $T_LASTFAST, %eax 1057 ja 1f 1058 jmp *%cs:fasttable(, %eax, CLONGSIZE) 1059 1: 1060 /* 1061 * Fast syscall number was illegal. Make it look 1062 * as if the INT failed. Modify %eip to point before the 1063 * INT, push the expected error code and fake a GP fault. 1064 * 1065 * XXX Why make the error code be offset into idt + 1? 1066 * Instead we should push a real (soft?) error code 1067 * on the stack and #gp handler could know about fasttraps? 1068 */ 1069 subl $2, (%esp) /* XXX int insn 2-bytes */ 1070 pushl $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2) 1071 jmp gptrap 1072 SET_SIZE(fasttrap) 1073 1074 #endif /* __i386 */ 1075 1076 ENTRY_NP(dtrace_ret) 1077 TRAP_NOERR(T_DTRACE_RET) 1078 jmp dtrace_trap 1079 SET_SIZE(dtrace_ret) 1080 1081 #if defined(__amd64) 1082 1083 /* 1084 * RFLAGS 24 bytes up the stack from %rsp. 1085 * XXX a constant would be nicer. 1086 */ 1087 ENTRY_NP(fast_null) 1088 XPV_TRAP_POP 1089 orq $PS_C, 24(%rsp) /* set carry bit in user flags */ 1090 call x86_md_clear 1091 jmp tr_iret_auto 1092 /*NOTREACHED*/ 1093 SET_SIZE(fast_null) 1094 1095 #elif defined(__i386) 1096 1097 ENTRY_NP(fast_null) 1098 orw $PS_C, 8(%esp) /* set carry bit in user flags */ 1099 IRET 1100 SET_SIZE(fast_null) 1101 1102 #endif /* __i386 */ 1103 1104 /* 1105 * Interrupts start at 32 1106 */ 1107 #define MKIVCT(n) \ 1108 ENTRY_NP(ivct/**/n) \ 1109 push $0; \ 1110 push $n - 0x20; \ 1111 jmp cmnint; \ 1112 SET_SIZE(ivct/**/n) 1113 1114 MKIVCT(32) 1115 MKIVCT(33) 1116 MKIVCT(34) 1117 MKIVCT(35) 1118 MKIVCT(36) 1119 MKIVCT(37) 1120 MKIVCT(38) 1121 MKIVCT(39) 1122 MKIVCT(40) 1123 MKIVCT(41) 1124 MKIVCT(42) 1125 MKIVCT(43) 1126 MKIVCT(44) 1127 MKIVCT(45) 1128 MKIVCT(46) 1129 MKIVCT(47) 1130 MKIVCT(48) 1131 MKIVCT(49) 1132 MKIVCT(50) 1133 MKIVCT(51) 1134 MKIVCT(52) 1135 MKIVCT(53) 1136 MKIVCT(54) 1137 MKIVCT(55) 1138 MKIVCT(56) 1139 MKIVCT(57) 1140 MKIVCT(58) 1141 MKIVCT(59) 1142 MKIVCT(60) 1143 MKIVCT(61) 1144 MKIVCT(62) 1145 MKIVCT(63) 1146 MKIVCT(64) 1147 MKIVCT(65) 1148 MKIVCT(66) 1149 MKIVCT(67) 1150 MKIVCT(68) 1151 MKIVCT(69) 1152 MKIVCT(70) 1153 MKIVCT(71) 1154 MKIVCT(72) 1155 MKIVCT(73) 1156 MKIVCT(74) 1157 MKIVCT(75) 1158 MKIVCT(76) 1159 MKIVCT(77) 1160 MKIVCT(78) 1161 MKIVCT(79) 1162 MKIVCT(80) 1163 MKIVCT(81) 1164 MKIVCT(82) 1165 MKIVCT(83) 1166 MKIVCT(84) 1167 MKIVCT(85) 1168 MKIVCT(86) 1169 MKIVCT(87) 1170 MKIVCT(88) 1171 MKIVCT(89) 1172 MKIVCT(90) 1173 MKIVCT(91) 1174 MKIVCT(92) 1175 MKIVCT(93) 1176 MKIVCT(94) 1177 MKIVCT(95) 1178 MKIVCT(96) 1179 MKIVCT(97) 1180 MKIVCT(98) 1181 MKIVCT(99) 1182 MKIVCT(100) 1183 MKIVCT(101) 1184 MKIVCT(102) 1185 MKIVCT(103) 1186 MKIVCT(104) 1187 MKIVCT(105) 1188 MKIVCT(106) 1189 MKIVCT(107) 1190 MKIVCT(108) 1191 MKIVCT(109) 1192 MKIVCT(110) 1193 MKIVCT(111) 1194 MKIVCT(112) 1195 MKIVCT(113) 1196 MKIVCT(114) 1197 MKIVCT(115) 1198 MKIVCT(116) 1199 MKIVCT(117) 1200 MKIVCT(118) 1201 MKIVCT(119) 1202 MKIVCT(120) 1203 MKIVCT(121) 1204 MKIVCT(122) 1205 MKIVCT(123) 1206 MKIVCT(124) 1207 MKIVCT(125) 1208 MKIVCT(126) 1209 MKIVCT(127) 1210 MKIVCT(128) 1211 MKIVCT(129) 1212 MKIVCT(130) 1213 MKIVCT(131) 1214 MKIVCT(132) 1215 MKIVCT(133) 1216 MKIVCT(134) 1217 MKIVCT(135) 1218 MKIVCT(136) 1219 MKIVCT(137) 1220 MKIVCT(138) 1221 MKIVCT(139) 1222 MKIVCT(140) 1223 MKIVCT(141) 1224 MKIVCT(142) 1225 MKIVCT(143) 1226 MKIVCT(144) 1227 MKIVCT(145) 1228 MKIVCT(146) 1229 MKIVCT(147) 1230 MKIVCT(148) 1231 MKIVCT(149) 1232 MKIVCT(150) 1233 MKIVCT(151) 1234 MKIVCT(152) 1235 MKIVCT(153) 1236 MKIVCT(154) 1237 MKIVCT(155) 1238 MKIVCT(156) 1239 MKIVCT(157) 1240 MKIVCT(158) 1241 MKIVCT(159) 1242 MKIVCT(160) 1243 MKIVCT(161) 1244 MKIVCT(162) 1245 MKIVCT(163) 1246 MKIVCT(164) 1247 MKIVCT(165) 1248 MKIVCT(166) 1249 MKIVCT(167) 1250 MKIVCT(168) 1251 MKIVCT(169) 1252 MKIVCT(170) 1253 MKIVCT(171) 1254 MKIVCT(172) 1255 MKIVCT(173) 1256 MKIVCT(174) 1257 MKIVCT(175) 1258 MKIVCT(176) 1259 MKIVCT(177) 1260 MKIVCT(178) 1261 MKIVCT(179) 1262 MKIVCT(180) 1263 MKIVCT(181) 1264 MKIVCT(182) 1265 MKIVCT(183) 1266 MKIVCT(184) 1267 MKIVCT(185) 1268 MKIVCT(186) 1269 MKIVCT(187) 1270 MKIVCT(188) 1271 MKIVCT(189) 1272 MKIVCT(190) 1273 MKIVCT(191) 1274 MKIVCT(192) 1275 MKIVCT(193) 1276 MKIVCT(194) 1277 MKIVCT(195) 1278 MKIVCT(196) 1279 MKIVCT(197) 1280 MKIVCT(198) 1281 MKIVCT(199) 1282 MKIVCT(200) 1283 MKIVCT(201) 1284 MKIVCT(202) 1285 MKIVCT(203) 1286 MKIVCT(204) 1287 MKIVCT(205) 1288 MKIVCT(206) 1289 MKIVCT(207) 1290 MKIVCT(208) 1291 MKIVCT(209) 1292 MKIVCT(210) 1293 MKIVCT(211) 1294 MKIVCT(212) 1295 MKIVCT(213) 1296 MKIVCT(214) 1297 MKIVCT(215) 1298 MKIVCT(216) 1299 MKIVCT(217) 1300 MKIVCT(218) 1301 MKIVCT(219) 1302 MKIVCT(220) 1303 MKIVCT(221) 1304 MKIVCT(222) 1305 MKIVCT(223) 1306 MKIVCT(224) 1307 MKIVCT(225) 1308 MKIVCT(226) 1309 MKIVCT(227) 1310 MKIVCT(228) 1311 MKIVCT(229) 1312 MKIVCT(230) 1313 MKIVCT(231) 1314 MKIVCT(232) 1315 MKIVCT(233) 1316 MKIVCT(234) 1317 MKIVCT(235) 1318 MKIVCT(236) 1319 MKIVCT(237) 1320 MKIVCT(238) 1321 MKIVCT(239) 1322 MKIVCT(240) 1323 MKIVCT(241) 1324 MKIVCT(242) 1325 MKIVCT(243) 1326 MKIVCT(244) 1327 MKIVCT(245) 1328 MKIVCT(246) 1329 MKIVCT(247) 1330 MKIVCT(248) 1331 MKIVCT(249) 1332 MKIVCT(250) 1333 MKIVCT(251) 1334 MKIVCT(252) 1335 MKIVCT(253) 1336 MKIVCT(254) 1337 MKIVCT(255) 1338 1339 #endif /* __lint */