1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2019 Joyent, Inc. 24 */ 25 26 #include <sys/asm_linkage.h> 27 #include <sys/asm_misc.h> 28 #include <sys/regset.h> 29 #include <sys/privregs.h> 30 #include <sys/x86_archext.h> 31 #include <sys/cpr_wakecode.h> 32 33 #include <sys/segments.h> 34 #include "assym.h" 35 36 #ifdef DEBUG 37 #define LED 1 38 #define SERIAL 1 39 #endif /* DEBUG */ 40 41 #ifdef DEBUG 42 #define COM1 0x3f8 43 #define COM2 0x2f8 44 #define WC_COM COM2 /* either COM1 or COM2 */ 45 #define WC_LED 0x80 /* diagnostic led port ON motherboard */ 46 47 /* 48 * defined as offsets from the data register 49 */ 50 #define DLL 0 /* divisor latch (lsb) */ 51 #define DLH 1 /* divisor latch (msb) */ 52 #define LCR 3 /* line control register */ 53 #define MCR 4 /* modem control register */ 54 55 56 #define DLAB 0x80 /* divisor latch access bit */ 57 #define B9600L 0X0c /* lsb bit pattern for 9600 baud */ 58 #define B9600H 0X0 /* hsb bit pattern for 9600 baud */ 59 #define DTR 0x01 /* Data Terminal Ready */ 60 #define RTS 0x02 /* Request To Send */ 61 #define STOP1 0x00 /* 1 stop bit */ 62 #define BITS8 0x03 /* 8 bits per char */ 63 64 #endif /* DEBUG */ 65 66 /* 67 * This file contains the low level routines involved in getting 68 * into and out of ACPI S3, including those needed for restarting 69 * the non-boot cpus. 70 * 71 * Our assumptions: 72 * 73 * Our actions: 74 * 75 */ 76 77 ENTRY_NP(wc_save_context) 78 79 movq (%rsp), %rdx / return address 80 movq %rdx, WC_RETADDR(%rdi) 81 pushq %rbp 82 movq %rsp,%rbp 83 84 movq %rdi, WC_VIRTADDR(%rdi) 85 movq %rdi, WC_RDI(%rdi) 86 87 movq %rdx, WC_RDX(%rdi) 88 89 / stash everything else we need 90 sgdt WC_GDT(%rdi) 91 sidt WC_IDT(%rdi) 92 sldt WC_LDT(%rdi) 93 str WC_TR(%rdi) 94 95 movq %cr0, %rdx 96 movq %rdx, WC_CR0(%rdi) 97 movq %cr3, %rdx 98 movq %rdx, WC_CR3(%rdi) 99 movq %cr4, %rdx 100 movq %rdx, WC_CR4(%rdi) 101 movq %cr8, %rdx 102 movq %rdx, WC_CR8(%rdi) 103 104 movq %r8, WC_R8(%rdi) 105 movq %r9, WC_R9(%rdi) 106 movq %r10, WC_R10(%rdi) 107 movq %r11, WC_R11(%rdi) 108 movq %r12, WC_R12(%rdi) 109 movq %r13, WC_R13(%rdi) 110 movq %r14, WC_R14(%rdi) 111 movq %r15, WC_R15(%rdi) 112 movq %rax, WC_RAX(%rdi) 113 movq %rbp, WC_RBP(%rdi) 114 movq %rbx, WC_RBX(%rdi) 115 movq %rcx, WC_RCX(%rdi) 116 movq %rsi, WC_RSI(%rdi) 117 movq %rsp, WC_RSP(%rdi) 118 119 movw %ss, WC_SS(%rdi) 120 movw %cs, WC_CS(%rdi) 121 movw %ds, WC_DS(%rdi) 122 movw %es, WC_ES(%rdi) 123 124 movq $0, %rcx / save %fs register 125 movw %fs, %cx 126 movq %rcx, WC_FS(%rdi) 127 128 movl $MSR_AMD_FSBASE, %ecx 129 rdmsr 130 movl %eax, WC_FSBASE(%rdi) 131 movl %edx, WC_FSBASE+4(%rdi) 132 133 movq $0, %rcx / save %gs register 134 movw %gs, %cx 135 movq %rcx, WC_GS(%rdi) 136 137 movl $MSR_AMD_GSBASE, %ecx / save gsbase msr 138 rdmsr 139 movl %eax, WC_GSBASE(%rdi) 140 movl %edx, WC_GSBASE+4(%rdi) 141 142 movl $MSR_AMD_KGSBASE, %ecx / save kgsbase msr 143 rdmsr 144 movl %eax, WC_KGSBASE(%rdi) 145 movl %edx, WC_KGSBASE+4(%rdi) 146 147 movq %gs:CPU_ID, %rax / save current cpu id 148 movq %rax, WC_CPU_ID(%rdi) 149 150 pushfq 151 popq WC_EFLAGS(%rdi) 152 153 wbinvd / flush the cache 154 mfence 155 156 movq $1, %rax / at suspend return 1 157 158 leave 159 160 ret 161 162 SET_SIZE(wc_save_context) 163 164 165 /* 166 * Our assumptions: 167 * - We are running in real mode. 168 * - Interrupts are disabled. 169 * 170 * Our actions: 171 * - We start using our GDT by loading correct values in the 172 * selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL, 173 * gs=KGS_SEL). 174 * - We change over to using our IDT. 175 * - We load the default LDT into the hardware LDT register. 176 * - We load the default TSS into the hardware task register. 177 * - We restore registers 178 * - We return to original caller (a la setjmp) 179 */ 180 181 ENTRY_NP(wc_rm_start) 182 183 /* 184 * For the Sun Studio 10 assembler we needed to do a .code32 and 185 * mentally invert the meaning of the addr16 and data16 prefixes to 186 * get 32-bit access when generating code to be executed in 16-bit 187 * mode (sigh...) 188 * 189 * This code, despite always being built with GNU as, has inherited 190 * the conceptual damage. 191 */ 192 193 .code32 194 195 cli 196 movw %cs, %ax 197 movw %ax, %ds / establish ds ... 198 movw %ax, %ss / ... and ss:esp 199 D16 movl $WC_STKSTART, %esp 200 / using the following value blows up machines! - DO NOT USE 201 / D16 movl 0xffc, %esp 202 203 204 #if LED 205 D16 movl $WC_LED, %edx 206 D16 movb $0xd1, %al 207 outb (%dx) 208 #endif 209 210 #if SERIAL 211 D16 movl $WC_COM, %edx 212 D16 movb $0x61, %al 213 outb (%dx) 214 #endif 215 216 D16 call cominit 217 218 /* 219 * Enable protected-mode, write protect, and alignment mask 220 * %cr0 has already been initialsed to zero 221 */ 222 movl %cr0, %eax 223 D16 orl $_CONST(CR0_PE|CR0_WP|CR0_AM), %eax 224 movl %eax, %cr0 225 226 /* 227 * Do a jmp immediately after writing to cr0 when enabling protected 228 * mode to clear the real mode prefetch queue (per Intel's docs) 229 */ 230 jmp pestart 231 pestart: 232 233 #if LED 234 D16 movl $WC_LED, %edx 235 D16 movb $0xd2, %al 236 outb (%dx) 237 #endif 238 239 #if SERIAL 240 D16 movl $WC_COM, %edx 241 D16 movb $0x62, %al 242 outb (%dx) 243 #endif 244 245 /* 246 * 16-bit protected mode is now active, so prepare to turn on long 247 * mode 248 */ 249 250 #if LED 251 D16 movl $WC_LED, %edx 252 D16 movb $0xd3, %al 253 outb (%dx) 254 #endif 255 256 #if SERIAL 257 D16 movl $WC_COM, %edx 258 D16 movb $0x63, %al 259 outb (%dx) 260 #endif 261 262 /* 263 * Add any initial cr4 bits 264 */ 265 movl %cr4, %eax 266 A16 D16 orl CR4OFF, %eax 267 268 /* 269 * Enable PAE mode (CR4.PAE) 270 */ 271 D16 orl $CR4_PAE, %eax 272 movl %eax, %cr4 273 274 #if LED 275 D16 movl $WC_LED, %edx 276 D16 movb $0xd4, %al 277 outb (%dx) 278 #endif 279 280 #if SERIAL 281 D16 movl $WC_COM, %edx 282 D16 movb $0x64, %al 283 outb (%dx) 284 #endif 285 286 /* 287 * Point cr3 to the 64-bit long mode page tables. 288 * 289 * Note that these MUST exist in 32-bit space, as we don't have 290 * a way to load %cr3 with a 64-bit base address for the page tables 291 * until the CPU is actually executing in 64-bit long mode. 292 */ 293 A16 D16 movl CR3OFF, %eax 294 movl %eax, %cr3 295 296 /* 297 * Set long mode enable in EFER (EFER.LME = 1) 298 */ 299 D16 movl $MSR_AMD_EFER, %ecx 300 rdmsr 301 302 D16 orl $AMD_EFER_LME, %eax 303 wrmsr 304 305 #if LED 306 D16 movl $WC_LED, %edx 307 D16 movb $0xd5, %al 308 outb (%dx) 309 #endif 310 311 #if SERIAL 312 D16 movl $WC_COM, %edx 313 D16 movb $0x65, %al 314 outb (%dx) 315 #endif 316 317 /* 318 * Finally, turn on paging (CR0.PG = 1) to activate long mode. 319 */ 320 movl %cr0, %eax 321 D16 orl $CR0_PG, %eax 322 movl %eax, %cr0 323 324 /* 325 * The instruction after enabling paging in CR0 MUST be a branch. 326 */ 327 jmp long_mode_active 328 329 long_mode_active: 330 331 #if LED 332 D16 movl $WC_LED, %edx 333 D16 movb $0xd6, %al 334 outb (%dx) 335 #endif 336 337 #if SERIAL 338 D16 movl $WC_COM, %edx 339 D16 movb $0x66, %al 340 outb (%dx) 341 #endif 342 343 /* 344 * Long mode is now active but since we're still running with the 345 * original 16-bit CS we're actually in 16-bit compatability mode. 346 * 347 * We have to load an intermediate GDT and IDT here that we know are 348 * in 32-bit space before we can use the kernel's GDT and IDT, which 349 * may be in the 64-bit address space, and since we're in compatability 350 * mode, we only have access to 16 and 32-bit instructions at the 351 * moment. 352 */ 353 A16 D16 lgdt TEMPGDTOFF /* load temporary GDT */ 354 A16 D16 lidt TEMPIDTOFF /* load temporary IDT */ 355 356 357 /* 358 * Do a far transfer to 64-bit mode. Set the CS selector to a 64-bit 359 * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump 360 * to the real mode platter address of wc_long_mode_64 as until the 361 * 64-bit CS is in place we don't have access to 64-bit instructions 362 * and thus can't reference a 64-bit %rip. 363 */ 364 365 #if LED 366 D16 movl $WC_LED, %edx 367 D16 movb $0xd7, %al 368 outb (%dx) 369 #endif 370 371 #if SERIAL 372 D16 movl $WC_COM, %edx 373 D16 movb $0x67, %al 374 outb (%dx) 375 #endif 376 377 D16 pushl $TEMP_CS64_SEL 378 A16 D16 pushl LM64OFF 379 380 D16 lret 381 382 383 /* 384 * Support routine to re-initialize VGA subsystem 385 */ 386 vgainit: 387 D16 ret 388 389 /* 390 * Support routine to re-initialize keyboard (which is USB - help!) 391 */ 392 kbdinit: 393 D16 ret 394 395 /* 396 * Support routine to re-initialize COM ports to something sane 397 */ 398 cominit: 399 / init COM1 & COM2 400 401 #if DEBUG 402 /* 403 * on debug kernels we need to initialize COM1 & COM2 here, so that 404 * we can get debug output before the asy driver has resumed 405 */ 406 407 / select COM1 408 D16 movl $_CONST(COM1+LCR), %edx 409 D16 movb $DLAB, %al / divisor latch 410 outb (%dx) 411 412 D16 movl $_CONST(COM1+DLL), %edx / divisor latch lsb 413 D16 movb $B9600L, %al / divisor latch 414 outb (%dx) 415 416 D16 movl $_CONST(COM1+DLH), %edx / divisor latch hsb 417 D16 movb $B9600H, %al / divisor latch 418 outb (%dx) 419 420 D16 movl $_CONST(COM1+LCR), %edx / select COM1 421 D16 movb $_CONST(STOP1|BITS8), %al / 1 stop bit, 8bit word len 422 outb (%dx) 423 424 D16 movl $_CONST(COM1+MCR), %edx / select COM1 425 D16 movb $_CONST(RTS|DTR), %al / data term ready & req to send 426 outb (%dx) 427 428 / select COM2 429 D16 movl $_CONST(COM2+LCR), %edx 430 D16 movb $DLAB, %al / divisor latch 431 outb (%dx) 432 433 D16 movl $_CONST(COM2+DLL), %edx / divisor latch lsb 434 D16 movb $B9600L, %al / divisor latch 435 outb (%dx) 436 437 D16 movl $_CONST(COM2+DLH), %edx / divisor latch hsb 438 D16 movb $B9600H, %al / divisor latch 439 outb (%dx) 440 441 D16 movl $_CONST(COM2+LCR), %edx / select COM1 442 D16 movb $_CONST(STOP1|BITS8), %al / 1 stop bit, 8bit word len 443 outb (%dx) 444 445 D16 movl $_CONST(COM2+MCR), %edx / select COM1 446 D16 movb $_CONST(RTS|DTR), %al / data term ready & req to send 447 outb (%dx) 448 #endif /* DEBUG */ 449 450 D16 ret 451 452 .code64 453 454 .globl wc_long_mode_64 455 wc_long_mode_64: 456 457 #if LED 458 movw $WC_LED, %dx 459 movb $0xd8, %al 460 outb (%dx) 461 #endif 462 463 #if SERIAL 464 movw $WC_COM, %dx 465 movb $0x68, %al 466 outb (%dx) 467 #endif 468 469 /* 470 * We are now running in long mode with a 64-bit CS (EFER.LMA=1, 471 * CS.L=1) so we now have access to 64-bit instructions. 472 * 473 * First, set the 64-bit GDT base. 474 */ 475 .globl rm_platter_pa 476 movl rm_platter_pa, %eax 477 478 lgdtq GDTROFF(%rax) /* load 64-bit GDT */ 479 480 /* 481 * Save the CPU number in %r11; get the value here since it's saved in 482 * the real mode platter. 483 */ 484 / JAN 485 / the following is wrong! need to figure out MP systems 486 / movl CPUNOFF(%rax), %r11d 487 488 /* 489 * Add rm_platter_pa to %rsp to point it to the same location as seen 490 * from 64-bit mode. 491 */ 492 addq %rax, %rsp 493 494 /* 495 * Now do an lretq to load CS with the appropriate selector for the 496 * kernel's 64-bit GDT and to start executing 64-bit setup code at the 497 * virtual address where boot originally loaded this code rather than 498 * the copy in the real mode platter's rm_code array as we've been 499 * doing so far. 500 */ 501 502 #if LED 503 movw $WC_LED, %dx 504 movb $0xd9, %al 505 outb (%dx) 506 #endif 507 508 / JAN this should produce 'i' but we get 'g' instead ??? 509 #if SERIAL 510 movw $WC_COM, %dx 511 movb $0x69, %al 512 outb (%dx) 513 #endif 514 515 pushq $KCS_SEL 516 pushq $kernel_wc_code 517 lretq 518 519 .globl kernel_wc_code 520 kernel_wc_code: 521 522 #if LED 523 movw $WC_LED, %dx 524 movb $0xda, %al 525 outb (%dx) 526 #endif 527 528 / JAN this should produce 'j' but we get 'g' instead ??? 529 #if SERIAL 530 movw $WC_COM, %dx 531 movb $0x6a, %al 532 outb (%dx) 533 #endif 534 535 /* 536 * Complete the balance of the setup we need to before executing 537 * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS). 538 */ 539 .globl rm_platter_va 540 movq rm_platter_va, %rbx 541 addq $WC_CPU, %rbx 542 543 #if LED 544 movw $WC_LED, %dx 545 movb $0xdb, %al 546 outb (%dx) 547 #endif 548 549 #if SERIAL 550 movw $WC_COM, %dx 551 movw $0x6b, %ax 552 outb (%dx) 553 #endif 554 555 /* 556 * restore the rest of the registers 557 */ 558 559 lidtq WC_IDT(%rbx) 560 561 #if LED 562 movw $WC_LED, %dx 563 movb $0xdc, %al 564 outb (%dx) 565 #endif 566 567 #if SERIAL 568 movw $WC_COM, %dx 569 movw $0x6c, %ax 570 outb (%dx) 571 #endif 572 573 /* 574 * restore the rest of the registers 575 */ 576 577 movw $KDS_SEL, %ax 578 movw %ax, %ds 579 movw %ax, %es 580 movw %ax, %ss 581 582 /* 583 * Before proceeding, enable usage of the page table NX bit if 584 * that's how the page tables are set up. 585 */ 586 bt $X86FSET_NX, x86_featureset(%rip) 587 jnc 1f 588 movl $MSR_AMD_EFER, %ecx 589 rdmsr 590 orl $AMD_EFER_NXE, %eax 591 wrmsr 592 1: 593 594 movq WC_CR4(%rbx), %rax / restore full cr4 (with Global Enable) 595 movq %rax, %cr4 596 597 lldt WC_LDT(%rbx) 598 movzwq WC_TR(%rbx), %rax / clear TSS busy bit 599 addq WC_GDT+2(%rbx), %rax 600 andl $0xfffffdff, 4(%rax) 601 movq 4(%rax), %rcx 602 ltr WC_TR(%rbx) 603 604 #if LED 605 movw $WC_LED, %dx 606 movb $0xdd, %al 607 outb (%dx) 608 #endif 609 610 #if SERIAL 611 movw $WC_COM, %dx 612 movw $0x6d, %ax 613 outb (%dx) 614 #endif 615 616 / restore %fsbase %gsbase %kgbase registers using wrmsr instruction 617 618 movq WC_FS(%rbx), %rcx / restore fs register 619 movw %cx, %fs 620 621 movl $MSR_AMD_FSBASE, %ecx 622 movl WC_FSBASE(%rbx), %eax 623 movl WC_FSBASE+4(%rbx), %edx 624 wrmsr 625 626 movq WC_GS(%rbx), %rcx / restore gs register 627 movw %cx, %gs 628 629 movl $MSR_AMD_GSBASE, %ecx / restore gsbase msr 630 movl WC_GSBASE(%rbx), %eax 631 movl WC_GSBASE+4(%rbx), %edx 632 wrmsr 633 634 movl $MSR_AMD_KGSBASE, %ecx / restore kgsbase msr 635 movl WC_KGSBASE(%rbx), %eax 636 movl WC_KGSBASE+4(%rbx), %edx 637 wrmsr 638 639 movq WC_CR0(%rbx), %rdx 640 movq %rdx, %cr0 641 movq WC_CR3(%rbx), %rdx 642 movq %rdx, %cr3 643 movq WC_CR8(%rbx), %rdx 644 movq %rdx, %cr8 645 646 #if LED 647 movw $WC_LED, %dx 648 movb $0xde, %al 649 outb (%dx) 650 #endif 651 652 #if SERIAL 653 movw $WC_COM, %dx 654 movb $0x6e, %al 655 outb (%dx) 656 #endif 657 658 /* 659 * if we are not running on the boot CPU restore stack contents by 660 * calling i_cpr_restore_stack(curthread, save_stack); 661 */ 662 movq %rsp, %rbp 663 call i_cpr_bootcpuid 664 cmpl %eax, WC_CPU_ID(%rbx) 665 je 2f 666 667 movq %gs:CPU_THREAD, %rdi 668 movq WC_SAVED_STACK(%rbx), %rsi 669 call i_cpr_restore_stack 670 2: 671 672 movq WC_RSP(%rbx), %rsp / restore stack pointer 673 674 /* 675 * APIC initialization 676 */ 677 movq %rsp, %rbp 678 679 /* 680 * skip iff function pointer is NULL 681 */ 682 cmpq $0, ap_mlsetup 683 je 3f 684 leaq ap_mlsetup, %rax 685 INDIRECT_CALL_REG(rax) 686 3: 687 688 leaq cpr_start_cpu_func, %rax 689 INDIRECT_CALL_REG(rax) 690 691 / restore %rbx to the value it ahd before we called the functions above 692 movq rm_platter_va, %rbx 693 addq $WC_CPU, %rbx 694 695 movq WC_R8(%rbx), %r8 696 movq WC_R9(%rbx), %r9 697 movq WC_R10(%rbx), %r10 698 movq WC_R11(%rbx), %r11 699 movq WC_R12(%rbx), %r12 700 movq WC_R13(%rbx), %r13 701 movq WC_R14(%rbx), %r14 702 movq WC_R15(%rbx), %r15 703 / movq WC_RAX(%rbx), %rax 704 movq WC_RBP(%rbx), %rbp 705 movq WC_RCX(%rbx), %rcx 706 / movq WC_RDX(%rbx), %rdx 707 movq WC_RDI(%rbx), %rdi 708 movq WC_RSI(%rbx), %rsi 709 710 711 / assume that %cs does not need to be restored 712 / %ds, %es & %ss are ignored in 64bit mode 713 movw WC_SS(%rbx), %ss 714 movw WC_DS(%rbx), %ds 715 movw WC_ES(%rbx), %es 716 717 #if LED 718 movw $WC_LED, %dx 719 movb $0xdf, %al 720 outb (%dx) 721 #endif 722 723 #if SERIAL 724 movw $WC_COM, %dx 725 movb $0x6f, %al 726 outb (%dx) 727 #endif 728 729 730 movq WC_RBP(%rbx), %rbp 731 movq WC_RSP(%rbx), %rsp 732 733 #if LED 734 movw $WC_LED, %dx 735 movb $0xe0, %al 736 outb (%dx) 737 #endif 738 739 #if SERIAL 740 movw $WC_COM, %dx 741 movb $0x70, %al 742 outb (%dx) 743 #endif 744 745 746 movq WC_RCX(%rbx), %rcx 747 748 pushq WC_EFLAGS(%rbx) / restore flags 749 popfq 750 751 #if LED 752 movw $WC_LED, %dx 753 movb $0xe1, %al 754 outb (%dx) 755 #endif 756 757 #if SERIAL 758 movw $WC_COM, %dx 759 movb $0x71, %al 760 outb (%dx) 761 #endif 762 763 /* 764 * can not use outb after this point, because doing so would mean using 765 * %dx which would modify %rdx which is restored here 766 */ 767 768 movq %rbx, %rax 769 movq WC_RDX(%rax), %rdx 770 movq WC_RBX(%rax), %rbx 771 772 leave 773 774 movq WC_RETADDR(%rax), %rax 775 movq %rax, (%rsp) / return to caller of wc_save_context 776 777 xorl %eax, %eax / at wakeup return 0 778 ret 779 780 781 SET_SIZE(wc_rm_start) 782 783 ENTRY_NP(asmspin) 784 785 movl %edi, %ecx 786 A1: 787 loop A1 788 789 SET_SIZE(asmspin) 790 791 .globl wc_rm_end 792 wc_rm_end: 793 nop 794